Compare commits

..
Author SHA1 Message Date
Ian 1063a2851e feat(llm): stop sending images for the rest of the run once the model rejects one 2026-10-07 15:42:51 -04:00
Ian 9fda996d1f fix(execution): match only replies that mean the model takes no images 2026-10-07 15:42:21 -04:00
Ian 8b6b63f596 fix(execution): only strip images on image-rejection errors
The image-strip recovery fired on any 400/404/422, wiping a session's screenshots and replaying the turn on unrelated errors (e.g. an upstream incomplete-tool-call 400, a context overflow). Require an image-related message too.
2026-10-07 15:42:21 -04:00
Ian b775adb5e3 feat(llm): scrub images from every request to a text-only model
Replace images at the MCP call before any result_transform, give load_skill the
same agent_browser screenshot swap as the system prompt, and add a
call_model_input_filter that turns any image left in the input into a tool error
while chaining any filter already set.
2026-10-07 15:42:21 -04:00
Ian 247fcdd74e Tweaked skill 2026-10-07 15:42:21 -04:00
Ian 76baba8edf refactor(skills): render opt-in Jinja skills instead of comment markers 2026-10-07 15:42:21 -04:00
Ian e543884adf refactor(llm): disable view_image via is_enabled and gate skill image guidance by modality 2026-10-07 15:42:21 -04:00
Ian cda895e99f feat(llm): keep images away from text-only models
Leave view_image out of the tools, replace MCP image blocks with a text note,
and swap agent_browser's screenshot section for text-only guidance when the
run's model does not accept images.
2026-10-07 15:42:21 -04:00
Ian a6666c5848 style: drop a trailing space in the image-support docstring 2026-10-07 15:42:20 -04:00
Ian 49eb38b775 refactor(llm): read image support from litellm's catalog, assuming it for unknown models 2026-10-07 15:33:59 -04:00
Ian 4d1803a72c feat(llm): detect whether a model accepts image input
Add model_supports_images(): OpenRouter models are checked against architecture.input_modalities from OpenRouter's public model list (fetched once per process), others against LiteLLM's supports_vision. Unknown models and lookup failures count as image-capable. Not wired in yet.
2026-10-06 21:33:14 -04:00
21 changed files with 375 additions and 70 deletions

No files matched your search

+10
View File
@@ -18,6 +18,7 @@ from pydantic import ValidationError
from strix.agents.prompt import render_system_prompt
from strix.config import load_settings
from strix.config.models import images_allowed
from strix.tools.agents_graph.tools import (
agent_finish,
create_agent,
@@ -375,11 +376,16 @@ def _configure_filesystem_tools(
)
def _view_image_enabled(ctx: Any, _agent: Any) -> bool:
return images_allowed(ctx.context)
def _make_filesystem_configurator(*, chat_completions: bool, strict_schemas: bool) -> Any:
def configure(toolset: Any) -> None:
_configure_filesystem_tools(
toolset, chat_completions=chat_completions, strict_schemas=strict_schemas
)
toolset.view_image.is_enabled = _view_image_enabled
return configure
@@ -670,6 +676,7 @@ def build_strix_agent(
system_prompt_context: dict[str, Any] | None = None,
extra_tools: Sequence[Tool] | None = None,
instructions_override: str | None = None,
supports_images: bool = True,
) -> SandboxAgent[Any]:
"""Build a SandboxAgent for either root or child use.
@@ -694,6 +701,7 @@ def build_strix_agent(
is_diff_scoped=is_diff_scoped,
interactive=interactive,
system_prompt_context=system_prompt_context,
supports_images=supports_images,
)
agent_tools = [*_EXTRA_TOOLS, *(extra_tools or [])]
@@ -754,6 +762,7 @@ def make_child_factory(
chat_completions_tools: bool = False,
strict_tool_schemas: bool = True,
system_prompt_context: dict[str, Any] | None = None,
supports_images: bool = True,
) -> Any:
"""Return the runner-owned builder used by ``spawn_child_agent``.
@@ -774,6 +783,7 @@ def make_child_factory(
chat_completions_tools=chat_completions_tools,
strict_tool_schemas=strict_tool_schemas,
system_prompt_context=system_prompt_context,
supports_images=supports_images,
)
return _factory
+2 -1
View File
@@ -85,6 +85,7 @@ def render_system_prompt(
interactive: bool = False,
system_prompt_context: dict[str, Any] | None = None,
include_scope: bool = True,
supports_images: bool = True,
) -> str:
"""Render the system prompt. Returns empty string on template failure.
@@ -119,7 +120,7 @@ def render_system_prompt(
is_root=is_root,
is_diff_scoped=is_diff_scoped,
)
skill_content = load_skills(skills_to_load)
skill_content = load_skills(skills_to_load, supports_images=supports_images)
env.globals["get_skill"] = lambda name: skill_content.get(name, "")
# Skills every agent of this kind loads come first, so siblings share them
+23
View File
@@ -7,6 +7,7 @@ import contextlib
import inspect
import logging
import os
import threading
import time
import uuid
from collections.abc import AsyncGenerator
@@ -905,6 +906,28 @@ def model_supports_reasoning(model_name: str) -> bool:
return bool(entry and entry.get("supports_reasoning"))
def model_supports_images(model_name: str) -> bool:
"""Return whether the model accepts image input. Assume yes until proven otherwise."""
entry = _catalog_entry(model_name)
return entry is None or bool(entry.get("supports_vision"))
# Set once the model rejects an image; one process is one run, so this covers every agent.
_IMAGES_REJECTED = threading.Event()
def mark_images_rejected() -> None:
if not _IMAGES_REJECTED.is_set():
logger.warning("The model rejected image input; sending no images for the rest of the run")
_IMAGES_REJECTED.set()
def images_allowed(context: object) -> bool:
"""Whether a request may carry images: the model should take them and hasn't refused one."""
supported = context.get("supports_images", True) if isinstance(context, dict) else True
return bool(supported) and not _IMAGES_REJECTED.is_set()
def _bare_openai_name(model_name: str) -> str:
name = model_name.strip().lower()
for prefix in ("litellm/", "any-llm/", "openai/"):
+49 -6
View File
@@ -4,9 +4,12 @@ from __future__ import annotations
import asyncio
import contextlib
import inspect
import logging
import re
import uuid
from collections.abc import Callable
from dataclasses import replace
from functools import cache
from typing import TYPE_CHECKING, Any, cast
@@ -20,6 +23,7 @@ from openai import (
)
from strix.config import codex
from strix.config.models import images_allowed, mark_images_rejected
from strix.core.hooks import (
BudgetExceededError,
BudgetPausedError,
@@ -30,6 +34,7 @@ from strix.core.sessions import (
enforce_image_budget,
open_agent_session,
replace_session_items,
scrub_images_from_items,
seed_initial_input,
strip_all_images_from_session,
)
@@ -44,6 +49,7 @@ if TYPE_CHECKING:
from agents.lifecycle import RunHooks
from agents.memory import Session, SQLiteSession
from agents.result import RunResultBase
from agents.run_config import CallModelData, ModelInputData
from strix.core.agents import AgentCoordinator, Status
@@ -53,6 +59,16 @@ logger = logging.getLogger(__name__)
StreamEventSink = Callable[[str, Any], None]
_INPUT_REJECTION_CODES = frozenset({400, 404, 422})
# Replies meaning "this model takes no images", not image errors in general: a
# context overflow that counts "image/vision expansion" must not match.
_IMAGE_REJECTION = re.compile(
r"no endpoints found that support image input" # OpenRouter
r"|image_url is only supported by certain models" # OpenAI
r"|is not a multimodal model|at most 0 image\(s\)" # vLLM
r"|does not support image input" # LiteLLM's Fireworks check
r"|doesn't support the image field", # Bedrock Converse
re.IGNORECASE,
)
_MAX_COMPACTIONS_PER_CYCLE = 2
@@ -120,6 +136,28 @@ async def _compact_session(
)
_TEXT_ONLY_IMAGE_TEXT = "[error: this model cannot view images; use `snapshot -i` instead]"
def _with_image_scrub(run_config: RunConfig, context: dict[str, Any]) -> RunConfig:
if images_allowed(context):
return run_config
# Chain any filter already set; it sees the scrubbed input.
inner = run_config.call_model_input_filter
async def _scrub(data: CallModelData[Any]) -> ModelInputData:
model_data = replace(
data.model_data,
input=scrub_images_from_items(data.model_data.input, text=_TEXT_ONLY_IMAGE_TEXT),
)
if inner is None:
return model_data
result = inner(replace(data, model_data=model_data))
return await result if inspect.isawaitable(result) else result
return replace(run_config, call_model_input_filter=_scrub)
_MAX_TRANSIENT_MODEL_RETRIES = 5
_TRANSIENT_MODEL_RETRY_BASE_DELAY_S = 2.0
_TRANSIENT_MODEL_RETRY_MAX_DELAY_S = 90.0
@@ -130,6 +168,12 @@ def _model_error_status_code(exc: BaseException) -> int | None:
return code if isinstance(code, int) else None
def _is_image_rejection(exc: BaseException) -> bool:
return _model_error_status_code(exc) in _INPUT_REJECTION_CODES and bool(
_IMAGE_REJECTION.search(str(exc))
)
def _is_transient_model_error(exc: BaseException) -> bool:
if codex.is_content_guardrail_error(exc):
return False
@@ -747,7 +791,7 @@ async def _run_cycle( # noqa: PLR0912, PLR0915
stream = Runner.run_streamed(
agent,
input=input_data,
run_config=run_config,
run_config=_with_image_scrub(run_config, context),
context=context,
max_turns=max_turns,
session=session,
@@ -805,11 +849,10 @@ async def _run_cycle( # noqa: PLR0912, PLR0915
await coordinator.trigger_budget_stop()
raise
except Exception as exc:
if (
image_strips < 3
and session is not None
and getattr(exc, "status_code", None) in _INPUT_REJECTION_CODES
):
image_rejected = _is_image_rejection(exc)
if image_rejected:
mark_images_rejected()
if image_strips < 3 and session is not None and image_rejected:
try:
stripped = await strip_all_images_from_session(session)
except Exception:
+10
View File
@@ -23,6 +23,7 @@ from strix.config.models import (
StrixProvider,
configure_sdk_api_route,
configure_sdk_model_defaults,
model_supports_images,
supports_strict_tool_schemas,
uses_chat_completions_tool_schema,
)
@@ -153,6 +154,7 @@ def _compose_root_instructions_override(
is_diff_scoped: bool,
interactive: bool,
system_prompt_context: dict[str, Any],
supports_images: bool,
) -> str | None:
if root_instructions_override is None:
return None
@@ -166,6 +168,7 @@ def _compose_root_instructions_override(
interactive=interactive,
system_prompt_context=system_prompt_context,
include_scope=False,
supports_images=supports_images,
)
return (
f"{base_instructions}\n\n"
@@ -265,6 +268,9 @@ async def run_strix_scan(
strict_tool_schemas = supports_strict_tool_schemas(resolved_model)
if not strict_tool_schemas:
logger.info("Sending non-strict tool schemas: %s caps strict tools", resolved_model)
supports_images = model_supports_images(resolved_model)
if not supports_images:
logger.info("Leaving out image tools: %s does not accept images", resolved_model)
if budget_policy not in ("stop", "pause"):
raise ValueError(f"unknown budget_policy: {budget_policy!r}")
@@ -468,6 +474,7 @@ async def run_strix_scan(
is_diff_scoped=is_diff_scoped,
interactive=interactive,
system_prompt_context=root_context,
supports_images=supports_images,
)
root_agent = build_strix_agent(
@@ -482,6 +489,7 @@ async def run_strix_scan(
strict_tool_schemas=strict_tool_schemas,
system_prompt_context=root_context,
instructions_override=root_instructions,
supports_images=supports_images,
)
if not is_resume:
@@ -501,6 +509,7 @@ async def run_strix_scan(
chat_completions_tools=chat_completions_tools,
strict_tool_schemas=strict_tool_schemas,
system_prompt_context=scope_context,
supports_images=supports_images,
)
async def spawn_child_agent(**kwargs: Any) -> dict[str, Any]:
@@ -528,6 +537,7 @@ async def run_strix_scan(
"spawn_child_agent": spawn_child_agent,
"scan_targets": build_scan_targets(scan_config),
"max_context_images": settings.runtime.max_context_images,
"supports_images": supports_images,
}
root_session = open_agent_session(root_id, agents_db)
+2 -2
View File
@@ -199,13 +199,13 @@ async def enforce_image_budget(session: Session, max_images: int) -> bool:
return await _rewrite_session(session, _transform)
def scrub_images_from_items(items: list[Any]) -> list[Any]:
def scrub_images_from_items(items: list[Any], *, text: str = _INHERITED_IMAGE_TEXT) -> list[Any]:
"""Return a copy of ``items`` with every image block replaced by text."""
def _scrub(obj: Any) -> Any:
if isinstance(obj, dict):
if obj.get("type") == "input_image":
return {"type": "input_text", "text": _INHERITED_IMAGE_TEXT}
return {"type": "input_text", "text": text}
return {k: _scrub(v) for k, v in obj.items()}
if isinstance(obj, list):
return [_scrub(v) for v in obj]
+9 -20
View File
@@ -15,11 +15,7 @@ from typing import TYPE_CHECKING, Any
from agents.model_settings import ModelSettings
from agents.models.interface import ModelTracing
from openai.types.responses import (
ResponseOutputMessage,
ResponseOutputText,
ResponseReasoningItem,
)
from openai.types.responses import ResponseOutputMessage, ResponseOutputText
from strix.config import load_settings
from strix.config.models import StrixProvider
@@ -292,22 +288,15 @@ def _checkpoint_item(summary: str) -> dict[str, Any]:
def _extract_text(response: ModelResponse) -> str:
parts: list[str] = []
reasoning: list[str] = []
for item in response.output:
if isinstance(item, ResponseOutputMessage):
parts.extend(
chunk.text
for chunk in item.content
if isinstance(chunk, ResponseOutputText) and chunk.text
)
elif isinstance(item, ResponseReasoningItem):
reasoning.extend(summary.text for summary in item.summary if summary.text)
if parts or not reasoning:
return "".join(parts)
# Reasoning models can spend the whole output budget thinking and return no
# message; their reasoning is a degraded but usable summary.
logger.warning("compaction summary had no output text; using reasoning text")
return "\n".join(reasoning)
if not isinstance(item, ResponseOutputMessage):
continue
parts.extend(
chunk.text
for chunk in item.content
if isinstance(chunk, ResponseOutputText) and chunk.text
)
return "".join(parts)
async def _summarize(model: str, prompt: str, max_tokens: int) -> str | None:
+15 -2
View File
@@ -6,6 +6,7 @@ from pathlib import Path
from typing import TypeGuard
import yaml
from jinja2 import Environment, StrictUndefined
from strix.utils.resource_paths import get_strix_resource_path
@@ -13,6 +14,14 @@ from strix.utils.resource_paths import get_strix_resource_path
logger = logging.getLogger(__name__)
_FRONTMATTER_PATTERN = re.compile(r"^---\s*\n(?P<body>.*?)\n---\s*\n", re.DOTALL)
# Only skills with `template: jinja` in their frontmatter are rendered; the rest hold
# literal `{{ }}` payloads.
_SKILL_TEMPLATES = Environment(
autoescape=False, # noqa: S701 - prompts, not HTML
undefined=StrictUndefined,
trim_blocks=True,
lstrip_blocks=True,
)
_INTERNAL_SKILL_CATEGORIES: frozenset[str] = frozenset({"scan_modes", "coordination", "analysis"})
_ROOT_SKILL_CATEGORY = "root"
@@ -261,7 +270,7 @@ def _candidate_skill_files(skill_name: str) -> list[Path]:
return _bare_skill_files(skill_name)
def load_skills(skill_names: list[str]) -> dict[str, str]:
def load_skills(skill_names: list[str], *, supports_images: bool = True) -> dict[str, str]:
"""Load skill markdown bodies (frontmatter stripped) by name.
Skill files live at ``strix/skills/<category>/<name>.md`` (or any
@@ -291,7 +300,11 @@ def load_skills(skill_names: list[str]) -> dict[str, str]:
continue
var_name = skill_name.split("/")[-1]
_, skill_body = _parse_skill_content(content, file_path)
metadata, skill_body = _parse_skill_content(content, file_path)
if metadata.get("template") == "jinja":
skill_body = _SKILL_TEMPLATES.from_string(skill_body).render(
supports_images=supports_images
)
skill_content[var_name] = skill_body
logger.debug("Loaded skill: %s -> %s", skill_name, var_name)
_track_skill_loaded(var_name, file_path)
+15 -6
View File
@@ -1,6 +1,7 @@
---
name: agent_browser
description: agent-browser CLI for headless Chrome via shell. Snapshot-and-ref workflow, click/fill/extract, screenshots, multi-tab, multi-session, network mocking. Pre-installed in the sandbox; invoke via exec_command.
template: jinja
---
@@ -260,8 +261,9 @@ only for simple expressions.
### Screenshot
`agent-browser screenshot` writes a PNG to disk in the sandbox. The
shell command alone does **not** put the image into your context —
`agent-browser screenshot` writes a PNG to disk in the sandbox.
{% if supports_images %}
The shell command alone does **not** put the image into your context —
chain it with the SDK ``view_image`` tool to actually see it:
```bash
@@ -275,6 +277,12 @@ the full path on stdout — pass that to ``view_image``). If you need a
specific filename, keep it inside that directory or a sibling hidden
dir under ``/workspace``. Never write screenshots to ``/tmp`` —
``view_image`` rejects anything outside the workspace root.
{% else %}
You are running on a text-only model and cannot view images. Screenshots
are only useful to you as a saved file (for example, login evidence).
Drive the page from `snapshot -i` refs, `eval` for any DOM/JS state you
need to read, and `text @ref` / `get text` for content extraction.
{% endif %}
```bash
agent-browser screenshot # path printed on stdout
@@ -283,6 +291,7 @@ agent-browser screenshot --full # full scroll height
agent-browser screenshot --annotate # numbered labels + legend keyed to snapshot refs
```
{% if supports_images %}
`--annotate` is designed for multimodal models: each label `[N]` maps
to ref `@eN`. Take the annotated screenshot, then ``view_image`` it,
and you can correlate visual layout with snapshot refs.
@@ -294,10 +303,10 @@ layout questions, captchas, custom widgets where the accessibility
tree is incomplete).
If ``view_image`` errors back at you (rejected image, "vision not
supported", or similar), you are running on a text-only model — stop
calling it and stop taking screenshots. Drive the page entirely from
`snapshot -i` refs, `eval` for any DOM/JS state you need to read, and
`text @ref` / `get text` for content extraction.
supported", or similar), you can drive the page from `snapshot -i`
refs, `eval` for any DOM/JS state you need to read, and `text @ref`
or `get text` for content extraction.
{% endif %}
### Handle multiple pages via tabs
+2 -2
View File
@@ -4,6 +4,7 @@ from __future__ import annotations
from agents import RunContextWrapper, function_tool
from strix.config.models import images_allowed
from strix.skills import load_skills, validate_requested_skills
@@ -24,12 +25,11 @@ async def load_skill(ctx: RunContextWrapper, skills: list[str]) -> str:
Max 5. Names match the bare files under
``strix/skills/<category>/<name>.md``.
"""
del ctx
requested = list(skills or [])
err = validate_requested_skills(requested)
if err:
return f"load_skill: {err}"
contents = load_skills(requested)
contents = load_skills(requested, supports_images=images_allowed(ctx.context))
if not contents:
return "load_skill: no content loaded for requested skills."
sections = [f"## Skill: {name}\n\n{body}" for name, body in contents.items()]
+2
View File
@@ -29,6 +29,7 @@ from typing import TYPE_CHECKING, Any
from agents import RunContextWrapper, function_tool
from strix.config.models import images_allowed
from strix.tools.mcp.client import _errored_tool_output
from strix.tools.mcp.naming import namespaced_tool_name
from strix.tools.mcp.registry import MCP_REGISTRY_CONTEXT_KEY, McpRegistry
@@ -293,4 +294,5 @@ async def call_mcp(
arguments or {},
label=namespaced_tool_name(connection, tool),
result_transform=entry.result_transform,
supports_images=images_allowed(ctx.context),
)
+16
View File
@@ -32,6 +32,7 @@ from agents.mcp import (
)
from mcp.client.stdio import stdio_client
from mcp.shared._httpx_utils import create_mcp_http_client
from mcp.types import TextContent
from strix.tools.mcp.failures import HttpStatusRecorder
from strix.tools.mcp.session import McpConnectionUnavailableError, SupervisedMcpSession
@@ -183,6 +184,18 @@ def _build_server(config: McpConnectionConfig) -> BuiltMcpServer:
)
def _without_images(result: Any) -> Any:
content = [
TextContent(
type="text", text=f"[{item.mimeType} image omitted: this model cannot view images]"
)
if item.type == "image"
else item
for item in result.content
]
return result.model_copy(update={"content": content})
def _mcp_result_to_tool_output(server: MCPServer, result: Any) -> Any:
"""Serialize a ``CallToolResult`` to a tool output, mirroring the agents SDK.
@@ -217,6 +230,7 @@ async def dispatch_mcp_call(
*,
label: str,
result_transform: ResultTransform | None = None,
supports_images: bool = True,
) -> Any:
"""Run one MCP tool call and convert its result to a tool output.
@@ -233,6 +247,8 @@ async def dispatch_mcp_call(
corrupt the content the agent receives).
"""
result = await server.call_tool(tool_name, arguments)
if not supports_images:
result = _without_images(result)
if result_transform is not None:
return result_transform(label, result.model_dump(mode="json"))
tool_output = _mcp_result_to_tool_output(server, result)
+2
View File
@@ -365,6 +365,7 @@ class SupervisedMcpSession:
*,
label: str,
result_transform: ResultTransform | None = None,
supports_images: bool = True,
) -> Any:
"""Run one tool call with bounded retries for transient session failures.
@@ -382,6 +383,7 @@ class SupervisedMcpSession:
arguments,
label=label,
result_transform=result_transform,
supports_images=supports_images,
)
outcome = await self._run_job(job, phase="call")
+17 -1
View File
@@ -3,6 +3,7 @@
from __future__ import annotations
import json
import threading
from types import SimpleNamespace
from typing import Any, cast
@@ -10,7 +11,7 @@ import pytest
from agents.tool import CustomTool, FunctionTool
from strix.agents import factory
from strix.config import load_settings
from strix.config import load_settings, models
def _capturing_exec_tool(captured: dict[str, str]) -> FunctionTool:
@@ -115,3 +116,18 @@ def test_function_tools_are_result_bounded() -> None:
by_name = {t.name: t for t in agent.tools}
assert getattr(by_name["think"], "_strix_bounded", False) is True
def test_view_image_is_left_out_once_images_are_not_allowed(
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setattr(models, "_IMAGES_REJECTED", threading.Event())
filesystem = factory.build_strix_agent(is_root=True).capabilities[0]
filesystem.bind(cast("Any", object()))
view_image = next(tool for tool in filesystem.tools() if tool.name == "view_image")
is_enabled = cast("Any", cast("FunctionTool", view_image).is_enabled)
assert is_enabled(SimpleNamespace(context={"supports_images": True}), None) is True
assert is_enabled(SimpleNamespace(context={"supports_images": False}), None) is False
models.mark_images_rejected()
assert is_enabled(SimpleNamespace(context={"supports_images": True}), None) is False
-25
View File
@@ -6,8 +6,6 @@ from types import SimpleNamespace
from typing import TYPE_CHECKING, Any
import pytest
from agents.extensions.models.litellm_model import InternalChatCompletionMessage
from agents.models.chatcmpl_converter import Converter
from litellm.exceptions import BadRequestError, ContextWindowExceededError, RateLimitError
from openai.types.responses import ResponseOutputMessage, ResponseOutputText
@@ -244,29 +242,6 @@ async def test_summarize_routes_through_provider_with_settings(
assert settings.max_tokens == 64
@pytest.mark.asyncio
async def test_summarize_falls_back_to_reasoning_when_no_output_text(
monkeypatch: pytest.MonkeyPatch,
) -> None:
_patch_budget(monkeypatch, keep_tokens=30, window=4_000)
message = InternalChatCompletionMessage(
role="assistant", content=None, reasoning_content="REASONED SUMMARY"
)
response = SimpleNamespace(output=Converter.message_to_output_items(message))
class FakeModel:
async def get_response(self, **_kwargs: Any) -> Any:
return response
class FakeProvider:
def get_model(self, _model_name: str | None) -> Any:
return FakeModel()
monkeypatch.setattr(compaction, "StrixProvider", FakeProvider)
assert await compaction._summarize("m", "p", 64) == "REASONED SUMMARY"
def test_fit_to_tokens_truncates_oversized_text(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(compaction, "count_tokens", lambda _m, t: len(t))
text = "x" * 10_000
+31
View File
@@ -9,9 +9,11 @@ from typing import Any, cast
from unittest.mock import MagicMock
import pytest
from agents import RunConfig
from agents.exceptions import MaxTurnsExceeded
from agents.items import MessageOutputItem
from agents.memory import SQLiteSession
from agents.run_config import CallModelData, ModelInputData
from agents.tool_context import ToolContext
from openai.types.responses import ResponseOutputMessage, ResponseOutputRefusal, ResponseOutputText
@@ -1531,3 +1533,32 @@ async def test_autonomous_nudge_does_not_offer_the_user() -> None:
)
assert "wait_for_user" not in items[0]["content"]
@pytest.mark.asyncio
async def test_text_only_filter_scrubs_images_then_chains_existing_filter() -> None:
run_config = RunConfig(model="m")
assert execution._with_image_scrub(run_config, {"supports_images": True}) is run_config
seen: list[ModelInputData] = []
def _inner(data: CallModelData[Any]) -> ModelInputData:
seen.append(data.model_data)
return data.model_data
run_config = RunConfig(model="m", call_model_input_filter=_inner)
scrub: Any = execution._with_image_scrub(run_config, {"supports_images": False})
image = {"type": "input_image", "image_url": "data:image/png;base64,aGk="}
output = {"type": "function_call_output", "call_id": "c1", "output": [image]}
data = CallModelData(
model_data=ModelInputData(input=[cast("Any", output)], instructions="sys"),
agent=MagicMock(),
context=None,
)
result = await scrub.call_model_input_filter(data)
assert result is seen[0]
assert result.input[0]["output"] == [
{"type": "input_text", "text": execution._TEXT_ONLY_IMAGE_TEXT}
]
assert data.model_data.input[0]["output"] == [image]
+79 -4
View File
@@ -1,6 +1,7 @@
from __future__ import annotations
from typing import Any, cast
import threading
from typing import Any
import httpx
import pytest
@@ -15,7 +16,7 @@ from openai import (
RateLimitError,
)
from strix.config import codex
from strix.config import codex, models
from strix.core import execution
from strix.core.agents import AgentCoordinator
@@ -109,6 +110,7 @@ def _patch_fast_backoff(monkeypatch: pytest.MonkeyPatch) -> None:
async def _run_once(
monkeypatch: pytest.MonkeyPatch,
streams: list[_FakeStream],
session: Any = None,
) -> Any:
_patch_fast_backoff(monkeypatch)
calls = {"n": 0}
@@ -128,10 +130,10 @@ async def _run_once(
coordinator,
"root",
input_data="task",
run_config=cast("RunConfig", object()),
run_config=RunConfig(),
context={},
max_turns=5,
session=None,
session=session,
interactive=False,
event_sink=None,
hooks=None,
@@ -172,3 +174,76 @@ async def test_run_cycle_does_not_retry_permanent_error(
streams = [_FakeStream(exc=bad_request), _FakeStream()]
with pytest.raises(BadRequestError):
await _run_once(monkeypatch, streams)
@pytest.mark.parametrize(
("status", "message", "expected"),
[
(
404,
"litellm.NotFoundError: NotFoundError: OpenrouterException - "
'{"error":{"message":"No endpoints found that support image input","code":404,'
'"metadata":{"failed_routing_step":"Filter by Image Support"}}}',
True,
),
(400, "Invalid content type. image_url is only supported by certain models.", True),
(400, "my-model is not a multimodal model", True),
(400, "This model doesn't support the image field for user messages.", True),
(
400,
"OpenRouterException: Message: This model's maximum context length is 1048576 "
"tokens, but the request requires 1073484 tokens (942412 input including "
"image/vision expansion + 131072 for the completion). Reduce the input length "
"or max_tokens.",
False,
),
(400, "At most 5 image(s) may be provided in one prompt.", False),
(
400,
"Upstream error from Relace: Generation stopped with an incomplete tool call",
False,
),
(500, "No endpoints found that support image input", False),
],
)
def test_is_image_rejection(status: int, message: str, expected: bool) -> None:
exc = APIStatusError(message, response=httpx.Response(status, request=_request()), body=None)
assert execution._is_image_rejection(exc) is expected
async def _run_with_image_session(
monkeypatch: pytest.MonkeyPatch, streams: list[_FakeStream]
) -> tuple[Any, int, int]:
strips = {"n": 0}
async def _fake_strip(_session: Any) -> bool:
strips["n"] += 1
return True
async def _no_compact(*_args: Any, **_kwargs: Any) -> bool:
return False
monkeypatch.setattr(execution, "strip_all_images_from_session", _fake_strip)
monkeypatch.setattr(execution, "_compact_session", _no_compact)
result, attempts, _coordinator = await _run_once(monkeypatch, streams, session=object())
return result, attempts, strips["n"]
@pytest.mark.asyncio
async def test_run_cycle_strips_images_on_image_rejection(
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setattr(models, "_IMAGES_REJECTED", threading.Event())
rejection = APIStatusError(
"No endpoints found that support image input",
response=httpx.Response(status_code=404, request=_request()),
body=None,
)
streams = [_FakeStream(exc=rejection), _FakeStream()]
result, attempts, strips = await _run_with_image_session(monkeypatch, streams)
assert result is streams[1]
assert attempts == 2
assert strips == 1
# Every later request in the run, from any agent, has its images scrubbed.
assert execution._with_image_scrub(RunConfig(), {}).call_model_input_filter is not None
+46 -1
View File
@@ -18,7 +18,7 @@ from typing import TYPE_CHECKING, Any
import pytest
from agents.mcp import MCPServer, MCPServerStdio, MCPServerStreamableHttp
from agents.tool_context import ToolContext
from mcp.types import CallToolResult, TextContent
from mcp.types import CallToolResult, ImageContent, TextContent
from mcp.types import Tool as MCPTool
from pydantic import ValidationError
@@ -884,6 +884,51 @@ async def test_call_mcp_flags_an_errored_result_failed_for_the_tui() -> None:
assert out == {"type": "text", "text": "boom:read_file", "success": False}
@pytest.mark.asyncio
async def test_call_mcp_replaces_images_for_a_text_only_model() -> None:
class _ImageServer(FakeMCPServer):
async def call_tool(
self,
tool_name: str,
arguments: dict[str, Any] | None,
meta: dict[str, Any] | None = None,
) -> CallToolResult:
return CallToolResult(
content=[
TextContent(type="text", text="shot"),
ImageContent(type="image", data="aGk=", mimeType="image/png"),
]
)
registry = McpRegistry()
registry.add(
name="fs", server=_ImageServer("fs", [_mcp_tool("shot")]), purpose=None, tool_count=1
)
ctx = _ctx(registry)
args = json.dumps({"connection": "fs", "tool": "shot"})
out = await call_mcp.on_invoke_tool(ctx, args)
assert out[1] == {"type": "image", "image_url": "data:image/png;base64,aGk="}
ctx.context["supports_images"] = False
out = await call_mcp.on_invoke_tool(ctx, args)
assert out == [
{"type": "text", "text": "shot"},
{"type": "text", "text": "[image/png image omitted: this model cannot view images]"},
]
transformed: list[Any] = []
registry.add(
name="pro",
server=_ImageServer("pro", [_mcp_tool("shot")]),
purpose=None,
tool_count=1,
result_transform=lambda _label, result: transformed.append(result),
)
await call_mcp.on_invoke_tool(ctx, json.dumps({"connection": "pro", "tool": "shot"}))
assert [block["type"] for block in transformed[0]["content"]] == ["text", "text"]
# --- generic MCP tools are the only MCP surface every agent gets -------------
+9
View File
@@ -16,6 +16,7 @@ from strix.config.models import (
_NonStreamingModel,
_TurnGuardModel,
configure_sdk_model_defaults,
model_supports_images,
request_timeout_extra_args,
resolve_api_type,
routes_through_litellm,
@@ -202,3 +203,11 @@ def test_configure_sdk_api_route_follows_the_given_model(
models.configure_sdk_api_route("my-private-model", settings)
assert routes == ["responses", "chat_completions"]
def test_image_support_follows_the_catalog(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setitem(litellm.model_cost, "acme-text", {})
monkeypatch.setitem(litellm.model_cost, "acme-vision", {"supports_vision": True})
assert model_supports_images("acme-text") is False
assert model_supports_images("litellm/openai/acme-vision") is True
assert model_supports_images("acme-unknown-model") is True
+25
View File
@@ -13,6 +13,7 @@ from typing import Any
import httpx
import pytest
from agents import ModelSettings
from agents.tool_context import ToolContext
from openai import RateLimitError
import strix.tools.mcp as mcp_pkg
@@ -24,6 +25,7 @@ from strix.core import runner
from strix.core.agents import AgentCoordinator
from strix.core.inputs import make_model_settings
from strix.runtime import session_manager
from strix.tools.load_skill.tool import load_skill
from strix.tools.mcp import BearerAuth, McpConnectionConfig, McpConnectionRequest
@@ -287,6 +289,29 @@ def test_requested_skills_follow_the_shared_prefix() -> None:
assert "<xss>" in xss.split("<cache_point>")[1]
def test_text_only_prompt_drops_screenshot_guidance() -> None:
assert "view_image" in render_system_prompt(include_scope=False)
prompt = render_system_prompt(include_scope=False, supports_images=False)
assert "view_image" not in prompt
assert "text-only model and cannot view images" in prompt
assert "<!--" not in prompt
@pytest.mark.asyncio
async def test_text_only_load_skill_drops_screenshot_guidance() -> None:
ctx = ToolContext(
context={"supports_images": False},
tool_name="load_skill",
tool_call_id="call-1",
tool_arguments="{}",
)
out = await load_skill.on_invoke_tool(ctx, '{"skills": ["agent_browser"]}')
assert "view_image" not in out
assert "text-only model" in out
def test_scope_is_sent_as_its_own_system_message_on_cache_point_routes() -> None:
settings = make_model_settings(None, model_name="anthropic/claude-sonnet-5-5")
prompt = render_system_prompt(
+11
View File
@@ -70,6 +70,17 @@ def test_registered_dir_adds_new_skill(tmp_path: Path) -> None:
assert load_skills(["widget"]) == {"widget": "widget body"}
def test_only_jinja_skills_are_rendered(tmp_path: Path) -> None:
body = "{% if supports_images %}\nlook\n{% else %}\nread\n{% endif %}\n"
_write_skill(tmp_path, "extra", "plain", body)
_write_skill(tmp_path, "extra", "templated", f"---\ntemplate: jinja\n---\n{body}")
register_skill_dir(tmp_path)
assert load_skills(["plain"]) == {"plain": body}
assert load_skills(["templated"]) == {"templated": "look\n"}
assert load_skills(["templated"], supports_images=False) == {"templated": "read\n"}
def test_available_skill_includes_frontmatter_description(tmp_path: Path) -> None:
_write_skill(
tmp_path,