Compare commits

..
Author SHA1 Message Date
Ian a6666c5848 style: drop a trailing space in the image-support docstring 2026-10-07 15:42:20 -04:00
Ian 49eb38b775 refactor(llm): read image support from litellm's catalog, assuming it for unknown models 2026-10-07 15:33:59 -04:00
Ian 4d1803a72c feat(llm): detect whether a model accepts image input
Add model_supports_images(): OpenRouter models are checked against architecture.input_modalities from OpenRouter's public model list (fetched once per process), others against LiteLLM's supports_vision. Unknown models and lookup failures count as image-capable. Not wired in yet.
2026-10-06 21:33:14 -04:00
21 changed files with 23 additions and 402 deletions

No files matched your search

+1 -10
View File
@@ -375,15 +375,11 @@ def _configure_filesystem_tools(
)
def _make_filesystem_configurator(
*, chat_completions: bool, strict_schemas: bool, supports_images: bool = True
) -> Any:
def _make_filesystem_configurator(*, chat_completions: bool, strict_schemas: bool) -> Any:
def configure(toolset: Any) -> None:
_configure_filesystem_tools(
toolset, chat_completions=chat_completions, strict_schemas=strict_schemas
)
if not supports_images:
toolset.view_image.is_enabled = False
return configure
@@ -674,7 +670,6 @@ def build_strix_agent(
system_prompt_context: dict[str, Any] | None = None,
extra_tools: Sequence[Tool] | None = None,
instructions_override: str | None = None,
supports_images: bool = True,
) -> SandboxAgent[Any]:
"""Build a SandboxAgent for either root or child use.
@@ -699,7 +694,6 @@ def build_strix_agent(
is_diff_scoped=is_diff_scoped,
interactive=interactive,
system_prompt_context=system_prompt_context,
supports_images=supports_images,
)
agent_tools = [*_EXTRA_TOOLS, *(extra_tools or [])]
@@ -739,7 +733,6 @@ def build_strix_agent(
configure_tools=_make_filesystem_configurator(
chat_completions=chat_completions_tools,
strict_schemas=strict_tool_schemas,
supports_images=supports_images,
),
),
Shell(
@@ -761,7 +754,6 @@ def make_child_factory(
chat_completions_tools: bool = False,
strict_tool_schemas: bool = True,
system_prompt_context: dict[str, Any] | None = None,
supports_images: bool = True,
) -> Any:
"""Return the runner-owned builder used by ``spawn_child_agent``.
@@ -782,7 +774,6 @@ def make_child_factory(
chat_completions_tools=chat_completions_tools,
strict_tool_schemas=strict_tool_schemas,
system_prompt_context=system_prompt_context,
supports_images=supports_images,
)
return _factory
+1 -2
View File
@@ -85,7 +85,6 @@ def render_system_prompt(
interactive: bool = False,
system_prompt_context: dict[str, Any] | None = None,
include_scope: bool = True,
supports_images: bool = True,
) -> str:
"""Render the system prompt. Returns empty string on template failure.
@@ -120,7 +119,7 @@ def render_system_prompt(
is_root=is_root,
is_diff_scoped=is_diff_scoped,
)
skill_content = load_skills(skills_to_load, supports_images=supports_images)
skill_content = load_skills(skills_to_load)
env.globals["get_skill"] = lambda name: skill_content.get(name, "")
# Skills every agent of this kind loads come first, so siblings share them
+6 -46
View File
@@ -4,12 +4,9 @@ from __future__ import annotations
import asyncio
import contextlib
import inspect
import logging
import re
import uuid
from collections.abc import Callable
from dataclasses import replace
from functools import cache
from typing import TYPE_CHECKING, Any, cast
@@ -33,7 +30,6 @@ from strix.core.sessions import (
enforce_image_budget,
open_agent_session,
replace_session_items,
scrub_images_from_items,
seed_initial_input,
strip_all_images_from_session,
)
@@ -48,7 +44,6 @@ if TYPE_CHECKING:
from agents.lifecycle import RunHooks
from agents.memory import Session, SQLiteSession
from agents.result import RunResultBase
from agents.run_config import CallModelData, ModelInputData
from strix.core.agents import AgentCoordinator, Status
@@ -58,17 +53,6 @@ logger = logging.getLogger(__name__)
StreamEventSink = Callable[[str, Any], None]
_INPUT_REJECTION_CODES = frozenset({400, 404, 422})
# Replies meaning "this model takes no images", not image errors in general: a
# context overflow that counts "image/vision expansion" must not match.
_IMAGE_REJECTION = re.compile(
r"no endpoints found that support image input" # OpenRouter
r"|image_url is only supported by certain models" # OpenAI
r"|is not a multimodal model|at most 0 image\(s\)" # vLLM
r"|does not support image input" # LiteLLM's Fireworks check
r"|doesn't support the image field" # Bedrock Converse
r"|unknown variant `image_url`", # DeepSeek, e.g. via Vercel AI Gateway
re.IGNORECASE,
)
_MAX_COMPACTIONS_PER_CYCLE = 2
@@ -136,28 +120,6 @@ async def _compact_session(
)
_TEXT_ONLY_IMAGE_TEXT = "[error: this model cannot view images; use `snapshot -i` instead]"
def _with_image_scrub(run_config: RunConfig, context: dict[str, Any]) -> RunConfig:
if context.get("supports_images", True):
return run_config
# Chain any filter already set; it sees the scrubbed input.
inner = run_config.call_model_input_filter
async def _scrub(data: CallModelData[Any]) -> ModelInputData:
model_data = replace(
data.model_data,
input=scrub_images_from_items(data.model_data.input, text=_TEXT_ONLY_IMAGE_TEXT),
)
if inner is None:
return model_data
result = inner(replace(data, model_data=model_data))
return await result if inspect.isawaitable(result) else result
return replace(run_config, call_model_input_filter=_scrub)
_MAX_TRANSIENT_MODEL_RETRIES = 5
_TRANSIENT_MODEL_RETRY_BASE_DELAY_S = 2.0
_TRANSIENT_MODEL_RETRY_MAX_DELAY_S = 90.0
@@ -168,12 +130,6 @@ def _model_error_status_code(exc: BaseException) -> int | None:
return code if isinstance(code, int) else None
def _is_image_rejection(exc: BaseException) -> bool:
return _model_error_status_code(exc) in _INPUT_REJECTION_CODES and bool(
_IMAGE_REJECTION.search(str(exc))
)
def _is_transient_model_error(exc: BaseException) -> bool:
if codex.is_content_guardrail_error(exc):
return False
@@ -791,7 +747,7 @@ async def _run_cycle( # noqa: PLR0912, PLR0915
stream = Runner.run_streamed(
agent,
input=input_data,
run_config=_with_image_scrub(run_config, context),
run_config=run_config,
context=context,
max_turns=max_turns,
session=session,
@@ -849,7 +805,11 @@ async def _run_cycle( # noqa: PLR0912, PLR0915
await coordinator.trigger_budget_stop()
raise
except Exception as exc:
if image_strips < 3 and session is not None and _is_image_rejection(exc):
if (
image_strips < 3
and session is not None
and getattr(exc, "status_code", None) in _INPUT_REJECTION_CODES
):
try:
stripped = await strip_all_images_from_session(session)
except Exception:
+1 -11
View File
@@ -23,7 +23,6 @@ from strix.config.models import (
StrixProvider,
configure_sdk_api_route,
configure_sdk_model_defaults,
model_supports_images,
supports_strict_tool_schemas,
uses_chat_completions_tool_schema,
)
@@ -154,7 +153,6 @@ def _compose_root_instructions_override(
is_diff_scoped: bool,
interactive: bool,
system_prompt_context: dict[str, Any],
supports_images: bool,
) -> str | None:
if root_instructions_override is None:
return None
@@ -168,7 +166,6 @@ def _compose_root_instructions_override(
interactive=interactive,
system_prompt_context=system_prompt_context,
include_scope=False,
supports_images=supports_images,
)
return (
f"{base_instructions}\n\n"
@@ -268,9 +265,6 @@ async def run_strix_scan(
strict_tool_schemas = supports_strict_tool_schemas(resolved_model)
if not strict_tool_schemas:
logger.info("Sending non-strict tool schemas: %s caps strict tools", resolved_model)
supports_images = model_supports_images(resolved_model)
if not supports_images:
logger.info("Leaving out image tools: %s does not accept images", resolved_model)
if budget_policy not in ("stop", "pause"):
raise ValueError(f"unknown budget_policy: {budget_policy!r}")
@@ -474,7 +468,6 @@ async def run_strix_scan(
is_diff_scoped=is_diff_scoped,
interactive=interactive,
system_prompt_context=root_context,
supports_images=supports_images,
)
root_agent = build_strix_agent(
@@ -489,7 +482,6 @@ async def run_strix_scan(
strict_tool_schemas=strict_tool_schemas,
system_prompt_context=root_context,
instructions_override=root_instructions,
supports_images=supports_images,
)
if not is_resume:
@@ -509,7 +501,6 @@ async def run_strix_scan(
chat_completions_tools=chat_completions_tools,
strict_tool_schemas=strict_tool_schemas,
system_prompt_context=scope_context,
supports_images=supports_images,
)
async def spawn_child_agent(**kwargs: Any) -> dict[str, Any]:
@@ -537,7 +528,6 @@ async def run_strix_scan(
"spawn_child_agent": spawn_child_agent,
"scan_targets": build_scan_targets(scan_config),
"max_context_images": settings.runtime.max_context_images,
"supports_images": supports_images,
}
root_session = open_agent_session(root_id, agents_db)
@@ -602,7 +592,7 @@ async def run_strix_scan(
)
if not interactive and result is not None:
final = getattr(result, "final_output", None)
# Lifecycle tools mark the root completed.
# Lifecycle tools mark the root completed.
async with coordinator._lock:
root_completed = coordinator.statuses.get(root_id) == "completed"
if not root_completed:
+2 -2
View File
@@ -199,13 +199,13 @@ async def enforce_image_budget(session: Session, max_images: int) -> bool:
return await _rewrite_session(session, _transform)
def scrub_images_from_items(items: list[Any], *, text: str = _INHERITED_IMAGE_TEXT) -> list[Any]:
def scrub_images_from_items(items: list[Any]) -> list[Any]:
"""Return a copy of ``items`` with every image block replaced by text."""
def _scrub(obj: Any) -> Any:
if isinstance(obj, dict):
if obj.get("type") == "input_image":
return {"type": "input_text", "text": text}
return {"type": "input_text", "text": _INHERITED_IMAGE_TEXT}
return {k: _scrub(v) for k, v in obj.items()}
if isinstance(obj, list):
return [_scrub(v) for v in obj]
@@ -90,7 +90,6 @@ type Model struct {
client *Client
width, height int
snapshot protocol.Snapshot
windowTitle string
input textarea.Model
viewport viewport.Model
viewportContent string
-27
View File
@@ -2,13 +2,10 @@ package app
import (
"encoding/json"
"fmt"
"os"
"strings"
"time"
tea "github.com/charmbracelet/bubbletea"
"github.com/charmbracelet/x/ansi"
"github.com/usestrix/strix/tui/internal/protocol"
"github.com/usestrix/strix/tui/internal/render"
)
@@ -59,10 +56,6 @@ func (m *Model) handleEnvelope(envelope protocol.Envelope) tea.Cmd {
// resize (not just refresh): status-row visibility changes the chat height.
m.resizeViewport()
m.resizeVulnerabilityViewport()
if title := windowTitle(m.snapshot.Targets); title != m.windowTitle {
m.windowTitle = title
return setTerminalTitle(title)
}
case "collection_bootstrap":
return m.handleCollectionBootstrap(envelope.Payload)
case "collection_delta":
@@ -125,26 +118,6 @@ func (m *Model) handleEnvelope(envelope protocol.Envelope) tea.Cmd {
return nil
}
// tea.SetWindowTitle only sends OSC 2 (window title); iTerm tabs show the OSC 1
// icon name, which shells usually set, so send OSC 0 to set both.
func setTerminalTitle(title string) tea.Cmd {
return func() tea.Msg {
_, _ = os.Stdout.WriteString(ansi.SetIconNameWindowTitle(title))
return nil
}
}
func windowTitle(targets []string) string {
switch len(targets) {
case 0:
return "strix"
case 1:
return targets[0]
default:
return fmt.Sprintf("%s +%d", targets[0], len(targets)-1)
}
}
func (m *Model) consumeMessages(messages []protocol.Message, setupMode bool) {
if m.seenMessages == nil {
m.seenMessages = map[string]bool{}
+2 -15
View File
@@ -6,7 +6,6 @@ from pathlib import Path
from typing import TypeGuard
import yaml
from jinja2 import Environment, StrictUndefined
from strix.utils.resource_paths import get_strix_resource_path
@@ -14,14 +13,6 @@ from strix.utils.resource_paths import get_strix_resource_path
logger = logging.getLogger(__name__)
_FRONTMATTER_PATTERN = re.compile(r"^---\s*\n(?P<body>.*?)\n---\s*\n", re.DOTALL)
# Only skills with `template: jinja` in their frontmatter are rendered; the rest hold
# literal `{{ }}` payloads.
_SKILL_TEMPLATES = Environment(
autoescape=False, # noqa: S701 # nosec B701 - prompts, not HTML
undefined=StrictUndefined,
trim_blocks=True,
lstrip_blocks=True,
)
_INTERNAL_SKILL_CATEGORIES: frozenset[str] = frozenset({"scan_modes", "coordination", "analysis"})
_ROOT_SKILL_CATEGORY = "root"
@@ -270,7 +261,7 @@ def _candidate_skill_files(skill_name: str) -> list[Path]:
return _bare_skill_files(skill_name)
def load_skills(skill_names: list[str], *, supports_images: bool = True) -> dict[str, str]:
def load_skills(skill_names: list[str]) -> dict[str, str]:
"""Load skill markdown bodies (frontmatter stripped) by name.
Skill files live at ``strix/skills/<category>/<name>.md`` (or any
@@ -300,11 +291,7 @@ def load_skills(skill_names: list[str], *, supports_images: bool = True) -> dict
continue
var_name = skill_name.split("/")[-1]
metadata, skill_body = _parse_skill_content(content, file_path)
if metadata.get("template") == "jinja":
skill_body = _SKILL_TEMPLATES.from_string(skill_body).render(
supports_images=supports_images
)
_, skill_body = _parse_skill_content(content, file_path)
skill_content[var_name] = skill_body
logger.debug("Loaded skill: %s -> %s", skill_name, var_name)
_track_skill_loaded(var_name, file_path)
+6 -15
View File
@@ -1,7 +1,6 @@
---
name: agent_browser
description: agent-browser CLI for headless Chrome via shell. Snapshot-and-ref workflow, click/fill/extract, screenshots, multi-tab, multi-session, network mocking. Pre-installed in the sandbox; invoke via exec_command.
template: jinja
---
@@ -261,9 +260,8 @@ only for simple expressions.
### Screenshot
`agent-browser screenshot` writes a PNG to disk in the sandbox.
{% if supports_images %}
The shell command alone does **not** put the image into your context —
`agent-browser screenshot` writes a PNG to disk in the sandbox. The
shell command alone does **not** put the image into your context —
chain it with the SDK ``view_image`` tool to actually see it:
```bash
@@ -277,12 +275,6 @@ the full path on stdout — pass that to ``view_image``). If you need a
specific filename, keep it inside that directory or a sibling hidden
dir under ``/workspace``. Never write screenshots to ``/tmp`` —
``view_image`` rejects anything outside the workspace root.
{% else %}
You are running on a text-only model and cannot view images. Screenshots
are only useful to you as a saved file (for example, login evidence).
Drive the page from `snapshot -i` refs, `eval` for any DOM/JS state you
need to read, and `text @ref` / `get text` for content extraction.
{% endif %}
```bash
agent-browser screenshot # path printed on stdout
@@ -291,7 +283,6 @@ agent-browser screenshot --full # full scroll height
agent-browser screenshot --annotate # numbered labels + legend keyed to snapshot refs
```
{% if supports_images %}
`--annotate` is designed for multimodal models: each label `[N]` maps
to ref `@eN`. Take the annotated screenshot, then ``view_image`` it,
and you can correlate visual layout with snapshot refs.
@@ -303,10 +294,10 @@ layout questions, captchas, custom widgets where the accessibility
tree is incomplete).
If ``view_image`` errors back at you (rejected image, "vision not
supported", or similar), you can drive the page from `snapshot -i`
refs, `eval` for any DOM/JS state you need to read, and `text @ref`
or `get text` for content extraction.
{% endif %}
supported", or similar), you are running on a text-only model — stop
calling it and stop taking screenshots. Drive the page entirely from
`snapshot -i` refs, `eval` for any DOM/JS state you need to read, and
`text @ref` / `get text` for content extraction.
### Handle multiple pages via tabs
-12
View File
@@ -3,7 +3,6 @@
from __future__ import annotations
import contextlib
import functools
import logging
import os
import sys
@@ -79,16 +78,6 @@ class _StdoutQuietFilter(logging.Filter):
)
class _PtyThresholdFilter(logging.Filter):
def filter(self, record: logging.LogRecord) -> bool:
return "PTY process count reached warning threshold" not in record.getMessage()
@functools.cache # install once; tests reset via ``cache_clear()``
def _silence_pty_threshold_warning() -> None:
logging.getLogger("agents.sandbox.sandboxes.docker").addFilter(_PtyThresholdFilter())
def configure_dependency_logging() -> None:
"""Quiet dependency logging/warnings that obscure Strix scan logs."""
litellm = sys.modules.get("litellm")
@@ -100,7 +89,6 @@ def configure_dependency_logging() -> None:
logging.getLogger("asyncio").propagate = False
warnings.filterwarnings("ignore", category=RuntimeWarning, module="asyncio")
_route_unraisable_to_log()
_silence_pty_threshold_warning()
def _route_unraisable_to_log() -> None:
+2 -2
View File
@@ -24,12 +24,12 @@ async def load_skill(ctx: RunContextWrapper, skills: list[str]) -> str:
Max 5. Names match the bare files under
``strix/skills/<category>/<name>.md``.
"""
del ctx
requested = list(skills or [])
err = validate_requested_skills(requested)
if err:
return f"load_skill: {err}"
context = ctx.context if isinstance(ctx.context, dict) else {}
contents = load_skills(requested, supports_images=context.get("supports_images", True))
contents = load_skills(requested)
if not contents:
return "load_skill: no content loaded for requested skills."
sections = [f"## Skill: {name}\n\n{body}" for name, body in contents.items()]
-2
View File
@@ -288,11 +288,9 @@ async def call_mcp(
"Call search_mcp_tools, then get_mcp_tool_schema."
)
session = await entry.ensure_connected()
context = ctx.context if isinstance(ctx.context, dict) else {}
return await session.dispatch(
tool,
arguments or {},
label=namespaced_tool_name(connection, tool),
result_transform=entry.result_transform,
supports_images=context.get("supports_images", True),
)
-16
View File
@@ -32,7 +32,6 @@ from agents.mcp import (
)
from mcp.client.stdio import stdio_client
from mcp.shared._httpx_utils import create_mcp_http_client
from mcp.types import TextContent
from strix.tools.mcp.failures import HttpStatusRecorder
from strix.tools.mcp.session import McpConnectionUnavailableError, SupervisedMcpSession
@@ -184,18 +183,6 @@ def _build_server(config: McpConnectionConfig) -> BuiltMcpServer:
)
def _without_images(result: Any) -> Any:
content = [
TextContent(
type="text", text=f"[{item.mimeType} image omitted: this model cannot view images]"
)
if item.type == "image"
else item
for item in result.content
]
return result.model_copy(update={"content": content})
def _mcp_result_to_tool_output(server: MCPServer, result: Any) -> Any:
"""Serialize a ``CallToolResult`` to a tool output, mirroring the agents SDK.
@@ -230,7 +217,6 @@ async def dispatch_mcp_call(
*,
label: str,
result_transform: ResultTransform | None = None,
supports_images: bool = True,
) -> Any:
"""Run one MCP tool call and convert its result to a tool output.
@@ -247,8 +233,6 @@ async def dispatch_mcp_call(
corrupt the content the agent receives).
"""
result = await server.call_tool(tool_name, arguments)
if not supports_images:
result = _without_images(result)
if result_transform is not None:
return result_transform(label, result.model_dump(mode="json"))
tool_output = _mcp_result_to_tool_output(server, result)
-2
View File
@@ -365,7 +365,6 @@ class SupervisedMcpSession:
*,
label: str,
result_transform: ResultTransform | None = None,
supports_images: bool = True,
) -> Any:
"""Run one tool call with bounded retries for transient session failures.
@@ -383,7 +382,6 @@ class SupervisedMcpSession:
arguments,
label=label,
result_transform=result_transform,
supports_images=supports_images,
)
outcome = await self._run_job(job, phase="call")
-12
View File
@@ -115,15 +115,3 @@ def test_function_tools_are_result_bounded() -> None:
by_name = {t.name: t for t in agent.tools}
assert getattr(by_name["think"], "_strix_bounded", False) is True
@pytest.mark.parametrize(("supports_images", "has_view_image"), [(True, True), (False, False)])
def test_view_image_is_left_out_for_text_only_models(
supports_images: bool, has_view_image: bool
) -> None:
agent = factory.build_strix_agent(is_root=True, supports_images=supports_images)
filesystem = agent.capabilities[0]
filesystem.bind(cast("Any", object()))
view_image = next(tool for tool in filesystem.tools() if tool.name == "view_image")
assert cast("FunctionTool", view_image).is_enabled is has_view_image
-31
View File
@@ -9,11 +9,9 @@ from typing import Any, cast
from unittest.mock import MagicMock
import pytest
from agents import RunConfig
from agents.exceptions import MaxTurnsExceeded
from agents.items import MessageOutputItem
from agents.memory import SQLiteSession
from agents.run_config import CallModelData, ModelInputData
from agents.tool_context import ToolContext
from openai.types.responses import ResponseOutputMessage, ResponseOutputRefusal, ResponseOutputText
@@ -1533,32 +1531,3 @@ async def test_autonomous_nudge_does_not_offer_the_user() -> None:
)
assert "wait_for_user" not in items[0]["content"]
@pytest.mark.asyncio
async def test_text_only_filter_scrubs_images_then_chains_existing_filter() -> None:
run_config = RunConfig(model="m")
assert execution._with_image_scrub(run_config, {"supports_images": True}) is run_config
seen: list[ModelInputData] = []
def _inner(data: CallModelData[Any]) -> ModelInputData:
seen.append(data.model_data)
return data.model_data
run_config = RunConfig(model="m", call_model_input_filter=_inner)
scrub: Any = execution._with_image_scrub(run_config, {"supports_images": False})
image = {"type": "input_image", "image_url": "data:image/png;base64,aGk="}
output = {"type": "function_call_output", "call_id": "c1", "output": [image]}
data = CallModelData(
model_data=ModelInputData(input=[cast("Any", output)], instructions="sys"),
agent=MagicMock(),
context=None,
)
result = await scrub.call_model_input_filter(data)
assert result is seen[0]
assert result.input[0]["output"] == [
{"type": "input_text", "text": execution._TEXT_ONLY_IMAGE_TEXT}
]
assert data.model_data.input[0]["output"] == [image]
+1 -73
View File
@@ -109,7 +109,6 @@ def _patch_fast_backoff(monkeypatch: pytest.MonkeyPatch) -> None:
async def _run_once(
monkeypatch: pytest.MonkeyPatch,
streams: list[_FakeStream],
session: Any = None,
) -> Any:
_patch_fast_backoff(monkeypatch)
calls = {"n": 0}
@@ -132,7 +131,7 @@ async def _run_once(
run_config=cast("RunConfig", object()),
context={},
max_turns=5,
session=session,
session=None,
interactive=False,
event_sink=None,
hooks=None,
@@ -173,74 +172,3 @@ async def test_run_cycle_does_not_retry_permanent_error(
streams = [_FakeStream(exc=bad_request), _FakeStream()]
with pytest.raises(BadRequestError):
await _run_once(monkeypatch, streams)
@pytest.mark.parametrize(
("status", "message", "expected"),
[
(
404,
"litellm.NotFoundError: NotFoundError: OpenrouterException - "
'{"error":{"message":"No endpoints found that support image input","code":404,'
'"metadata":{"failed_routing_step":"Filter by Image Support"}}}',
True,
),
(400, "Invalid content type. image_url is only supported by certain models.", True),
(400, "my-model is not a multimodal model", True),
(400, "This model doesn't support the image field for user messages.", True),
(422, "messages[3]: unknown variant `image_url`, expected `text`", True),
(
400,
"OpenRouterException: Message: This model's maximum context length is 1048576 "
"tokens, but the request requires 1073484 tokens (942412 input including "
"image/vision expansion + 131072 for the completion). Reduce the input length "
"or max_tokens.",
False,
),
(400, "At most 5 image(s) may be provided in one prompt.", False),
(
400,
"Upstream error from Relace: Generation stopped with an incomplete tool call",
False,
),
(500, "No endpoints found that support image input", False),
],
)
def test_is_image_rejection(status: int, message: str, expected: bool) -> None:
exc = APIStatusError(message, response=httpx.Response(status, request=_request()), body=None)
assert execution._is_image_rejection(exc) is expected
async def _run_with_image_session(
monkeypatch: pytest.MonkeyPatch, streams: list[_FakeStream]
) -> tuple[Any, int, int]:
strips = {"n": 0}
async def _fake_strip(_session: Any) -> bool:
strips["n"] += 1
return True
async def _no_compact(*_args: Any, **_kwargs: Any) -> bool:
return False
monkeypatch.setattr(execution, "strip_all_images_from_session", _fake_strip)
monkeypatch.setattr(execution, "_compact_session", _no_compact)
result, attempts, _coordinator = await _run_once(monkeypatch, streams, session=object())
return result, attempts, strips["n"]
@pytest.mark.asyncio
async def test_run_cycle_strips_images_on_image_rejection(
monkeypatch: pytest.MonkeyPatch,
) -> None:
rejection = APIStatusError(
"No endpoints found that support image input",
response=httpx.Response(status_code=404, request=_request()),
body=None,
)
streams = [_FakeStream(exc=rejection), _FakeStream()]
result, attempts, strips = await _run_with_image_session(monkeypatch, streams)
assert result is streams[1]
assert attempts == 2
assert strips == 1
+1 -46
View File
@@ -18,7 +18,7 @@ from typing import TYPE_CHECKING, Any
import pytest
from agents.mcp import MCPServer, MCPServerStdio, MCPServerStreamableHttp
from agents.tool_context import ToolContext
from mcp.types import CallToolResult, ImageContent, TextContent
from mcp.types import CallToolResult, TextContent
from mcp.types import Tool as MCPTool
from pydantic import ValidationError
@@ -884,51 +884,6 @@ async def test_call_mcp_flags_an_errored_result_failed_for_the_tui() -> None:
assert out == {"type": "text", "text": "boom:read_file", "success": False}
@pytest.mark.asyncio
async def test_call_mcp_replaces_images_for_a_text_only_model() -> None:
class _ImageServer(FakeMCPServer):
async def call_tool(
self,
tool_name: str,
arguments: dict[str, Any] | None,
meta: dict[str, Any] | None = None,
) -> CallToolResult:
return CallToolResult(
content=[
TextContent(type="text", text="shot"),
ImageContent(type="image", data="aGk=", mimeType="image/png"),
]
)
registry = McpRegistry()
registry.add(
name="fs", server=_ImageServer("fs", [_mcp_tool("shot")]), purpose=None, tool_count=1
)
ctx = _ctx(registry)
args = json.dumps({"connection": "fs", "tool": "shot"})
out = await call_mcp.on_invoke_tool(ctx, args)
assert out[1] == {"type": "image", "image_url": "data:image/png;base64,aGk="}
ctx.context["supports_images"] = False
out = await call_mcp.on_invoke_tool(ctx, args)
assert out == [
{"type": "text", "text": "shot"},
{"type": "text", "text": "[image/png image omitted: this model cannot view images]"},
]
transformed: list[Any] = []
registry.add(
name="pro",
server=_ImageServer("pro", [_mcp_tool("shot")]),
purpose=None,
tool_count=1,
result_transform=lambda _label, result: transformed.append(result),
)
await call_mcp.on_invoke_tool(ctx, json.dumps({"connection": "pro", "tool": "shot"}))
assert [block["type"] for block in transformed[0]["content"]] == ["text", "text"]
# --- generic MCP tools are the only MCP surface every agent gets -------------
-41
View File
@@ -1,41 +0,0 @@
import logging
from collections.abc import Iterator
import pytest
from strix.telemetry import logging as tlog
_SDK_LOGGER = "agents.sandbox.sandboxes.docker"
@pytest.fixture
def sdk_logger() -> Iterator[logging.Logger]:
logger = logging.getLogger(_SDK_LOGGER)
original = list(logger.filters)
logger.filters.clear()
tlog._silence_pty_threshold_warning.cache_clear()
yield logger
logger.filters[:] = original
tlog._silence_pty_threshold_warning.cache_clear()
def _record(msg: str, *args: object) -> logging.LogRecord:
return logging.LogRecord(_SDK_LOGGER, logging.WARNING, __file__, 0, msg, args, None)
def test_installed_filter_drops_pty_threshold_warning(sdk_logger: logging.Logger) -> None:
tlog._silence_pty_threshold_warning()
record = _record("PTY process count reached warning threshold: %s active sessions", 62)
assert not sdk_logger.filter(record)
def test_installed_filter_keeps_other_warnings(sdk_logger: logging.Logger) -> None:
tlog._silence_pty_threshold_warning()
assert sdk_logger.filter(_record("something else went wrong"))
def test_install_is_idempotent(sdk_logger: logging.Logger) -> None:
tlog._silence_pty_threshold_warning()
tlog._silence_pty_threshold_warning()
assert len(sdk_logger.filters) == 1
-25
View File
@@ -13,7 +13,6 @@ from typing import Any
import httpx
import pytest
from agents import ModelSettings
from agents.tool_context import ToolContext
from openai import RateLimitError
import strix.tools.mcp as mcp_pkg
@@ -25,7 +24,6 @@ from strix.core import runner
from strix.core.agents import AgentCoordinator
from strix.core.inputs import make_model_settings
from strix.runtime import session_manager
from strix.tools.load_skill.tool import load_skill
from strix.tools.mcp import BearerAuth, McpConnectionConfig, McpConnectionRequest
@@ -289,29 +287,6 @@ def test_requested_skills_follow_the_shared_prefix() -> None:
assert "<xss>" in xss.split("<cache_point>")[1]
def test_text_only_prompt_drops_screenshot_guidance() -> None:
assert "view_image" in render_system_prompt(include_scope=False)
prompt = render_system_prompt(include_scope=False, supports_images=False)
assert "view_image" not in prompt
assert "text-only model and cannot view images" in prompt
assert "<!--" not in prompt
@pytest.mark.asyncio
async def test_text_only_load_skill_drops_screenshot_guidance() -> None:
ctx = ToolContext(
context={"supports_images": False},
tool_name="load_skill",
tool_call_id="call-1",
tool_arguments="{}",
)
out = await load_skill.on_invoke_tool(ctx, '{"skills": ["agent_browser"]}')
assert "view_image" not in out
assert "text-only model" in out
def test_scope_is_sent_as_its_own_system_message_on_cache_point_routes() -> None:
settings = make_model_settings(None, model_name="anthropic/claude-sonnet-5-5")
prompt = render_system_prompt(
-11
View File
@@ -70,17 +70,6 @@ def test_registered_dir_adds_new_skill(tmp_path: Path) -> None:
assert load_skills(["widget"]) == {"widget": "widget body"}
def test_only_jinja_skills_are_rendered(tmp_path: Path) -> None:
body = "{% if supports_images %}\nlook\n{% else %}\nread\n{% endif %}\n"
_write_skill(tmp_path, "extra", "plain", body)
_write_skill(tmp_path, "extra", "templated", f"---\ntemplate: jinja\n---\n{body}")
register_skill_dir(tmp_path)
assert load_skills(["plain"]) == {"plain": body}
assert load_skills(["templated"]) == {"templated": "look\n"}
assert load_skills(["templated"], supports_images=False) == {"templated": "read\n"}
def test_available_skill_includes_frontmatter_description(tmp_path: Path) -> None:
_write_skill(
tmp_path,