mirror of
https://github.com/usestrix/strix.git
synced 2026-11-09 05:19:39 -10:00
Compare commits
7
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
99c0711687 | ||
|
|
2635fb5307 | ||
|
|
8ab5e39c06 | ||
|
|
e1ec259ac2 | ||
|
|
066bd60a03 | ||
|
|
6ab123484d | ||
|
|
007ed1a94e |
No files matched your search
+47
-15
@@ -58,7 +58,7 @@ if TYPE_CHECKING:
|
||||
from openai import AsyncOpenAI
|
||||
from openai.types.responses.response_prompt_param import ResponsePromptParam
|
||||
|
||||
from strix.config.settings import LlmSettings, ReasoningEffort, Settings
|
||||
from strix.config.settings import ApiType, LlmSettings, ReasoningEffort, Settings
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -649,14 +649,40 @@ def configure_sdk_model_defaults(settings: Settings) -> None:
|
||||
if llm.api_base:
|
||||
os.environ["OPENAI_BASE_URL"] = llm.api_base
|
||||
_configure_litellm_default("api_base", llm.api_base)
|
||||
api_type = llm.api_type
|
||||
if api_type is None:
|
||||
api_type = "chat_completions" if llm.api_base else "responses"
|
||||
|
||||
set_default_openai_api(api_type)
|
||||
configure_sdk_api_route(llm.model or "", settings)
|
||||
_configure_extra_headers(llm)
|
||||
|
||||
|
||||
def configure_sdk_api_route(model_name: str, settings: Settings) -> None:
|
||||
"""Point SDK-native OpenAI requests for ``model_name`` at Responses or chat completions."""
|
||||
api_type = resolve_api_type(model_name, settings)
|
||||
logger.info("OpenAI API route for %s: %s", model_name, api_type)
|
||||
set_default_openai_api(api_type)
|
||||
|
||||
|
||||
_RESPONSES_ENDPOINT = "/v1/responses"
|
||||
|
||||
|
||||
def resolve_api_type(model_name: str, settings: Settings) -> ApiType:
|
||||
"""The SDK-native OpenAI route for ``model_name``: Responses or chat completions.
|
||||
|
||||
An explicit ``STRIX_API_TYPE`` wins. Otherwise the model decides: Responses
|
||||
when LiteLLM's catalog lists ``/v1/responses`` for it, chat completions for
|
||||
everything else.
|
||||
"""
|
||||
if settings.llm.api_type is not None:
|
||||
return settings.llm.api_type
|
||||
if _RESPONSES_ENDPOINT in _catalog_supported_endpoints(model_name):
|
||||
return "responses"
|
||||
return "chat_completions"
|
||||
|
||||
|
||||
def _catalog_supported_endpoints(model_name: str) -> list[str]:
|
||||
entry = _catalog_entry(model_name)
|
||||
endpoints = entry.get("supported_endpoints") if entry else None
|
||||
return [str(e) for e in endpoints] if isinstance(endpoints, list) else []
|
||||
|
||||
|
||||
def _mirror_api_key_to_provider_env(model_name: str | None, api_key: str) -> None:
|
||||
if not model_name:
|
||||
return
|
||||
@@ -859,11 +885,7 @@ def uses_chat_completions_tool_schema(model_name: str, settings: Settings) -> bo
|
||||
model = model_name.strip().lower()
|
||||
if "/" in model and not model.startswith("openai/"):
|
||||
return True
|
||||
if settings.llm.api_type is not None:
|
||||
return settings.llm.api_type == "chat_completions"
|
||||
if settings.llm.api_base:
|
||||
return True
|
||||
return not model_supports_reasoning(model_name)
|
||||
return resolve_api_type(model_name, settings) == "chat_completions"
|
||||
|
||||
|
||||
def supports_strict_tool_schemas(model_name: str) -> bool:
|
||||
@@ -879,17 +901,27 @@ def supports_strict_tool_schemas(model_name: str) -> bool:
|
||||
|
||||
|
||||
def model_supports_reasoning(model_name: str) -> bool:
|
||||
import litellm
|
||||
entry = _catalog_entry(model_name)
|
||||
return bool(entry and entry.get("supports_reasoning"))
|
||||
|
||||
|
||||
def _bare_openai_name(model_name: str) -> str:
|
||||
name = model_name.strip().lower()
|
||||
for prefix in ("litellm/", "any-llm/", "openai/"):
|
||||
if name.startswith(prefix):
|
||||
name = name[len(prefix) :]
|
||||
break
|
||||
return name[len(prefix) :]
|
||||
return name
|
||||
|
||||
|
||||
def _catalog_entry(model_name: str) -> dict[str, Any] | None:
|
||||
"""LiteLLM's cost-map entry for the model, looked up as it would route it."""
|
||||
import litellm
|
||||
|
||||
name = _bare_openai_name(model_name)
|
||||
entry = litellm.model_cost.get(name)
|
||||
if entry is None and "/" in name:
|
||||
entry = litellm.model_cost.get(name.rsplit("/", 1)[1])
|
||||
return bool(entry and entry.get("supports_reasoning"))
|
||||
return entry if isinstance(entry, dict) else None
|
||||
|
||||
|
||||
def is_known_openai_bare_model(model_name: str) -> bool:
|
||||
|
||||
@@ -4,7 +4,7 @@ from __future__ import annotations
|
||||
|
||||
from typing import Literal
|
||||
|
||||
from pydantic import AliasChoices, Field
|
||||
from pydantic import AliasChoices, Field, field_validator
|
||||
from pydantic_settings import BaseSettings, SettingsConfigDict
|
||||
|
||||
|
||||
@@ -13,6 +13,12 @@ ApiType = Literal["responses", "chat_completions"]
|
||||
|
||||
DEFAULT_MAX_TURNS = 500
|
||||
|
||||
|
||||
def _lowercase(value: object) -> object:
|
||||
"""Enum-like env values are matched case-insensitively (``None`` is ``none``)."""
|
||||
return value.strip().lower() if isinstance(value, str) else value
|
||||
|
||||
|
||||
_BASE_CONFIG = SettingsConfigDict(
|
||||
case_sensitive=False,
|
||||
populate_by_name=True,
|
||||
@@ -78,6 +84,11 @@ class LlmSettings(BaseSettings):
|
||||
alias="LLM_MAX_TOOL_CALLS_PER_TURN",
|
||||
)
|
||||
|
||||
@field_validator("api_type", "reasoning_effort", mode="before")
|
||||
@classmethod
|
||||
def _normalize_case(cls, value: object) -> object:
|
||||
return _lowercase(value)
|
||||
|
||||
|
||||
class DedupeSettings(BaseSettings):
|
||||
model_config = _BASE_CONFIG
|
||||
@@ -95,6 +106,11 @@ class DedupeSettings(BaseSettings):
|
||||
repr=False,
|
||||
)
|
||||
|
||||
@field_validator("reasoning_effort", mode="before")
|
||||
@classmethod
|
||||
def _normalize_case(cls, value: object) -> object:
|
||||
return _lowercase(value)
|
||||
|
||||
|
||||
class ContextSettings(BaseSettings):
|
||||
"""Context-window management: per-tool-output caps and history compaction."""
|
||||
|
||||
@@ -267,9 +267,7 @@ def make_model_settings(
|
||||
and reasoning_effort != "none"
|
||||
and model_supports_reasoning(model_name)
|
||||
):
|
||||
model_settings = model_settings.resolve(
|
||||
_reasoning_settings(reasoning_effort),
|
||||
)
|
||||
model_settings = model_settings.resolve(_reasoning_settings(reasoning_effort))
|
||||
if force_required_tool_choice and _accepts_required_tool_choice(model_name):
|
||||
model_settings = model_settings.resolve(ModelSettings(tool_choice="required"))
|
||||
|
||||
|
||||
@@ -18,9 +18,10 @@ from openai import RateLimitError
|
||||
|
||||
from strix.agents.factory import build_strix_agent, make_child_factory
|
||||
from strix.agents.prompt import render_scope_prompt, render_system_prompt
|
||||
from strix.config import load_settings
|
||||
from strix.config import codex, load_settings
|
||||
from strix.config.models import (
|
||||
StrixProvider,
|
||||
configure_sdk_api_route,
|
||||
configure_sdk_model_defaults,
|
||||
supports_strict_tool_schemas,
|
||||
uses_chat_completions_tool_schema,
|
||||
@@ -257,6 +258,10 @@ async def run_strix_scan(
|
||||
raise RuntimeError(
|
||||
"No LLM model configured. Set STRIX_LLM env or pass model= to run_strix_scan().",
|
||||
)
|
||||
if resolved_model != (settings.llm.model or "").strip() and not codex.subscription_model(
|
||||
resolved_model
|
||||
):
|
||||
configure_sdk_api_route(resolved_model, settings)
|
||||
logger.info("LLM model resolved: %s", resolved_model)
|
||||
chat_completions_tools = uses_chat_completions_tool_schema(resolved_model, settings)
|
||||
strict_tool_schemas = supports_strict_tool_schemas(resolved_model)
|
||||
|
||||
@@ -1075,11 +1075,11 @@ async def create_vulnerability_report(
|
||||
"Techniques" that read like an engineering runbook rather than a
|
||||
client deliverable.
|
||||
- **Use markdown in every text field**: ``**bold**`` for emphasis,
|
||||
``inline code`` for identifiers/values/parameters, and fenced
|
||||
code blocks (```` ```language ````) for any code/payload/HTTP
|
||||
``inline code`` for identifiers/values/parameters, and
|
||||
language-tagged fenced code blocks for any code/payload/HTTP
|
||||
excerpt. Never leave code bare/unformatted. When referencing a
|
||||
file, annotate the fence, e.g.
|
||||
```` ```python title=app.py startLineNumber=42 endLineNumber=50 ````.
|
||||
file, annotate the opening fence with
|
||||
``title=app.py startLineNumber=42 endLineNumber=50`` after the language.
|
||||
- Field discipline: ``poc_description`` is steps only — NO code (all
|
||||
code goes in ``poc_script_code``); ``remediation_steps`` is prose
|
||||
only — NO code/diffs (code fixes go in ``code_locations``).
|
||||
@@ -1191,6 +1191,57 @@ async def create_vulnerability_report(
|
||||
Broken / Risky Crypto, CWE-311 Missing Encryption, CWE-916 Weak
|
||||
Password Hashing.
|
||||
|
||||
Example (abbreviated — mirror this structure)::
|
||||
|
||||
title: "Reflected XSS in /search q parameter"
|
||||
description:
|
||||
The **`q`** parameter of `/search` reflects user input into
|
||||
the HTML response without encoding, allowing script
|
||||
injection.
|
||||
technical_analysis:
|
||||
The handler interpolates `q` directly into the page body:
|
||||
|
||||
```python title=views.py startLineNumber=42 endLineNumber=44
|
||||
html = f"<h2>Results for {q}</h2>"
|
||||
return HttpResponse(html)
|
||||
```
|
||||
|
||||
No output encoding is applied, so `<script>` executes.
|
||||
poc_description:
|
||||
1. Navigate to `/search?q=<payload>`.
|
||||
2. Observe the payload executes in the victim's browser.
|
||||
poc_script_code:
|
||||
```
|
||||
GET /search?q=<script>alert(document.domain)</script>
|
||||
```
|
||||
evidence:
|
||||
Response echoes the payload verbatim:
|
||||
|
||||
```html
|
||||
<h2>Results for <script>alert(document.domain)</script></h2>
|
||||
```
|
||||
assumptions:
|
||||
Assumes a victim can be induced to open a crafted link.
|
||||
remediation_steps:
|
||||
Context-encode all user input rendered into HTML; prefer the
|
||||
template engine's auto-escaping over string interpolation.
|
||||
counterevidence:
|
||||
No output encoding, CSP, or WAF observed on this response;
|
||||
payload executed in a current browser. The parameter is
|
||||
reflected on an unauthenticated route, so no privileged
|
||||
position is required.
|
||||
confidence: "high"
|
||||
severity_change_conditions:
|
||||
A restrictive CSP that blocks inline script execution would
|
||||
reduce impact and lower the severity.
|
||||
fix_effort: "low"
|
||||
|
||||
Nice to have: for code findings, if the checkout has git history, a quick
|
||||
``git blame`` (quote the paths) on the vulnerable line is worth weaving into
|
||||
``technical_analysis`` — who last touched it, when, and in which commit, as
|
||||
part of the prose, not a separate section. Skip it if the line is
|
||||
uncommitted or the command fails.
|
||||
|
||||
Args:
|
||||
title: Specific finding title (e.g.
|
||||
``"SQL Injection in /api/users login parameter"``). Don't
|
||||
@@ -1359,57 +1410,6 @@ async def create_vulnerability_report(
|
||||
fix (summary + rationale). Prose/markdown only — the code
|
||||
change itself belongs in ``code_locations``. Omit for
|
||||
black-box findings.
|
||||
|
||||
Example (abbreviated — mirror this structure)::
|
||||
|
||||
title: "Reflected XSS in /search q parameter"
|
||||
description:
|
||||
The **`q`** parameter of `/search` reflects user input into
|
||||
the HTML response without encoding, allowing script
|
||||
injection.
|
||||
technical_analysis:
|
||||
The handler interpolates `q` directly into the page body:
|
||||
|
||||
```python title=views.py startLineNumber=42 endLineNumber=44
|
||||
html = f"<h2>Results for {q}</h2>"
|
||||
return HttpResponse(html)
|
||||
```
|
||||
|
||||
No output encoding is applied, so `<script>` executes.
|
||||
poc_description:
|
||||
1. Navigate to `/search?q=<payload>`.
|
||||
2. Observe the payload executes in the victim's browser.
|
||||
poc_script_code:
|
||||
```
|
||||
GET /search?q=<script>alert(document.domain)</script>
|
||||
```
|
||||
evidence:
|
||||
Response echoes the payload verbatim:
|
||||
|
||||
```html
|
||||
<h2>Results for <script>alert(document.domain)</script></h2>
|
||||
```
|
||||
assumptions:
|
||||
Assumes a victim can be induced to open a crafted link.
|
||||
remediation_steps:
|
||||
Context-encode all user input rendered into HTML; prefer the
|
||||
template engine's auto-escaping over string interpolation.
|
||||
counterevidence:
|
||||
No output encoding, CSP, or WAF observed on this response;
|
||||
payload executed in a current browser. The parameter is
|
||||
reflected on an unauthenticated route, so no privileged
|
||||
position is required.
|
||||
confidence: "high"
|
||||
severity_change_conditions:
|
||||
A restrictive CSP that blocks inline script execution would
|
||||
reduce impact and lower the severity.
|
||||
fix_effort: "low"
|
||||
|
||||
Nice to have: for code findings, if the checkout has git history, a quick
|
||||
``git blame`` (quote the paths) on the vulnerable line is worth weaving into
|
||||
``technical_analysis`` — who last touched it, when, and in which commit, as
|
||||
part of the prose, not a separate section. Skip it if the line is
|
||||
uncommitted or the command fails.
|
||||
"""
|
||||
(
|
||||
http_exchange_ids,
|
||||
|
||||
@@ -430,3 +430,10 @@ def test_user_headers_override_openrouter_attribution() -> None:
|
||||
assert headers["X-Title"] == "Custom"
|
||||
assert headers["X-Tenant"] == "acme"
|
||||
assert headers["HTTP-Referer"] == "https://strix.ai"
|
||||
|
||||
|
||||
def test_reasoning_effort_sent_as_configured_and_none_omitted() -> None:
|
||||
assert make_model_settings("none", model_name="gpt-5.6-sol").reasoning is None
|
||||
settings = make_model_settings("high", model_name="gpt-daybreak-blue-latest")
|
||||
assert settings.reasoning is not None
|
||||
assert settings.reasoning.effort == "high"
|
||||
+66
-2
@@ -10,12 +10,14 @@ from agents.models import _openai_shared
|
||||
from agents.models.openai_chatcompletions import OpenAIChatCompletionsModel
|
||||
from agents.models.openai_responses import OpenAIResponsesModel
|
||||
|
||||
from strix.config import models
|
||||
from strix.config.models import (
|
||||
StrixProvider,
|
||||
_NonStreamingModel,
|
||||
_TurnGuardModel,
|
||||
configure_sdk_model_defaults,
|
||||
request_timeout_extra_args,
|
||||
resolve_api_type,
|
||||
routes_through_litellm,
|
||||
supports_strict_tool_schemas,
|
||||
uses_chat_completions_tool_schema,
|
||||
@@ -111,7 +113,7 @@ def test_api_type_override_settings(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
@pytest.mark.parametrize(
|
||||
("api_type", "expected"),
|
||||
[
|
||||
(None, OpenAIChatCompletionsModel),
|
||||
(None, OpenAIResponsesModel),
|
||||
("chat_completions", OpenAIChatCompletionsModel),
|
||||
("responses", OpenAIResponsesModel),
|
||||
],
|
||||
@@ -119,7 +121,7 @@ def test_api_type_override_settings(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
def test_api_type_overrides_the_api_base_route(
|
||||
monkeypatch: pytest.MonkeyPatch, api_type: str | None, expected: type
|
||||
) -> None:
|
||||
"""``LLM_API_BASE`` defaults to chat completions. ``STRIX_API_TYPE`` must win."""
|
||||
"""gpt-5 is catalogued on /v1/responses, so a base URL alone changes nothing."""
|
||||
monkeypatch.setattr(_openai_shared, "_use_responses_by_default", True)
|
||||
monkeypatch.setattr(_openai_shared, "_default_openai_client", None)
|
||||
monkeypatch.setattr(_openai_shared, "_default_openai_key", None)
|
||||
@@ -138,3 +140,65 @@ def test_api_type_overrides_the_api_base_route(
|
||||
while isinstance(model, _NonStreamingModel | _TurnGuardModel | RequestLoggingModel):
|
||||
model = model._inner
|
||||
assert isinstance(model, expected)
|
||||
|
||||
|
||||
def _settings(monkeypatch: pytest.MonkeyPatch, model: str, api_base: str | None) -> Settings:
|
||||
monkeypatch.setenv("STRIX_LLM", model)
|
||||
monkeypatch.delenv("STRIX_API_TYPE", raising=False)
|
||||
for name in ("LLM_API_BASE", "OPENAI_API_BASE", "OPENAI_BASE_URL"):
|
||||
monkeypatch.delenv(name, raising=False)
|
||||
if api_base is not None:
|
||||
monkeypatch.setenv("LLM_API_BASE", api_base)
|
||||
return Settings()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"api_base", [None, "https://api.openai.com/v1", "https://gateway.example/v1"]
|
||||
)
|
||||
def test_resolve_api_type_follows_the_catalog_not_the_base_url(
|
||||
monkeypatch: pytest.MonkeyPatch, api_base: str | None
|
||||
) -> None:
|
||||
"""Responses when LiteLLM lists /v1/responses for the model, chat completions otherwise."""
|
||||
settings = _settings(monkeypatch, "gpt-5", api_base)
|
||||
for model in ("gpt-5", "gpt-5.6-sol", "openai/gpt-5.4", "gpt-daybreak-blue-latest"):
|
||||
assert resolve_api_type(model, settings) == "responses", model
|
||||
assert uses_chat_completions_tool_schema(model, settings) is False, model
|
||||
for model in ("gpt-4o", "my-private-model"):
|
||||
assert resolve_api_type(model, settings) == "chat_completions", model
|
||||
assert uses_chat_completions_tool_schema(model, settings) is True, model
|
||||
|
||||
|
||||
def test_resolve_api_type_explicit_override_wins(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
_settings(monkeypatch, "gpt-daybreak-blue-latest", "https://gateway.example/v1")
|
||||
monkeypatch.setenv("STRIX_API_TYPE", "chat_completions")
|
||||
assert resolve_api_type("gpt-daybreak-blue-latest", Settings()) == "chat_completions"
|
||||
monkeypatch.setenv("STRIX_API_TYPE", "Responses")
|
||||
assert resolve_api_type("gpt-5", Settings()) == "responses"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("raw", "expected"),
|
||||
[("None", "none"), ("HIGH", "high"), (" xhigh ", "xhigh"), ("Max", "max")],
|
||||
)
|
||||
def test_reasoning_effort_is_case_insensitive(
|
||||
monkeypatch: pytest.MonkeyPatch, raw: str, expected: str
|
||||
) -> None:
|
||||
monkeypatch.setenv("STRIX_REASONING_EFFORT", raw)
|
||||
monkeypatch.setenv("STRIX_DEDUPE_REASONING_EFFORT", raw)
|
||||
settings = Settings()
|
||||
assert settings.llm.reasoning_effort == expected
|
||||
assert settings.dedupe.reasoning_effort == expected
|
||||
|
||||
|
||||
def test_configure_sdk_api_route_follows_the_given_model(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""A ``model=`` override picks its own route, not ``STRIX_LLM``'s."""
|
||||
routes: list[str] = []
|
||||
monkeypatch.setattr(models, "set_default_openai_api", routes.append)
|
||||
settings = _settings(monkeypatch, "gpt-5", "https://gateway.example/v1")
|
||||
|
||||
models.configure_sdk_api_route("gpt-5", settings)
|
||||
models.configure_sdk_api_route("my-private-model", settings)
|
||||
|
||||
assert routes == ["responses", "chat_completions"]
|
||||
@@ -0,0 +1,45 @@
|
||||
"""Every parameter of every scan-agent tool carries a description in its JSON schema.
|
||||
|
||||
The model only sees a parameter's docstring text through the schema. An ``Args:``
|
||||
section that the docstring parser fails to recognise (for example because an
|
||||
earlier line opens a markdown code fence it never closes) silently drops every
|
||||
parameter description, so optional-but-conditionally-required fields such as
|
||||
``confidence_rationale`` become anonymous nullable strings the model never fills.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
from agents.tool import FunctionTool
|
||||
|
||||
from strix.agents import factory
|
||||
from strix.tools.agents_graph.tools import agent_finish
|
||||
from strix.tools.finish.tool import finish_scan
|
||||
from strix.tools.reporting.tool import create_vulnerability_report
|
||||
from strix.tools.respond.tool import respond_to_user
|
||||
|
||||
|
||||
_SCAN_AGENT_TOOLS = [
|
||||
tool
|
||||
for tool in (*factory._BASE_TOOLS, finish_scan, agent_finish, respond_to_user)
|
||||
if isinstance(tool, FunctionTool)
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("tool", _SCAN_AGENT_TOOLS, ids=lambda tool: tool.name)
|
||||
def test_every_parameter_has_a_description(tool: FunctionTool) -> None:
|
||||
properties = tool.params_json_schema.get("properties", {})
|
||||
missing = sorted(
|
||||
name
|
||||
for name, schema in properties.items()
|
||||
if not (isinstance(schema.get("description"), str) and schema["description"].strip())
|
||||
)
|
||||
assert not missing, f"{tool.name}: parameters without a description: {missing}"
|
||||
|
||||
|
||||
def test_create_vulnerability_report_explains_confidence_rationale() -> None:
|
||||
schema = create_vulnerability_report.params_json_schema["properties"]
|
||||
assert "confidence" in create_vulnerability_report.params_json_schema["required"]
|
||||
rationale = schema["confidence_rationale"]["description"]
|
||||
assert "Required when" in rationale
|
||||
assert "high" in rationale
|
||||
Reference in new issue
Block a user