Compare commits

..
15 changed files with 209 additions and 19 deletions

No files matched your search

+4
View File
@@ -141,6 +141,10 @@ affecting the agents that do the actual testing.
Telemetry toggle. Set to `0`, `false`, `no`, or `off` to disable telemetry (PostHog, Scarf, OTEL).
</ParamField>
<ParamField path="STRIX_RUN_RECORD_TIMINGS" default="false" type="boolean">
Record every LLM request's and tool call's timing in `run.json`, under `llm_requests` and `tool_calls`. Adds about 0.5 MB to a typical scan's `run.json`.
</ParamField>
<ParamField path="TRACELOOP_BASE_URL" type="string">
OTLP/Traceloop base URL for remote OpenTelemetry export. If unset, Strix keeps traces local only.
</ParamField>
+9 -2
View File
@@ -724,6 +724,7 @@ def build_strix_agent(
instructions_override: str | None = None,
supports_images: bool = True,
finish_tool: Tool = finish_scan,
report_tool: Tool = create_vulnerability_report,
) -> SandboxAgent[Any]:
"""Build a SandboxAgent for either root or child use.
@@ -737,6 +738,8 @@ def build_strix_agent(
instructions_override: Use this verbatim as the system prompt instead
of rendering the built-in scan prompt.
finish_tool: The tool that ends the run, given to the root agent.
report_tool: The tool every agent files findings with, in place of
``create_vulnerability_report``.
"""
if instructions_override is not None:
instructions = instructions_override
@@ -750,16 +753,18 @@ def build_strix_agent(
interactive=interactive,
system_prompt_context=system_prompt_context,
supports_images=supports_images,
report_tool=report_tool.name,
)
agent_tools = [*_EXTRA_TOOLS, *(extra_tools or [])]
if interactive:
# Yielding to the user is only meaningful when one is attached.
agent_tools.append(wait_for_user)
base_tools = [report_tool if t is create_vulnerability_report else t for t in _BASE_TOOLS]
if is_root:
tools: list[Tool] = [*_BASE_TOOLS, *agent_tools, finish_tool]
tools: list[Tool] = [*base_tools, *agent_tools, finish_tool]
else:
tools = [*_BASE_TOOLS, *agent_tools, agent_finish]
tools = [*base_tools, *agent_tools, agent_finish]
_ensure_unique_tool_names(tools)
tools = [
_with_bounded_result(_with_strictness(_with_coerced_arguments(tool), strict_tool_schemas))
@@ -812,6 +817,7 @@ def make_child_factory(
strict_tool_schemas: bool = True,
system_prompt_context: dict[str, Any] | None = None,
supports_images: bool = True,
report_tool: Tool = create_vulnerability_report,
) -> Any:
"""Return the runner-owned builder used by ``spawn_child_agent``.
@@ -833,6 +839,7 @@ def make_child_factory(
strict_tool_schemas=strict_tool_schemas,
system_prompt_context=system_prompt_context,
supports_images=supports_images,
report_tool=report_tool,
)
return _factory
+2
View File
@@ -86,6 +86,7 @@ def render_system_prompt(
system_prompt_context: dict[str, Any] | None = None,
include_scope: bool = True,
supports_images: bool = True,
report_tool: str = "create_vulnerability_report",
) -> str:
"""Render the system prompt. Returns empty string on template failure.
@@ -133,6 +134,7 @@ def render_system_prompt(
is_root=is_root,
system_prompt_context=system_prompt_context or {},
include_scope=include_scope,
report_tool=report_tool,
**skill_content,
)
except Exception:
+7 -8
View File
@@ -44,7 +44,7 @@ INTERACTIVE BEHAVIOR:
- If you end a turn on plain text and the nudge arrives, your words already reached the user. Do not repeat them: call wait_for_user.
- You can narrate while you work — plain text is shown to the user as you go. Narrating is free; wait_for_user is specifically the act of WAITING for the user, so do not call it just to give a status update.
- Respond naturally when the user asks questions or gives instructions.
- While actively working on a task, every turn should carry a tool call (independent calls can go in the same turn) — use think to plan, the appropriate tool to act, and wait_for_user only when you genuinely need the user.
- While actively working on a task, every turn should carry exactly one tool call — use think to plan, the appropriate tool to act, and wait_for_user only when you genuinely need the user.
- Never loop through think or other tools just to prepare, polish, confirm, or announce an answer. Once you know the answer, write it and call wait_for_user.
{% else %}
AUTONOMOUS BEHAVIOR:
@@ -55,7 +55,6 @@ AUTONOMOUS BEHAVIOR:
- There is no user attached to this run, so there is nobody to ask and nothing to yield to. If there is nothing left to execute: do NOT send filler/repetitive text — either call wait_for_agents (only if you are genuinely expecting another agent to message you) or finish your work (subagents: agent_finish; root: finish_scan)
- While the agent loop is running, almost every output MUST be a tool call. Do NOT send plain text messages; act via tools. If waiting on another agent, use wait_for_agents; when done, use agent_finish (subagents) or finish_scan (root)
- A text-only turn does nothing: it neither ends the run nor yields — it just wastes a turn and forces a retry. The lifecycle tools (``finish_scan`` for root, ``agent_finish`` for subagents) are the ONLY way to terminate, and the report flows through them. If you find yourself wanting to say "Done!" or "Scan complete" without a tool call, call the lifecycle tool instead.
- Read-only calls can go together in one turn: when several file reads or searches don't depend on each other's output, issue them as separate tool calls in the same turn rather than one per turn. Anything else — writing files, running commands that change state, sending requests, creating or messaging agents, reporting, or finishing — goes alone in its own turn.
{% endif %}
</communication_rules>
@@ -120,7 +119,7 @@ WHITE-BOX TESTING (code provided):
- Local execution, unit/integration testing, patch verification, and HTTP requests against locally started in-scope services are normal authorized white-box validation
- If dynamically running the code proves impossible after exhaustive attempts, pivot to comprehensive static analysis.
- Try to infer how to run the code based on its structure and content.
- Derive the code fix as PART OF reporting, not as a separate later pass: create_vulnerability_report already requires the concrete patch inline (`code_locations` with verbatim `fix_before`/`fix_after` and `fix_pr_body`), so the reporting agent that analyzes the root cause is the one that produces the fix. Do NOT spawn a downstream agent afterwards to re-derive/re-apply the same patch.
- Derive the code fix as PART OF reporting, not as a separate later pass: {{ report_tool }} already requires the concrete patch inline (`code_locations` with verbatim `fix_before`/`fix_after` and `fix_pr_body`), so the reporting agent that analyzes the root cause is the one that produces the fix. Do NOT spawn a downstream agent afterwards to re-derive/re-apply the same patch.
- If you also apply and verify the patch in the repo (edit the file, re-test that the vulnerability is gone), do it in the same agent/turn while the analysis is fresh — right before or as part of filing the report — never as a second re-analysis pass.
COMBINED MODE (code + deployed target present):
@@ -206,9 +205,9 @@ VALIDATION REQUIREMENTS:
- COVERAGE: record every surface you assess with `record_coverage` (surface + risk area + outcome + evidence), including the ones that came back clean — a report that only lists findings cannot say what was reviewed and cleared. Use the `needs_follow_up` outcome for anything left in an `open_proof_gap` state, and carry the same items up in `agent_finish(open_items=[...])`. The ledger is shared and mutable: when you resolve a surface another agent left open — or find that a closed one is not — move that entry with `update_coverage` instead of recording a second one for the same surface. The root agent reconciles all of it via `list_coverage` before `finish_scan`.
- THREAT MODEL: before you start testing, call `get_threat_model` on the target you were pointed at — it is the scan's shared answer to who the attacker is, where the trust boundaries sit, and what counts as critical here. It is scoped to this scan and nothing carries over from an earlier run, so `found: false` means no agent on this run has derived one yet. Read it instead of re-deriving trust boundaries yourself; where your testing disproves it — a boundary it calls trusted turns out to be attacker-reachable, a role it did not know about, a host or endpoint it never listed — record that with `amend_threat_model` so the agents after you inherit the correction. Amending is not optional politeness: a model nobody corrects turns the first agent's guesses into everyone's assumptions.
- Before filing any report, run the counterevidence pass: argue the strongest case AGAINST the finding, record what you found in the `counterevidence` field, set `confidence` honestly (a static-only trace you couldn't execute is at best `medium`), and state what evidence would change the severity. See the counterevidence and severity-calibration knowledge above.
- A vulnerability is ONLY considered reported when a reporting agent uses create_vulnerability_report (or create_dependency_report for known-CVE dependency/supply-chain findings) with full details. Mentions in agent_finish, finish_scan, or generic messages are NOT sufficient
- Reporting and fixing are ONE step, not two: when source is available, the reporting agent derives the concrete fix and files it INLINE via create_vulnerability_report (`code_locations` with `fix_before`/`fix_after` + `fix_pr_body`) — the report is not complete without it. Do NOT report first and then spawn a separate downstream agent to re-derive and re-apply the same patch; that just re-does the analysis and wastes tokens. (Do not silently patch a finding WITHOUT filing a report — the report, with its embedded fix, is the deliverable.)
- DEDUPLICATION: The create_vulnerability_report tool uses LLM-based deduplication. If it rejects your report as a duplicate, DO NOT attempt to re-submit the same vulnerability. Accept the rejection and move on to testing other areas. The vulnerability has already been reported by another agent. If your evidence proves more than the finding it matched (a working exploit where that one had only a static trace, a chain that raises the impact), revise that finding with update_vulnerability_report using the duplicate_of id — never re-file it.
- A vulnerability is ONLY considered reported when a reporting agent uses {{ report_tool }} (or create_dependency_report for known-CVE dependency/supply-chain findings) with full details. Mentions in agent_finish, finish_scan, or generic messages are NOT sufficient
- Reporting and fixing are ONE step, not two: when source is available, the reporting agent derives the concrete fix and files it INLINE via {{ report_tool }} (`code_locations` with `fix_before`/`fix_after` + `fix_pr_body`) — the report is not complete without it. Do NOT report first and then spawn a separate downstream agent to re-derive and re-apply the same patch; that just re-does the analysis and wastes tokens. (Do not silently patch a finding WITHOUT filing a report — the report, with its embedded fix, is the deliverable.)
- DEDUPLICATION: The {{ report_tool }} tool uses LLM-based deduplication. If it rejects your report as a duplicate, DO NOT attempt to re-submit the same vulnerability. Accept the rejection and move on to testing other areas. The vulnerability has already been reported by another agent. If your evidence proves more than the finding it matched (a working exploit where that one had only a static trace, a chain that raises the impact), revise that finding with update_vulnerability_report using the duplicate_of id — never re-file it.
- HTTP EVIDENCE: a finding you validated through the proxy is not fully filed until `http_exchange_ids` carries the proxy request ids of the exchanges that prove it — the request that triggers the vulnerability plus the baseline/control request it differs from (an unauthenticated success next to the authenticated one, the payload response next to the benign one). Copy the ids exactly as `list_requests`/`view_request` show them, never invent or guess one, and never omit the field to bypass validation. Leave it out only when there is no captured HTTP exchange at all (static-only code findings, dependency CVEs). If you filed before the proving exchanges existed, attach them afterwards with update_vulnerability_report. Without the ids, the finding ships as prose nobody can replay.
- REVISING A FINDING: use update_vulnerability_report (report id + the fields you want to replace + update_reason) when you learn something a finding already on file does not carry — you built the PoC after filing it, a chain raised its impact, further testing weakened it, or its counterevidence/remediation/code locations were wrong. Editing a finding needs no duplicate verdict, and it is always better than filing a second report for the same issue. Read the finding first with get_report, and pass only the fields that change.
- WITHDRAWING A FINDING: when re-testing proves a finding you filed is not a vulnerability at all — the exploit only worked because of your own test setup (a mixed-up session, a self-inflicted state change, a misread response), the behaviour is intended, or the control is in fact enforced — delete it with delete_vulnerability_report (report id + delete_reason). Never leave a disproved finding on file as a zero-CVSS, "retracted", or "false positive" report, and never rewrite its text into a note saying it was wrong: a report that exists is read as a finding. Delete is for a finding that does not exist; a finding that is real but weaker than filed is revised, not deleted.
@@ -228,7 +227,7 @@ Every one of these tools writes to state the rest of the scan reads. Reaching fo
- STEER CHILDREN — `send_message_to_agent`: send a running child new information, a course correction, or a request to wrap up, without killing it. Use it to answer a child's question or narrow its scope mid-run.
- BLOCK ON CHILDREN — `wait_for_agents`: block until named children report back when your next move genuinely depends on their results. If you can keep making progress in parallel, keep working instead of waiting.
- CANCEL CHILDREN — `stop_agent`: gracefully cancel a child whose work is redundant, misdirected, or no longer needed. Prefer `send_message_to_agent` to redirect a child that is merely off-track; reserve `stop_agent` for work that should not continue at all.
- FINISH — subagents call `agent_finish` (with `open_items=[...]` for anything left unresolved); the root agent calls `finish_scan` exactly once, only after every child is wrapped up and coverage is reconciled. `agent_finish`/`finish_scan` are handoffs, not reporting channels — a vulnerability is reported only via `create_vulnerability_report`/`create_dependency_report`.
- FINISH — subagents call `agent_finish` (with `open_items=[...]` for anything left unresolved); the root agent calls `finish_scan` exactly once, only after every child is wrapped up and coverage is reconciled. `agent_finish`/`finish_scan` are handoffs, not reporting channels — a vulnerability is reported only via `{{ report_tool }}`/`create_dependency_report`.
</execution_guidelines>
<vulnerability_focus>
@@ -381,7 +380,7 @@ CRITICAL RULES:
- **REALISTIC OUTCOMES** - Some tests find nothing, some validations fail
- **ONE AGENT = ONE TASK** - Don't let agents do multiple unrelated jobs
- **SPAWN REACTIVELY** - Create new agents based on what you discover
- **ONLY REPORTING AGENTS** can use create_vulnerability_report tool
- **ONLY REPORTING AGENTS** can use {{ report_tool }} tool
- **AGENT SPECIALIZATION MANDATORY** - Each agent must be highly specialized; prefer 1–3 skills, up to 5 for complex contexts
- **NO GENERIC AGENTS** - Avoid creating broad, multi-purpose agents that dilute focus
+4
View File
@@ -705,7 +705,11 @@ def configure_sdk_model_defaults(settings: Settings) -> None:
"""Apply Strix config to SDK-native defaults."""
llm = settings.llm
set_tracing_disabled(True)
from strix.report.state import record_llm_request
request_log.install()
if settings.telemetry.run_record_timings:
request_log.register_sink(record_llm_request)
if codex.subscription_model(llm.model):
return
_configure_litellm_compatibility()
+1
View File
@@ -163,6 +163,7 @@ class TelemetrySettings(BaseSettings):
model_config = _BASE_CONFIG
enabled: bool = Field(default=True, alias="STRIX_TELEMETRY")
run_record_timings: bool = Field(default=False, alias="STRIX_RUN_RECORD_TIMINGS")
WebSearchProvider = Literal["auto", "perplexity", "exa"]
+2 -2
View File
@@ -1004,7 +1004,7 @@ async def _append_tool_required_message(
message = (
"Your previous message ended a turn without a tool call. Plain text never ends "
"execution and never hands control to the user: it is shown to the user, and the "
"run continues. Continue immediately and call a tool. "
"run continues. Continue immediately and call exactly one tool. "
"If you have nothing to do until the user replies, call wait_for_user; your "
"text already reached them, so do not repeat it. "
"If you are blocked waiting for another agent, call wait_for_agents. "
@@ -1016,7 +1016,7 @@ async def _append_tool_required_message(
message = (
"Your previous response ended the autonomous run without a lifecycle tool "
"call. That is invalid in non-interactive mode; plain text final answers are "
"ignored. Continue immediately and call a tool. "
"ignored. Continue immediately and call exactly one tool. "
f"If your work is complete, call {finish_tool}. "
"If you are blocked waiting for another agent, call wait_for_agents. "
"Otherwise use the appropriate execution or planning tool. "
+38 -1
View File
@@ -4,18 +4,21 @@ from __future__ import annotations
import logging
import math
import time
from datetime import UTC, datetime
from typing import TYPE_CHECKING, Any
from agents.lifecycle import RunHooks
from strix.core.agents import BudgetPolicy, coordinator_from_context
from strix.report.state import get_global_report_state
from strix.report.state import get_global_report_state, record_tool_call
if TYPE_CHECKING:
from agents import RunContextWrapper
from agents.agent import Agent
from agents.items import ModelResponse, TResponseInputItem
from agents.tool import Tool
logger = logging.getLogger(__name__)
@@ -163,6 +166,7 @@ class ReportUsageHooks(RunHooks[dict[str, Any]]):
max_turns: int | None = None,
interactive: bool = False,
budget_policy: BudgetPolicy = "stop",
record_tool_timings: bool = False,
) -> None:
_validate_budget(max_budget_usd)
if max_turns is not None and max_turns <= 0:
@@ -175,6 +179,8 @@ class ReportUsageHooks(RunHooks[dict[str, Any]]):
self._max_turns = max_turns
self._interactive = interactive
self._budget_policy: BudgetPolicy = budget_policy
self._record_tool_timings = record_tool_timings
self._tool_starts: dict[str, tuple[datetime, float]] = {}
@property
def max_budget_usd(self) -> float | None:
@@ -315,6 +321,37 @@ class ReportUsageHooks(RunHooks[dict[str, Any]]):
)
input_items.append({"role": "user", "content": content})
async def on_tool_start(
self,
context: RunContextWrapper[dict[str, Any]],
agent: Agent[dict[str, Any]], # noqa: ARG002
tool: Tool, # noqa: ARG002
) -> None:
call_id = getattr(context, "tool_call_id", None)
if self._record_tool_timings and isinstance(call_id, str):
self._tool_starts[call_id] = (datetime.now(UTC), time.monotonic())
async def on_tool_end(
self,
context: RunContextWrapper[dict[str, Any]],
agent: Agent[dict[str, Any]], # noqa: ARG002
tool: Tool,
result: object, # noqa: ARG002
) -> None:
call_id = getattr(context, "tool_call_id", None)
start = self._tool_starts.pop(call_id, None) if isinstance(call_id, str) else None
if start is None:
return
started_at, started = start
ctx = context.context if isinstance(context.context, dict) else {}
agent_id = ctx.get("agent_id")
record_tool_call(
started_at=started_at,
agent_id=agent_id if isinstance(agent_id, str) else None,
tool=tool.name,
duration_ms=round((time.monotonic() - started) * 1000),
)
async def on_llm_end(
self,
context: RunContextWrapper[dict[str, Any]],
+1 -1
View File
@@ -255,7 +255,7 @@ def make_model_settings(
) -> ModelSettings:
headers = _request_headers(model_name, extra_headers)
model_settings = ModelSettings(
parallel_tool_calls=True if has_tools else None,
parallel_tool_calls=False if has_tools else None,
retry=DEFAULT_MODEL_RETRY,
include_usage=True,
extra_args=request_timeout_extra_args(request_timeout),
+10
View File
@@ -54,6 +54,7 @@ from strix.tools.output_store import (
WORKSPACE_SPILL_DIR,
configure_spill_writer,
)
from strix.tools.reporting.tool import create_vulnerability_report
if TYPE_CHECKING:
@@ -172,6 +173,7 @@ def _compose_root_instructions_override(
interactive: bool,
system_prompt_context: dict[str, Any],
supports_images: bool,
report_tool: str,
) -> str | None:
if root_instructions_override is None:
return None
@@ -186,6 +188,7 @@ def _compose_root_instructions_override(
system_prompt_context=system_prompt_context,
include_scope=False,
supports_images=supports_images,
report_tool=report_tool,
)
return (
f"{base_instructions}\n\n"
@@ -218,6 +221,7 @@ async def run_strix_scan(
mcp_connection_requests: list[McpConnectionRequest] | None = None,
mcp_status_sink: McpStatusSink | None = None,
root_finish_tool: Tool = finish_scan,
report_tool: Tool = create_vulnerability_report,
) -> RunResultBase | None:
"""Run or resume one Strix scan against a sandbox.
@@ -241,6 +245,8 @@ async def run_strix_scan(
way the engine does the connecting, so the caller passes inert configs plus
metadata and never live sessions.
``root_finish_tool`` is the tool the root agent ends the run with.
``report_tool`` is the tool every agent files findings with, in place of
``create_vulnerability_report``.
"""
def report(phase: str) -> None:
@@ -419,6 +425,7 @@ async def run_strix_scan(
max_turns=max_turns,
interactive=interactive,
budget_policy=budget_policy,
record_tool_timings=settings.telemetry.run_record_timings,
)
coordinator.set_budget_limit_setter(hooks.set_max_budget_usd)
if interactive and budget_policy != "pause":
@@ -494,6 +501,7 @@ async def run_strix_scan(
interactive=interactive,
system_prompt_context=root_context,
supports_images=supports_images,
report_tool=report_tool.name,
)
root_agent = build_strix_agent(
@@ -510,6 +518,7 @@ async def run_strix_scan(
instructions_override=root_instructions,
supports_images=supports_images,
finish_tool=root_finish_tool,
report_tool=report_tool,
)
if not is_resume:
@@ -530,6 +539,7 @@ async def run_strix_scan(
strict_tool_schemas=strict_tool_schemas,
system_prompt_context=scope_context,
supports_images=supports_images,
report_tool=report_tool,
)
async def spawn_child_agent(**kwargs: Any) -> dict[str, Any]:
+43
View File
@@ -28,6 +28,8 @@ from strix.telemetry import posthog, scarf
if TYPE_CHECKING:
from agents.usage import Usage
from strix.llm.request_log import LlmRequestEvent
logger = logging.getLogger(__name__)
@@ -1019,6 +1021,47 @@ class StreamedOpenRouterCosts:
streamed_openrouter_costs = StreamedOpenRouterCosts()
def record_llm_request(event: "LlmRequestEvent") -> None:
"""Request-log sink: each provider attempt's timing, kept in run.json's ``llm_requests``."""
report_state = get_global_report_state()
if report_state is None:
return
details = event.details or {}
report_state.run_record.setdefault("llm_requests", []).append(
{
"started_at": event.started_at.isoformat(),
"agent_id": event.agent_id,
"model": event.model,
"provider": details.get("upstream_provider") or event.provider,
"outcome": event.outcome,
"status": event.status_code,
"duration_ms": event.duration_ms,
"ttft_ms": event.time_to_first_token_ms,
"input_tokens": event.input_tokens,
"cached_tokens": event.cached_input_tokens,
"output_tokens": event.output_tokens,
"retry_attempt": event.retry_attempt,
}
)
def record_tool_call(
*, started_at: datetime, agent_id: str | None, tool: str, duration_ms: int
) -> None:
"""Each tool call's timing, kept in run.json's ``tool_calls``."""
report_state = get_global_report_state()
if report_state is None:
return
report_state.run_record.setdefault("tool_calls", []).append(
{
"started_at": started_at.isoformat(),
"agent_id": agent_id,
"tool": tool,
"duration_ms": duration_ms,
}
)
def record_openrouter_provider(provider: Any, usage: Any) -> None:
"""Tally which upstream provider served a stream, from its final usage chunk.
+2 -3
View File
@@ -20,7 +20,7 @@ state, and "I moved on" is not one of them.
**1. `confirmed`** — you have a working PoC or, in white-box, a complete
source → control → sink → impact trace plus evidence the path is
reachable. File it with `create_vulnerability_report`.
reachable. File it as a vulnerability report.
**2. `ruled_out`** — you can name the **specific control** that makes the
code safe, at a specific location, and you have checked that the control
@@ -144,8 +144,7 @@ endpoint is broken/unreachable for unrelated reasons".
## Before You File a Report
Run this pass on every finding before calling
`create_vulnerability_report`:
Run this pass on every finding before filing it:
1. **Argue the other side.** Spend real effort building the strongest
case that this is *not* exploitable, or not as severe as you think.
+11
View File
@@ -93,6 +93,17 @@ def test_per_call_extra_tools_reject_duplicate_registered_names() -> None:
factory.build_strix_agent(is_root=True, extra_tools=[_tool("same_name")])
def test_report_tool_replaces_create_vulnerability_report() -> None:
for is_root in (True, False):
agent = factory.build_strix_agent(is_root=is_root, report_tool=_tool("file_pr_finding"))
names = [t.name for t in agent.tools]
assert "file_pr_finding" in names
assert "create_vulnerability_report" not in names
assert isinstance(agent.instructions, str)
assert "file_pr_finding" in agent.instructions
assert "create_vulnerability_report" not in agent.instructions
def test_instructions_override_is_used_verbatim() -> None:
custom = "You are a scan agent. Follow the provided scope."
+73
View File
@@ -4,6 +4,7 @@ from __future__ import annotations
import json
import uuid
from datetime import UTC, datetime, timedelta
from types import SimpleNamespace
from typing import TYPE_CHECKING, Any
from unittest.mock import MagicMock, call, patch
@@ -18,11 +19,13 @@ from strix.config.models import (
_configure_litellm_compatibility,
_install_openrouter_stream_cost_capture,
)
from strix.core.hooks import ReportUsageHooks
from strix.llm import request_log
from strix.report.state import (
ReportState,
litellm_cost_callback,
openrouter_stream_cost,
record_llm_request,
set_global_report_state,
streamed_openrouter_costs,
)
@@ -455,3 +458,73 @@ def test_openrouter_records_upstream_of_pre_stream_error() -> None:
request_log._http_reply.reset(token)
assert reply.upstream_provider == "InferenceNet"
assert reply.upstream_error_type is None
def test_request_timings_land_in_run_record() -> None:
started = datetime(2026, 10, 19, 12, 0, tzinfo=UTC)
event = request_log.LlmRequestEvent(
call_id="call-1",
route="litellm",
provider="openrouter",
model="openrouter/deepseek/deepseek-v4.1-flash",
api_host="openrouter.ai",
streaming=True,
outcome="success",
status_code=200,
provider_request_id=None,
response_id="gen-1",
error_type=None,
error_message=None,
started_at=started,
finished_at=started + timedelta(milliseconds=4200),
duration_ms=4200,
input_tokens=90_000,
output_tokens=600,
cached_input_tokens=85_000,
agent_id="agent-1",
time_to_first_token_ms=900,
details={"upstream_provider": "Together"},
)
state = ReportState.__new__(ReportState)
state.run_record = {}
try:
set_global_report_state(state)
record_llm_request(event)
finally:
set_global_report_state(None)
assert state.run_record["llm_requests"] == [
{
"started_at": "2026-10-19T12:00:00+00:00",
"agent_id": "agent-1",
"model": "openrouter/deepseek/deepseek-v4.1-flash",
"provider": "Together",
"outcome": "success",
"status": 200,
"duration_ms": 4200,
"ttft_ms": 900,
"input_tokens": 90_000,
"cached_tokens": 85_000,
"output_tokens": 600,
"retry_attempt": 0,
}
]
async def test_tool_call_timings_land_in_run_record() -> None:
hooks = ReportUsageHooks(model="m", record_tool_timings=True)
context = SimpleNamespace(tool_call_id="call-1", context={"agent_id": "agent-1"})
tool = SimpleNamespace(name="terminal_execute")
state = ReportState.__new__(ReportState)
state.run_record = {}
try:
set_global_report_state(state)
await hooks.on_tool_start(context, MagicMock(), tool) # type: ignore[arg-type]
await hooks.on_tool_end(context, MagicMock(), tool, "ok") # type: ignore[arg-type]
finally:
set_global_report_state(None)
[entry] = state.run_record["tool_calls"]
assert entry["agent_id"] == "agent-1"
assert entry["tool"] == "terminal_execute"
assert entry["duration_ms"] >= 0
+2 -2
View File
@@ -310,8 +310,8 @@ def test_make_model_settings_forces_required_for_anyllm_routed_openai_model() ->
assert settings.tool_choice == "required"
def test_make_model_settings_enables_parallel_tool_calls_by_default() -> None:
assert make_model_settings("none", model_name="gpt-4o").parallel_tool_calls is True
def test_make_model_settings_disables_parallel_tool_calls_by_default() -> None:
assert make_model_settings("none", model_name="gpt-4o").parallel_tool_calls is False
def test_make_model_settings_omits_parallel_tool_calls_without_tools() -> None: