langchain-ai/deepagents
1
0
Fork 0
mirror of https://github.com/langchain-ai/deepagents.git synced 2026-07-22 01:35:28 -04:00
Code Issues 169 Packages Projects Releases 16 Wiki Activity

feat(code,sdk): add rubric iteration controls (#4405)

Browse Source 
Closes #4404

Deep Agents Code now uses clearer acceptance-criteria grading messages,
lets the rubric grader inspect offloaded tool-result evidence referenced
in the transcript, defers the rubric iteration default to the SDK, and
adds interactive grader controls for goals and rubrics.

---

This improves the goal and rubric grading UX so the status messages
describe the acceptance criteria check rather than implying the rubric
text itself is wrong. When grading fails with `needs_revision`, the UI
now says the changes need revision. Iteration numbers are hidden unless
the user explicitly configured an iteration cap, keeping the default
`/goal` and `/rubric` flow less noisy.

Deep Agents Code no longer sets its own default rubric iteration cap. If
the user does not pass `--rubric-max-iterations`, dcode omits
`max_iterations` and lets `RubricMiddleware` use the SDK default.
Explicit `--rubric-max-iterations` values are still forwarded, and the
SDK now accepts any positive cap instead of enforcing the previous hard
upper bound.

The rubric grader also gets a narrow `read_file` tool that can only
inspect `/large_tool_results/` paths explicitly referenced in the
transcript. This gives the grader a way to verify offloaded evidence
without exposing broader filesystem access or waiting for an SDK-level
transcript reconstruction change.

The TUI now exposes `/rubric max-iterations <N|clear>` and `/goal
max-iterations <N|clear>` for interactive sessions. Changing the value
restarts the app-owned LangGraph server so the construction-time
`RubricMiddleware` setting takes effect. `/goal model
[provider:model|clear]` and `/goal max-iterations` are goal-first entry
points for the same shared grader settings surfaced by `/rubric model`
and `/rubric max-iterations`.
This commit is contained in:
Mason Daugherty
2026-07-01 03:18:04 -04:00
committed by GitHub
parent a51c8d2b17
commit d6692a7c71
20 changed files with 1320 additions and 127 deletions
Download Patch File Download Diff File Expand all files Collapse all files
libs/code/deepagents_code/_server_config.py
+22 -10
View File
@@ -80,7 +80,7 @@ def _read_env_str(suffix: str) -> str | None:
return os.environ.get(f"{SERVER_ENV_PREFIX}{suffix}")
def _read_env_int(suffix: str, *, default: int) -> int:
def _read_env_int(suffix: str, *, default: int | None) -> int | None:
"""Read a `DEEPAGENTS_CODE_SERVER_*` integer from the environment.
Args:
@@ -88,7 +88,7 @@ def _read_env_int(suffix: str, *, default: int) -> int:
default: Value when the variable is absent or malformed.
Returns:
Parsed integer, or the default when parsing fails.
Parsed integer, or the default when absent or parsing fails.
"""
raw = os.environ.get(f"{SERVER_ENV_PREFIX}{suffix}")
if raw is None:
@@ -264,9 +264,8 @@ class ServerConfig:
`None` reuses the main agent model.
"""
rubric_max_iterations: int = 3
"""Grader iterations per rubric attempt before the agent stops with
`'max_iterations_reached'`."""
rubric_max_iterations: int | None = None
"""Explicit grader iterations per rubric attempt; `None` uses the SDK default."""
sandbox_type: str | None = None
"""Sandbox backend identifier (e.g. `'daytona'`); `None` runs tools on the
@@ -304,13 +303,21 @@ class ServerConfig:
"""Normalize fields and validate invariants.
Raises:
ValueError: If `shell_allow_list` is an empty list.
TypeError: If `rubric_max_iterations` is a boolean.
ValueError: If `shell_allow_list` is an empty list or
`rubric_max_iterations` is non-positive.
"""
if self.sandbox_type == "none":
object.__setattr__(self, "sandbox_type", None)
if self.shell_allow_list is not None and len(self.shell_allow_list) == 0:
msg = "shell_allow_list must be None or non-empty"
raise ValueError(msg)
if isinstance(self.rubric_max_iterations, bool):
msg = "rubric_max_iterations must be None or a positive integer"
raise TypeError(msg)
if self.rubric_max_iterations is not None and self.rubric_max_iterations <= 0:
msg = "rubric_max_iterations must be None or a positive integer"
raise ValueError(msg)
# ------------------------------------------------------------------
# Serialization
@@ -356,7 +363,11 @@ class ServerConfig:
self.interpreter_ptc_acknowledge_unsafe
).lower(),
"RUBRIC_MODEL": self.rubric_model,
"RUBRIC_MAX_ITERATIONS": str(self.rubric_max_iterations),
"RUBRIC_MAX_ITERATIONS": (
str(self.rubric_max_iterations)
if self.rubric_max_iterations is not None
else None
),
"SANDBOX_TYPE": self.sandbox_type,
"SANDBOX_ID": self.sandbox_id,
"SANDBOX_SNAPSHOT_NAME": self.sandbox_snapshot_name,
@@ -406,7 +417,7 @@ class ServerConfig:
"INTERPRETER_PTC_ACKNOWLEDGE_UNSAFE"
),
rubric_model=_read_env_str("RUBRIC_MODEL") or None,
rubric_max_iterations=_read_env_int("RUBRIC_MAX_ITERATIONS", default=3),
rubric_max_iterations=_read_env_int("RUBRIC_MAX_ITERATIONS", default=None),
sandbox_type=_read_env_str("SANDBOX_TYPE"),
sandbox_id=_read_env_str("SANDBOX_ID"),
sandbox_snapshot_name=_read_env_str("SANDBOX_SNAPSHOT_NAME") or None,
@@ -443,7 +454,7 @@ class ServerConfig:
interpreter_ptc: str | list[str] | None = None,
interpreter_ptc_acknowledge_unsafe: bool = False,
rubric_model: str | None = None,
rubric_max_iterations: int = 3,
rubric_max_iterations: int | None = None,
mcp_config_path: str | None,
no_mcp: bool,
trust_project_mcp: bool | None,
@@ -478,7 +489,8 @@ class ServerConfig:
interpreter_ptc_acknowledge_unsafe: Mirror of
`settings.interpreter_ptc_acknowledge_unsafe`.
rubric_model: Grader model spec; `None` reuses the main model.
rubric_max_iterations: Grader iterations per rubric attempt.
rubric_max_iterations: Explicit grader iterations per rubric attempt;
`None` uses the SDK default.
mcp_config_path: Path to MCP config.
no_mcp: Disable MCP.
trust_project_mcp: Trust project MCP servers.
libs/code/deepagents_code/agent.py
+90 -17
View File
@@ -10,13 +10,19 @@ import shutil
import tempfile
import tomllib
import warnings
from pathlib import Path
from pathlib import Path, PurePosixPath
from typing import TYPE_CHECKING, Any, cast
from deepagents import create_deep_agent
from deepagents.backends import CompositeBackend, LocalShellBackend
from deepagents.backends.filesystem import FilesystemBackend
from deepagents.middleware import MemoryMiddleware, RubricMiddleware, SkillsMiddleware
from deepagents.middleware import (
GRADER_SYSTEM_PROMPT,
FilesystemMiddleware,
MemoryMiddleware,
RubricMiddleware,
SkillsMiddleware,
)
# Backwards-compat flag: SDKs before 0.5.4 accept only `list[str]` for
# `SkillsMiddleware.sources`; newer SDKs expose the `SkillSource` alias
@@ -52,6 +58,10 @@ if TYPE_CHECKING:
from deepagents_code.output import OutputFormat
from langchain.agents.middleware.types import AgentMiddleware
from langchain.tools import (
ToolRuntime, # noqa: TC002 # LangChain inspects this annotation for runtime injection.
)
from langchain_core.tools import StructuredTool, tool
from deepagents_code import theme
from deepagents_code._cli_context import CLIContextSchema
@@ -90,6 +100,66 @@ logger = logging.getLogger(__name__)
REQUIRE_COMPACT_TOOL_APPROVAL: bool = True
"""When `True`, `compact_conversation` requires HITL approval like other gated tools."""
_RUBRIC_GRADER_READ_FILE_PREFIX = "/large_tool_results/"
_RUBRIC_GRADER_SYSTEM_PROMPT = (
GRADER_SYSTEM_PROMPT
+ "\n\nWhen the transcript says a tool result was saved under "
+ f"`{_RUBRIC_GRADER_READ_FILE_PREFIX}`, use the `read_file` tool to inspect "
+ "the referenced evidence before deciding that a criterion lacks support. "
+ "Only read paths that are explicitly present in the transcript."
)
def _validate_rubric_grader_read_path(file_path: str) -> str | None:
normalized = file_path.replace("\\", "/")
if not normalized.startswith(_RUBRIC_GRADER_READ_FILE_PREFIX):
return "Rubric grader can only read files under /large_tool_results/."
parts = PurePosixPath(normalized).parts
if ".." in parts or "~" in parts:
return "Invalid path."
return None
def _create_rubric_grader_tools(backend: CompositeBackend) -> list[BaseTool]:
filesystem = FilesystemMiddleware(backend=backend)
sdk_read_file: StructuredTool | None = None
for candidate in filesystem.tools:
if candidate.name == "read_file":
sdk_read_file = cast("StructuredTool", candidate)
break
if sdk_read_file is None:
msg = "SDK read_file tool is unavailable."
raise RuntimeError(msg)
sdk_read_file_func = sdk_read_file.func
if sdk_read_file_func is None:
msg = "SDK read_file tool is missing a sync implementation."
raise RuntimeError(msg)
@tool(description=sdk_read_file.description)
def read_file(
file_path: str,
runtime: ToolRuntime[None, Any],
offset: int = 0,
limit: int = 100,
) -> object:
"""Read an offloaded tool result referenced in the transcript.
Returns:
The SDK `read_file` tool result, or an error message when the path is
outside the grader evidence directory.
"""
if error := _validate_rubric_grader_read_path(file_path):
return error
return sdk_read_file_func(
file_path=file_path,
runtime=runtime,
offset=offset,
limit=limit,
)
return [read_file]
def _sanitize_agent_message_name(agent_name: str) -> str:
"""Return a provider-safe message name for a user-facing agent name.
@@ -275,17 +345,17 @@ def _resolve_ptc_option(
return None
live_names: list[str] = []
for tool in tools:
if isinstance(tool, _BaseTool):
name = tool.name
for candidate in tools:
if isinstance(candidate, _BaseTool):
name = candidate.name
if isinstance(name, str):
live_names.append(name)
elif isinstance(tool, dict):
raw_name = cast("dict[str, Any]", tool).get("name")
elif isinstance(candidate, dict):
raw_name = cast("dict[str, Any]", candidate).get("name")
if isinstance(raw_name, str):
live_names.append(raw_name)
else:
attr = getattr(tool, "name", None)
attr = getattr(candidate, "name", None)
if isinstance(attr, str):
live_names.append(attr)
live_set: set[str] = set(live_names)
@@ -1251,7 +1321,7 @@ def create_cli_agent(
enable_shell: bool = True,
enable_interpreter: bool = False,
rubric_model: str | BaseChatModel | None = None,
rubric_max_iterations: int = 3,
rubric_max_iterations: int | None = None,
checkpointer: BaseCheckpointSaver | None = None,
mcp_server_info: list[MCPServerInfo] | None = None,
cwd: str | Path | None = None,
@@ -1333,8 +1403,9 @@ def create_cli_agent(
A `'provider:model'` string or `BaseChatModel`.
When `None`, the main `model` is reused.
rubric_max_iterations: Grader iterations per rubric attempt before the
agent terminates with `'max_iterations_reached'`.
rubric_max_iterations: Explicit grader iterations per rubric attempt
before the agent terminates with `'max_iterations_reached'`; `None`
uses the SDK default.
checkpointer: Optional checkpointer for session persistence.
When `None`, the graph is compiled without a checkpointer.
mcp_server_info: MCP server metadata to surface in the system prompt.
@@ -1728,12 +1799,14 @@ def create_cli_agent(
message="The middleware `RubricMiddleware` is in beta",
category=Warning,
)
agent_middleware.append(
RubricMiddleware(
model=rubric_model if rubric_model is not None else model,
max_iterations=rubric_max_iterations,
)
)
rubric_kwargs: dict[str, Any] = {
"model": rubric_model if rubric_model is not None else model,
"system_prompt": _RUBRIC_GRADER_SYSTEM_PROMPT,
"tools": _create_rubric_grader_tools(composite_backend),
}
if rubric_max_iterations is not None:
rubric_kwargs["max_iterations"] = rubric_max_iterations
agent_middleware.append(RubricMiddleware(**rubric_kwargs))
# Create the agent
all_subagents: list[SubAgent | CompiledSubAgent | AsyncSubAgent] = [
libs/code/deepagents_code/app.py
+302 -27
View File
@@ -114,6 +114,34 @@ _BLOCKED_GOAL_RETRY_CONTEXT = (
"</dcode_blocked_goal_retry_context>"
)
def _parse_rubric_max_iterations(raw: str) -> tuple[int | None, str | None]:
"""Parse a grader `max-iterations` argument shared by `/rubric` and `/goal`.
Error strings are command-agnostic so they read correctly regardless of the
slash command the user typed.
Args:
raw: The raw argument text following the subcommand.
Returns:
A `(value, error)` pair. On success `error` is `None` and `value` is
either `None` (clear / reset to the SDK default) or a positive int.
On invalid input `value` is `None` and `error` carries a user-facing
message.
"""
value = raw.strip().lower()
if value in {"clear", "default"}:
return None, None
try:
parsed = int(value)
except ValueError:
return None, "Max iterations must be a whole number, or 'clear' to reset."
if parsed < 1:
return None, "Max iterations must be a positive whole number."
return parsed, None
# Serializes process-local read-modify-write operations for `config.toml`.
# Without this, overlapping global-theme and per-terminal-theme saves can each
# read the same pre-mutation state and then clobber the other's keys.
@@ -1258,6 +1286,7 @@ DeferredActionKind = Literal[
"agent_switch",
"mcp_login",
"rubric_model_switch",
"rubric_max_iterations_switch",
]
"""Valid `DeferredAction.kind` values for type-checked deduplication."""
@@ -2225,6 +2254,11 @@ class DeepAgentsApp(App):
self._rubric_model: str | None = (server_kwargs or {}).get("rubric_model")
"""Optional grader model spec for rubric evaluation."""
self._rubric_max_iterations: int | None = (server_kwargs or {}).get(
"rubric_max_iterations"
)
"""Optional grader iterations per rubric attempt."""
self._active_goal: str | None = None
"""Goal objective accepted by the user and backed by the active rubric."""
@@ -7880,9 +7914,10 @@ class DeepAgentsApp(App):
"""Clear every goal and rubric field (sticky, one-shot, goal, pending).
Single reset point so the clear paths cannot drift out of sync over the
nine correlated fields. The grader model (`_rubric_model`) is
intentionally left untouched — it is configured separately via
`/rubric model` and survives `/rubric clear` and `/clear`.
ten correlated fields. Grader settings (`_rubric_model` and
`_rubric_max_iterations`) are intentionally left untouched — they are
configured separately via `/rubric model` and `/rubric max-iterations`
and survive `/rubric clear` and `/clear`.
"""
self._active_rubric = None
self._next_rubric = None
@@ -8176,11 +8211,98 @@ class DeepAgentsApp(App):
self._last_consumed_next_rubric = None
self._last_consumed_next_previous_rubric = None
@staticmethod
def _is_grader_alias_arg(arg: str) -> bool:
"""Whether a `/goal` grader-alias argument is a grader value, not prose.
Grader arguments (`clear`, a model spec like `openai:gpt-5.1`, or an
iteration count) are always a single token, so a multi-word argument is
a plain-language objective that merely starts with `model` /
`max-iterations`. Such objectives must fall through to the objective
workflow instead of being hijacked as a grader command.
Returns:
`True` when the argument is empty or a single token (i.e. a grader
value); `False` for multi-word objective text.
"""
return len(arg.split()) <= 1
async def _dispatch_grader_model(self, command: str, arg: str) -> None:
"""Route a grader-model argument to the shared setter or picker.
Shared by `/rubric model` and the `/goal model` alias so both entry
points stay in lockstep.
"""
await self._mount_message(UserMessage(command))
if not arg:
await self._show_rubric_model_selector()
elif arg.lower() == "clear":
await self._set_rubric_model(None)
else:
await self._set_rubric_model(arg)
async def _dispatch_grader_max_iterations(
self, command: str, arg: str, *, usage_prefix: str
) -> None:
"""Route a grader `max-iterations` argument to the shared setter.
Shared by `/rubric max-iterations` and the `/goal max-iterations` alias.
`usage_prefix` names the invoking command in the empty-argument usage
hint so each entry point advertises its own spelling.
"""
await self._mount_message(UserMessage(command))
if not arg:
await self._mount_message(
AppMessage(f"Usage: {usage_prefix} max-iterations <N|clear>")
)
return
value, error = _parse_rubric_max_iterations(arg)
if error is not None:
await self._mount_message(ErrorMessage(error))
return
await self._set_rubric_max_iterations(value)
def _grader_display_values(self) -> tuple[str, str]:
"""Return display strings for the shared grader model and iteration cap.
Both fall back to human-readable defaults when unset. Shared by
`/goal show` and `/rubric show` so the default wording stays in sync.
"""
model = self._rubric_model or "current chat model"
iterations = (
str(self._rubric_max_iterations)
if self._rubric_max_iterations is not None
else "SDK default"
)
return model, iterations
async def _handle_goal_command(self, command: str) -> None:
"""Handle `/goal` as a user-approved rubric proposal workflow."""
remainder = command.strip()[len("/goal") :].strip()
subcommand = remainder.lower()
# Grader settings are shared with `/rubric` — one `RubricMiddleware`
# grades both goals and ad-hoc rubrics. Expose them here as aliases that
# call the same setters (no separate state) so goal-first users can tune
# grading without discovering `/rubric`. These intercept ahead of every
# `/goal` subcommand below, but only when the argument is a single grader
# token (a model spec, an iteration count, or `clear`); a multi-word
# objective that merely starts with `model` / `max-iterations` still
# falls through to the objective workflow.
grader_sub, _, grader_arg = remainder.partition(" ")
grader_sub = grader_sub.lower()
grader_arg = grader_arg.strip()
if grader_sub == "model" and self._is_grader_alias_arg(grader_arg):
await self._dispatch_grader_model(command, grader_arg)
return
if grader_sub in {"max-iterations", "max_iterations"} and (
self._is_grader_alias_arg(grader_arg)
):
await self._dispatch_grader_max_iterations(
command, grader_arg, usage_prefix="/goal"
)
return
if not remainder or subcommand in {"show", "status"}:
await self._mount_message(UserMessage(command))
await self._show_goal_state()
@@ -8224,10 +8346,13 @@ class DeepAgentsApp(App):
"Usage:\n"
" /goal <objective>\n"
" /goal show\n"
" /goal clear\n\n"
" /goal clear\n"
" /goal model [provider:model|clear]\n"
" /goal max-iterations <N|clear>\n\n"
"Use /goal when you have a plain-language objective; dcode will "
"draft a checklist and ask before applying it. Use /rubric to set "
"the checklist text directly."
"draft a checklist and ask before applying it. Once accepted, the "
"goal stays active for this thread until completed, blocked, or "
"cleared. Follow-up prompts continue working toward that goal."
)
async def _show_goal_state(self) -> None:
@@ -8253,7 +8378,21 @@ class DeepAgentsApp(App):
],
)
if lines:
lines.append("Commands:\n/goal clear\n/goal show")
grader_model, grader_iterations = self._grader_display_values()
lines.append(
f"Grader: {grader_model} · max iterations: {grader_iterations}"
)
if self._active_goal and self._goal_status == "active":
lines.append(
"Goal is active for this thread until completed, blocked, or "
"cleared.\nFollow-up prompts will continue working toward this "
"goal."
)
lines.append(
"Commands:\n/goal clear\n/goal show\n"
"/goal model [provider:model|clear]\n"
"/goal max-iterations <N|clear>"
)
await self._mount_message(AppMessage("\n\n".join(lines)))
return
await self._mount_message(
@@ -8461,7 +8600,13 @@ class DeepAgentsApp(App):
self._pending_goal_rubric = None
self._sync_status_rubric()
persisted = await self._persist_goal_rubric_state()
await self._mount_message(AppMessage("Goal accepted. Rubric set."))
await self._mount_message(
AppMessage(
"Goal accepted. It will stay active for this thread until completed, "
"blocked, or cleared.\nUse /goal show to inspect it or /goal clear "
"to remove it."
)
)
if not persisted:
await self._mount_message(
ErrorMessage(
@@ -8610,16 +8755,14 @@ class DeepAgentsApp(App):
await self._mount_goal_rubric_result("Rubric cleared.", persisted=persisted)
return
if subcommand in {"max-iterations", "max_iterations"}:
await self._dispatch_grader_max_iterations(
command, arg, usage_prefix="/rubric"
)
return
if subcommand == "model":
if not arg:
await self._mount_message(UserMessage(command))
await self._show_rubric_model_selector()
return
await self._mount_message(UserMessage(command))
if arg.lower() == "clear":
await self._set_rubric_model(None)
else:
await self._set_rubric_model(arg)
await self._dispatch_grader_model(command, arg)
return
await self._mount_message(UserMessage(command))
@@ -8635,14 +8778,21 @@ class DeepAgentsApp(App):
" /rubric file <path>\n"
" /rubric show\n"
" /rubric clear\n"
" /rubric model [provider:model|clear]\n\n"
"Alias: /criteria"
" /rubric model [provider:model|clear]\n"
" /rubric max-iterations <N|clear>\n\n"
"Use /rubric next for a one-turn quality gate. Use /rubric set "
"when you want explicit acceptance criteria to persist across turns."
)
async def _show_rubric_usage(self) -> None:
"""Render rubric command usage with current active state if present."""
parts = [self._rubric_usage_text()]
if self._active_rubric or self._next_rubric or self._rubric_model:
if (
self._active_rubric
or self._next_rubric
or self._rubric_model
or self._rubric_max_iterations is not None
):
state: list[str] = []
if self._active_rubric:
state.append("Sticky rubric is set.")
@@ -8650,6 +8800,8 @@ class DeepAgentsApp(App):
state.append("Next-turn rubric is set.")
if self._rubric_model:
state.append(f"Rubric grader model: {self._rubric_model}")
if self._rubric_max_iterations is not None:
state.append(f"Rubric max iterations: {self._rubric_max_iterations}")
parts.append(
"Current state:\n" + "\n".join(f" - {line}" for line in state)
)
@@ -8662,13 +8814,16 @@ class DeepAgentsApp(App):
lines.append(f"Rubric:\n{self._active_rubric}")
if self._next_rubric:
lines.append(f"Next-turn rubric:\n{self._next_rubric}")
if self._rubric_model:
lines.append(f"Rubric grader model: {self._rubric_model}")
if not lines:
if not lines and not self._rubric_model and self._rubric_max_iterations is None:
await self._mount_message(AppMessage("No rubric set."))
return
if not self._rubric_model:
lines.append("Rubric grader model: current chat model")
grader_model, grader_iterations = self._grader_display_values()
lines.extend(
[
f"Rubric grader model: {grader_model}",
f"Rubric max iterations: {grader_iterations}",
]
)
await self._mount_message(AppMessage("\n\n".join(lines)))
async def _set_rubric_from_file(self, path_arg: str) -> None:
@@ -8751,6 +8906,87 @@ class DeepAgentsApp(App):
)
self.push_screen(screen, handle_result)
async def _set_rubric_max_iterations(self, value: int | None) -> None:
"""Set the grader iterations per rubric attempt used by `RubricMiddleware`."""
from functools import partial
from deepagents_code._env_vars import SERVER_ENV_PREFIX
if self._agent_running or self._shell_running or self._connecting:
self._defer_action(
DeferredAction(
kind="rubric_max_iterations_switch",
execute=partial(self._set_rubric_max_iterations, value),
),
)
self.notify(
"Rubric max iterations will change after current work finishes."
)
return
if self._server_kwargs is None and self._server_proc is None:
await self._mount_message(
ErrorMessage(
"Rubric max-iterations switching is unavailable in this session "
"because it does not own a restartable server."
)
)
return
if self._rubric_max_iterations == value:
message = (
f"Rubric max iterations already set to {value}."
if value is not None
else "Rubric max iterations already use the SDK default."
)
await self._mount_message(AppMessage(message))
return
previous = self._rubric_max_iterations
self._rubric_max_iterations = value
if self._server_kwargs is not None:
self._server_kwargs["rubric_max_iterations"] = value
if self._server_proc is not None:
env_key = f"{SERVER_ENV_PREFIX}RUBRIC_MAX_ITERATIONS"
env_value = str(value) if value is not None else ""
self._server_proc.update_env(
**{env_key: env_value},
)
restarted = await self._respawn_server(
log_message=(
"Server restart failed while changing rubric max iterations"
),
mcp_failure_log=(
"MCP metadata preload after rubric max-iterations change failed"
),
mcp_failure_toast=(
"MCP tool metadata could not be refreshed. Use /mcp to check."
),
)
if not restarted:
self._rubric_max_iterations = previous
if self._server_kwargs is not None:
self._server_kwargs["rubric_max_iterations"] = previous
# A failed restart keeps `env_value` staged in the server's
# one-shot env overrides (retained for retry) and never persists
# it. Re-stage `previous` so a later restart cannot resurrect the
# value this command just rolled back.
self._server_proc.update_env(
**{env_key: str(previous) if previous is not None else ""},
)
return
self._server_proc.persist_env(**{env_key: env_value})
if value is None:
await self._mount_message(
AppMessage("Rubric max iterations cleared; using the SDK default."),
)
else:
await self._mount_message(
AppMessage(f"Rubric max iterations set to {value}."),
)
async def _set_rubric_model(self, model_spec: str | None) -> None:
"""Set the grader model used by `RubricMiddleware`."""
from functools import partial
@@ -8790,7 +9026,7 @@ class DeepAgentsApp(App):
ErrorMessage(
f"Missing credentials: {auth_status.missing_detail()}\n\n"
f"Run `/auth` for the '{auth_status.provider}' provider, "
f"then re-issue `/rubric model {model_spec}`.",
f"then set the grader model again.",
),
)
return
@@ -8830,6 +9066,13 @@ class DeepAgentsApp(App):
self._rubric_model = previous
if self._server_kwargs is not None:
self._server_kwargs["rubric_model"] = previous
# A failed restart keeps the new value staged in the server's
# one-shot env overrides (retained for retry). Re-stage
# `previous` so a later restart cannot resurrect the model this
# command just rolled back.
self._server_proc.update_env(
**{f"{SERVER_ENV_PREFIX}RUBRIC_MODEL": previous or ""},
)
return
if display:
@@ -9625,12 +9868,37 @@ class DeepAgentsApp(App):
# Any send (typed reply or skill invocation) counts as the user
# acting on a blocked goal, so reset it and attach one-turn context.
blocker_note = await self._reset_blocked_goal_for_user_turn()
resuming_blocked = blocker_note is not None
blocked_goal_retry_context = (
self._blocked_goal_retry_context(blocker_note)
if blocker_note is not None
if resuming_blocked
else None
)
# `_reset_blocked_goal_for_user_turn` flips a blocked goal to active
# just above, so the status alone can't distinguish an already-active
# goal from one resumed this turn. Branch the wording on the reset
# signal instead. The status guard still holds: a failed reset rolls
# the status back to blocked, so no notice fires in that case.
#
# The `message != _active_goal` check suppresses the notice on the
# goal-setting turn, and works only because acceptance sends the
# objective verbatim as the message. If a future change wraps or
# annotates the objective before sending (as the skill path already
# does with its envelope prompt), the equality would no longer match
# and the notice would wrongly fire on the initial turn.
if (
self._active_goal
and self._goal_status == "active"
and message.strip() != self._active_goal.strip()
):
notice = (
f"Resuming previously blocked goal: {self._active_goal}"
if resuming_blocked
else f"Continuing active goal: {self._active_goal}"
)
await self._mount_message(AppMessage(notice))
if self._chat_input:
self._chat_input.set_cursor_active(active=False)
@@ -12346,6 +12614,13 @@ class DeepAgentsApp(App):
self._default_assistant_id = previous_default_agent
if self._server_kwargs is not None:
self._server_kwargs["assistant_id"] = previous_agent
# A failed restart keeps `agent_name` staged in the server's
# one-shot env overrides (retained for retry). Re-stage the
# previous agent so a later restart cannot resurrect the swap
# target this handler just rolled back.
server_proc.update_env(
**{f"{SERVER_ENV_PREFIX}ASSISTANT_ID": previous_agent or ""},
)
self._agent = None
self._connecting = False
self._reconnecting = False
libs/code/deepagents_code/command_registry.py
+6 -4
View File
@@ -108,8 +108,10 @@ COMMANDS: tuple[SlashCommand, ...] = (
name="/goal",
description="Set a persistent objective by drafting acceptance criteria",
bypass_tier=BypassTier.QUEUED,
hidden_keywords="objective criteria acceptance rubric",
argument_hint="[<objective>|show|clear]",
hidden_keywords=(
"objective criteria acceptance rubric grader grading model iterations"
),
argument_hint="[<objective>|show|clear|model|max-iterations]",
),
SlashCommand(
name="/editor",
@@ -180,8 +182,8 @@ COMMANDS: tuple[SlashCommand, ...] = (
name="/rubric",
description="Set explicit acceptance criteria for rubric grading",
bypass_tier=BypassTier.IMMEDIATE_UI,
hidden_keywords="criteria acceptance grader grading evaluation",
argument_hint="[set|next|file|show|clear|model]",
hidden_keywords="criteria acceptance grader grading evaluation iterations",
argument_hint="[set|next|file|show|clear|model|max-iterations]",
aliases=("/criteria",),
),
SlashCommand(
libs/code/deepagents_code/main.py
+2 -2
View File
@@ -1585,8 +1585,8 @@ def parse_args() -> argparse.Namespace:
dest="rubric_max_iterations",
type=positive_int,
metavar="N",
help="Grader iterations per rubric attempt before stopping (must be "
">= 1, default 3).",
help="Override grader iterations per rubric attempt before stopping "
"(must be >= 1; defaults to the SDK setting).",
)
parser.add_argument(
libs/code/deepagents_code/non_interactive.py
+27 -12
View File
@@ -278,6 +278,9 @@ class StreamState:
spinner: _ConsoleSpinner | None = None
"""Optional animated spinner shown during agent work in verbose mode."""
show_rubric_iterations: bool = False
"""Whether rubric lifecycle messages should include iteration numbers."""
@dataclass
class ThreadUrlLookupState:
@@ -505,13 +508,17 @@ def _process_rubric_event(
if event_type == "rubric_evaluation_start":
# `iteration` is untrusted streamed payload; only render the 1-based
# number when it is actually an int, mirroring the interactive twin in
# `textual_adapter._format_rubric_event`. A non-int previously raised
# `TypeError` here and aborted the whole non-interactive run.
# number when it is actually an int and the user explicitly requested an
# iteration cap. A non-int previously raised `TypeError` here and aborted
# the whole non-interactive run.
iteration = data.get("iteration", 0)
label = f" (iteration {iteration + 1})" if isinstance(iteration, int) else ""
label = (
f" (iteration {iteration + 1})"
if state.show_rubric_iterations and isinstance(iteration, int)
else ""
)
console.print(
f"[dim]⏳ Grading against rubric{label}…[/dim]",
f"[dim]⏳ Checking acceptance criteria{label}…[/dim]",
highlight=False,
)
if state.spinner:
@@ -521,11 +528,11 @@ def _process_rubric_event(
result = data.get("result")
explanation = (data.get("explanation") or "").strip()
if result == "satisfied":
console.print("[green]✓ Rubric satisfied[/green]", highlight=False)
console.print("[green]✓ Acceptance criteria satisfied[/green]", highlight=False)
elif result == "needs_revision":
suffix = f": {escape_markup(explanation)}" if explanation else ""
console.print(
f"[yellow]↻ Rubric needs revision{suffix}[/yellow]", highlight=False
f"[yellow]↻ Changes need revision{suffix}[/yellow]", highlight=False
)
for criterion in data.get("criteria", []):
if isinstance(criterion, dict) and not criterion.get("passed", True):
@@ -535,7 +542,8 @@ def _process_rubric_event(
console.print(f"[yellow] ✗ {name}{detail}[/yellow]", highlight=False)
elif result == "max_iterations_reached":
console.print(
"[yellow]⚠ Rubric not satisfied (max iterations reached)[/yellow]",
"[yellow]⚠ Acceptance criteria not satisfied "
"(iteration limit reached)[/yellow]",
highlight=False,
)
elif result in {"failed", "grader_error"}:
@@ -778,6 +786,7 @@ async def _run_agent_loop(
thread_url_lookup: ThreadUrlLookupState | None = None,
max_turns: int | None = None,
rubric: str | None = None,
show_rubric_iterations: bool = False,
) -> None:
"""Run the agent and handle HITL interrupts until the task completes.
@@ -808,12 +817,19 @@ async def _run_agent_loop(
graph's `rubric` state field.
`None` leaves it unset (no grading).
show_rubric_iterations: Whether rubric lifecycle messages should include
iteration numbers.
Raises:
HITLIterationLimitError: If the effective turn limit is exceeded.
"""
spinner = None if quiet else _ConsoleSpinner(console)
state = StreamState(quiet=quiet, stream=stream, spinner=spinner)
state = StreamState(
quiet=quiet,
stream=stream,
spinner=spinner,
show_rubric_iterations=show_rubric_iterations,
)
user_msg: dict[str, Any] = {"role": "user", "content": message}
if message_kwargs:
user_msg.update(message_kwargs)
@@ -1297,9 +1313,7 @@ async def run_non_interactive(
interpreter_ptc=interpreter_ptc,
interpreter_ptc_acknowledge_unsafe=interpreter_ptc_acknowledge_unsafe,
rubric_model=rubric_model,
rubric_max_iterations=(
rubric_max_iterations if rubric_max_iterations is not None else 3
),
rubric_max_iterations=rubric_max_iterations,
mcp_config_path=mcp_config_path,
no_mcp=no_mcp,
trust_project_mcp=trust_project_mcp,
@@ -1336,6 +1350,7 @@ async def run_non_interactive(
thread_url_lookup=thread_url_lookup,
max_turns=max_turns,
rubric=rubric,
show_rubric_iterations=rubric_max_iterations is not None,
)
except KeyboardInterrupt:
libs/code/deepagents_code/server.py
+28
View File
@@ -20,6 +20,7 @@ from pathlib import Path
from typing import TYPE_CHECKING, Any, Self
from urllib.parse import quote
from deepagents_code._env_vars import SERVER_ENV_PREFIX
from deepagents_code.config import _INHERITED_PYTHONPATH_ENV
if TYPE_CHECKING:
@@ -406,6 +407,7 @@ class ServerProcess:
self._temp_dir: tempfile.TemporaryDirectory | None = None
self._log_file: tempfile.NamedTemporaryFile | None = None # ty: ignore[invalid-type-form]
self._env_overrides: dict[str, str] = {}
self._persistent_env_overrides: dict[str, str] = {}
@property
def url(self) -> str:
@@ -502,6 +504,13 @@ class ServerProcess:
cmd = _build_server_cmd(config_path, host=self.host, port=self.port)
env = _build_server_env()
# Persisted overrides are defaults; a one-shot override staged via
# `update_env()` for THIS restart must win over them. `_env_overrides`
# is already reflected in the `os.environ` copy above (applied by
# `_scoped_env_overrides`), but persisted values would otherwise shadow
# a freshly staged value, so re-apply the one-shot set last.
env.update(self._persistent_env_overrides)
env.update(self._env_overrides)
logger.info("Starting langgraph dev server: %s", " ".join(cmd))
self._log_file = tempfile.NamedTemporaryFile( # noqa: SIM115
@@ -693,6 +702,25 @@ class ServerProcess:
"""
self._env_overrides.update(overrides)
def persist_env(self, **overrides: str) -> None:
"""Persist env var overrides for every future subprocess start.
Args:
**overrides: Key/value env var pairs that should be passed to all
future server subprocesses.
Raises:
ValueError: If an override is not an app-owned server env var.
"""
invalid = [key for key in overrides if not key.startswith(SERVER_ENV_PREFIX)]
if invalid:
msg = (
"persistent server env overrides must use the "
f"{SERVER_ENV_PREFIX!r} prefix"
)
raise ValueError(msg)
self._persistent_env_overrides.update(overrides)
async def restart(self, *, timeout: float = _HEALTH_TIMEOUT) -> None: # noqa: ASYNC109
"""Restart the server process, reusing the existing config directory.
libs/code/deepagents_code/server_manager.py
+6 -4
View File
@@ -286,7 +286,7 @@ async def start_server_and_get_agent(
interpreter_ptc: str | list[str] | None = None,
interpreter_ptc_acknowledge_unsafe: bool = False,
rubric_model: str | None = None,
rubric_max_iterations: int = 3,
rubric_max_iterations: int | None = None,
mcp_config_path: str | None = None,
no_mcp: bool = False,
trust_project_mcp: bool | None = None,
@@ -315,7 +315,8 @@ async def start_server_and_get_agent(
interpreter_ptc_acknowledge_unsafe: Explicit acknowledgement for
`interpreter_ptc="all"` outside of `auto_approve`.
rubric_model: Grader model spec; `None` reuses the main model.
rubric_max_iterations: Grader iterations per rubric attempt.
rubric_max_iterations: Explicit grader iterations per rubric attempt;
`None` uses the SDK default.
mcp_config_path: Path to MCP config.
no_mcp: Disable MCP.
trust_project_mcp: Trust project MCP servers.
@@ -420,7 +421,7 @@ async def server_session(
interpreter_ptc: str | list[str] | None = None,
interpreter_ptc_acknowledge_unsafe: bool = False,
rubric_model: str | None = None,
rubric_max_iterations: int = 3,
rubric_max_iterations: int | None = None,
mcp_config_path: str | None = None,
no_mcp: bool = False,
trust_project_mcp: bool | None = None,
@@ -452,7 +453,8 @@ async def server_session(
interpreter_ptc_acknowledge_unsafe: Explicit acknowledgement for
`interpreter_ptc="all"` outside of `auto_approve`.
rubric_model: Grader model spec; `None` reuses the main model.
rubric_max_iterations: Grader iterations per rubric attempt.
rubric_max_iterations: Explicit grader iterations per rubric attempt;
`None` uses the SDK default.
mcp_config_path: Path to MCP config.
no_mcp: Disable MCP.
trust_project_mcp: Trust project MCP servers.
libs/code/deepagents_code/textual_adapter.py
+15 -9
View File
@@ -226,12 +226,15 @@ def _format_rubric_event(data: dict[str, Any]) -> str | None:
event_type = data.get("type")
if event_type == "rubric_evaluation_start":
iteration = data.get("iteration", 0)
if isinstance(iteration, int):
return (
f"{glyphs.hourglass} Grading against rubric "
f"(iteration {iteration + 1}){glyphs.ellipsis}"
)
return f"{glyphs.hourglass} Grading against rubric{glyphs.ellipsis}"
show_iteration = data.get("show_iteration") is True
label = (
f" (iteration {iteration + 1})"
if show_iteration and isinstance(iteration, int)
else ""
)
return (
f"{glyphs.hourglass} Checking acceptance criteria{label}{glyphs.ellipsis}"
)
if event_type != "rubric_evaluation_end":
return None
@@ -240,10 +243,10 @@ def _format_rubric_event(data: dict[str, Any]) -> str | None:
if result is None:
return None
if result == "satisfied":
return f"{glyphs.checkmark} Rubric satisfied"
return f"{glyphs.checkmark} Acceptance criteria satisfied"
if result == "needs_revision":
lines = [
f"{glyphs.retry} Rubric needs revision"
f"{glyphs.retry} Changes need revision"
+ (f": {explanation}" if explanation else ""),
]
for criterion in data.get("criteria", []):
@@ -253,7 +256,10 @@ def _format_rubric_event(data: dict[str, Any]) -> str | None:
lines.append(f" {glyphs.error} {name}" + (f" — {gap}" if gap else ""))
return "\n".join(lines)
if result == "max_iterations_reached":
return f"{glyphs.warning} Rubric not satisfied (max iterations reached)"
return (
f"{glyphs.warning} Acceptance criteria not satisfied "
"(iteration limit reached)"
)
if result in {"failed", "grader_error"}:
label = "grader failed" if result == "failed" else "grader error"
return (
libs/code/deepagents_code/ui.py
+1 -1
View File
@@ -198,7 +198,7 @@ def show_help() -> None:
"(defaults to main model)"
)
console.print(
" --rubric-max-iterations N Grader iterations per rubric attempt (default 3)"
" --rubric-max-iterations N Override grader iterations per rubric attempt"
)
console.print(
" --timeout SECONDS Hard wall-clock limit; exits 124 on expiry"
libs/code/tests/unit_tests/test_agent.py
+71
View File
@@ -24,6 +24,7 @@ from deepagents_code.agent import (
DEFAULT_AGENT_NAME,
_add_interrupt_on,
_apply_inherited_pythonpath,
_create_rubric_grader_tools,
_format_delete_description,
_format_edit_file_description,
_format_execute_description,
@@ -3217,6 +3218,76 @@ class TestCreateCliAgentInterpreterWiring:
assert len(rubrics) == 1
assert rubrics[0]._model == "custom-grader-model"
assert rubrics[0].max_iterations == 5
assert "use the `read_file` tool" in rubrics[0]._system_prompt
assert [tool.name for tool in rubrics[0]._tools] == ["read_file"]
def test_omits_default_rubric_max_iterations(self, tmp_path: Path) -> None:
mock_settings = self._build_mock_settings(tmp_path)
mock_agent = Mock()
mock_agent.with_config.return_value = mock_agent
fake_model = _make_fake_chat_model()
with (
patch("deepagents_code.agent.settings", mock_settings),
patch("deepagents_code.agent.SkillsMiddleware"),
patch("deepagents_code.agent.MemoryMiddleware"),
patch("deepagents_code.agent.RubricMiddleware") as mock_rubric,
patch(
"deepagents_code.agent.create_deep_agent",
return_value=mock_agent,
),
patch(
"deepagents._models.init_chat_model",
return_value=fake_model,
),
):
create_cli_agent(
model="fake-model",
assistant_id="test",
enable_memory=False,
enable_skills=False,
enable_shell=False,
)
_, kwargs = mock_rubric.call_args
assert "max_iterations" not in kwargs
def test_rubric_grader_read_tool_only_reads_large_results(
self, tmp_path: Path
) -> None:
from deepagents.backends import CompositeBackend
from deepagents.backends.filesystem import FilesystemBackend
large_results = FilesystemBackend(
root_dir=tmp_path / "large",
virtual_mode=True,
)
project = FilesystemBackend(
root_dir=tmp_path / "project",
virtual_mode=False,
)
backend = CompositeBackend(
default=project,
routes={"/large_tool_results/": large_results},
)
backend.upload_files(
[("/large_tool_results/tool-call-id", b"first\nsecond\nthird")]
)
read_tool = cast("Any", _create_rubric_grader_tools(backend)[0])
runtime = SimpleNamespace(tool_call_id="grader-read")
allowed = read_tool.func(
file_path="/large_tool_results/tool-call-id",
runtime=runtime,
limit=2,
)
denied = read_tool.func(
file_path="/Users/mason/.ssh/id_rsa",
runtime=runtime,
)
assert "1\tfirst" in allowed.content
assert "2\tsecond" in allowed.content
assert "can only read" in denied
def test_appends_interpreter_middleware_when_enabled(self, tmp_path: Path) -> None:
from langchain_quickjs import CodeInterpreterMiddleware
libs/code/tests/unit_tests/test_app.py
+564 -3
View File
@@ -49,6 +49,7 @@ from deepagents_code.app import (
_build_whats_new_message,
_display_model_label,
_extra_is_ready,
_parse_rubric_max_iterations,
_ThreadHistoryPayload,
_warn_discarded_goal_channels,
)
@@ -4732,14 +4733,225 @@ class TestGoalCommand:
return future
def test_goal_usage_text_explains_goal_vs_rubric(self) -> None:
"""Goal help should clearly distinguish drafting from explicit criteria."""
"""Goal help should explain drafting and that a goal persists once set."""
usage = DeepAgentsApp._goal_usage_text()
assert "Use /goal when you have a plain-language objective" in usage
assert "draft a checklist and ask before applying it" in usage
assert "Use /rubric to set the checklist text directly." in usage
assert "the goal stays active for this thread" in usage
assert "when you want dcode to propose" not in usage
def test_goal_usage_text_mentions_grader_settings(self) -> None:
"""Goal help should surface the grader settings without alias wording."""
usage = DeepAgentsApp._goal_usage_text()
assert "/goal model [provider:model|clear]" in usage
assert "/goal max-iterations <N|clear>" in usage
assert "aliases for /rubric model" not in usage
async def test_goal_model_alias_dispatches_to_rubric_setter(self) -> None:
"""`/goal model <spec>` should route to the shared grader-model setter."""
app = DeepAgentsApp(agent=MagicMock())
async with app.run_test() as pilot:
await pilot.pause()
with patch.object(
app, "_set_rubric_model", new_callable=AsyncMock
) as setter:
await app._handle_command("/goal model openai:gpt-5.1")
await pilot.pause()
setter.assert_awaited_once_with("openai:gpt-5.1")
async def test_goal_model_alias_bare_opens_selector(self) -> None:
"""Bare `/goal model` should open the shared grader-model picker."""
app = DeepAgentsApp(agent=MagicMock())
async with app.run_test() as pilot:
await pilot.pause()
with patch.object(
app, "_show_rubric_model_selector", new_callable=AsyncMock
) as selector:
await app._handle_command("/goal model")
await pilot.pause()
selector.assert_awaited_once()
async def test_goal_model_alias_clears_grader_model(self) -> None:
"""`/goal model clear` should clear the shared grader model."""
app = DeepAgentsApp(agent=MagicMock())
async with app.run_test() as pilot:
await pilot.pause()
with patch.object(
app, "_set_rubric_model", new_callable=AsyncMock
) as setter:
await app._handle_command("/goal model clear")
await pilot.pause()
setter.assert_awaited_once_with(None)
async def test_goal_max_iterations_alias_dispatches_to_setter(self) -> None:
"""`/goal max-iterations <n>` should route to the shared cap setter."""
app = DeepAgentsApp(agent=MagicMock())
async with app.run_test() as pilot:
await pilot.pause()
with patch.object(
app, "_set_rubric_max_iterations", new_callable=AsyncMock
) as setter:
await app._handle_command("/goal max-iterations 21")
await pilot.pause()
setter.assert_awaited_once_with(21)
async def test_goal_max_iterations_alias_rejects_non_positive(self) -> None:
"""A non-positive `/goal max-iterations` value should not call the setter."""
app = DeepAgentsApp(agent=MagicMock())
async with app.run_test() as pilot:
await pilot.pause()
with patch.object(
app, "_set_rubric_max_iterations", new_callable=AsyncMock
) as setter:
await app._handle_command("/goal max-iterations -5")
await pilot.pause()
setter.assert_not_awaited()
rendered = "\n".join(str(w._content) for w in app.query(ErrorMessage))
assert "positive whole number" in rendered
async def test_goal_show_displays_grader_line(self) -> None:
"""`/goal show` should surface the shared grader model and cap."""
app = DeepAgentsApp(agent=MagicMock())
async with app.run_test() as pilot:
await pilot.pause()
app._active_goal = "ship the feature"
app._goal_status = "active"
app._rubric_model = "openai:gpt-5.1"
app._rubric_max_iterations = 12
await app._handle_command("/goal show")
await pilot.pause()
rendered = "\n".join(str(w._content) for w in app.query(AppMessage))
assert "Grader: openai:gpt-5.1 · max iterations: 12" in rendered
async def test_goal_show_grader_line_reports_defaults(self) -> None:
"""The grader line should spell out defaults when the grader is unset."""
app = DeepAgentsApp(agent=MagicMock())
async with app.run_test() as pilot:
await pilot.pause()
app._active_goal = "ship the feature"
app._goal_status = "active"
await app._handle_command("/goal show")
await pilot.pause()
rendered = "\n".join(str(w._content) for w in app.query(AppMessage))
assert (
"Grader: current chat model · max iterations: SDK default" in rendered
)
async def test_goal_show_footer_lists_grader_aliases(self) -> None:
"""`/goal show` should advertise the grader-alias commands in its footer."""
app = DeepAgentsApp(agent=MagicMock())
async with app.run_test() as pilot:
await pilot.pause()
app._active_goal = "ship the feature"
app._goal_status = "active"
await app._handle_command("/goal show")
await pilot.pause()
rendered = "\n".join(str(w._content) for w in app.query(AppMessage))
assert "/goal model [provider:model|clear]" in rendered
assert "/goal max-iterations <N|clear>" in rendered
async def test_goal_max_iterations_alias_no_arg_shows_usage(self) -> None:
"""Bare `/goal max-iterations` shows goal-branded usage without setting."""
app = DeepAgentsApp(agent=MagicMock())
async with app.run_test() as pilot:
await pilot.pause()
with patch.object(
app, "_set_rubric_max_iterations", new_callable=AsyncMock
) as setter:
await app._handle_command("/goal max-iterations")
await pilot.pause()
setter.assert_not_awaited()
rendered = "\n".join(str(w._content) for w in app.query(AppMessage))
assert "Usage: /goal max-iterations <N|clear>" in rendered
async def test_goal_max_iterations_alias_clears_cap(self) -> None:
"""`/goal max-iterations clear` should route to the setter with `None`."""
app = DeepAgentsApp(agent=MagicMock())
async with app.run_test() as pilot:
await pilot.pause()
with patch.object(
app, "_set_rubric_max_iterations", new_callable=AsyncMock
) as setter:
await app._handle_command("/goal max-iterations clear")
await pilot.pause()
setter.assert_awaited_once_with(None)
async def test_goal_max_iterations_underscore_alias_dispatches(self) -> None:
"""The `max_iterations` underscore spelling should also route through."""
app = DeepAgentsApp(agent=MagicMock())
async with app.run_test() as pilot:
await pilot.pause()
with patch.object(
app, "_set_rubric_max_iterations", new_callable=AsyncMock
) as setter:
await app._handle_command("/goal max_iterations 8")
await pilot.pause()
setter.assert_awaited_once_with(8)
async def test_goal_model_alias_ignores_multiword_objective(self) -> None:
"""A multi-word objective starting with `model` stays an objective."""
app = DeepAgentsApp(agent=MagicMock())
async with app.run_test() as pilot:
await pilot.pause()
request = AsyncMock(
return_value=self._goal_review_future({"type": "cancelled"})
)
with (
patch.object(
app, "_set_rubric_model", new_callable=AsyncMock
) as setter,
patch.object(app, "_generate_goal_rubric", return_value="- tests pass"),
patch.object(app, "_request_goal_review", request),
patch.object(app, "_set_spinner", new_callable=AsyncMock),
):
await app._handle_command("/goal model the checkout flow")
await pilot.pause()
await pilot.pause()
setter.assert_not_awaited()
request.assert_awaited_once_with("model the checkout flow", "- tests pass")
async def test_goal_max_iterations_alias_ignores_multiword_objective(self) -> None:
"""A multi-word objective starting with `max-iterations` stays an objective."""
app = DeepAgentsApp(agent=MagicMock())
async with app.run_test() as pilot:
await pilot.pause()
request = AsyncMock(
return_value=self._goal_review_future({"type": "cancelled"})
)
with (
patch.object(
app, "_set_rubric_max_iterations", new_callable=AsyncMock
) as setter,
patch.object(app, "_generate_goal_rubric", return_value="- tests pass"),
patch.object(app, "_request_goal_review", request),
patch.object(app, "_set_spinner", new_callable=AsyncMock),
):
await app._handle_command("/goal max-iterations for the parser")
await pilot.pause()
await pilot.pause()
setter.assert_not_awaited()
request.assert_awaited_once_with(
"max-iterations for the parser", "- tests pass"
)
async def test_goal_command_proposes_pending_rubric(self) -> None:
"""`/goal <objective>` should draft criteria for widget review."""
app = DeepAgentsApp(agent=MagicMock())
@@ -5631,6 +5843,10 @@ class TestGoalCommand:
assert "Goal:\nmake the app start faster" in rendered
assert "Status:\nactive" in rendered
assert "Criteria:\n- measure baseline\n- improve startup" in rendered
assert "Goal is active for this thread until completed" in rendered
assert (
"Follow-up prompts will continue working toward this goal." in rendered
)
assert "Commands:\n/goal clear\n/goal show" in rendered
assert "Goal status:" not in rendered
assert "Accepted criteria:" not in rendered
@@ -5796,7 +6012,7 @@ class TestGoalCommand:
# does not render the accepted criteria as an error body.
assert app._active_goal == "add refresh tokens"
assert any(
str(w._content) == "Goal accepted. Rubric set."
"Goal accepted. It will stay active for this thread" in str(w._content)
for w in app.query(AppMessage)
)
assert any(
@@ -6145,8 +6361,78 @@ class TestRubricCommand:
assert mock_execute.await_args is not None
assert mock_execute.await_args.kwargs["user_input"] == "keep going"
assert mock_execute.await_args.kwargs["blocked_goal_retry_context"] is None
assert any(
str(w._content) == "Continuing active goal: add refresh tokens"
for w in app.query(AppMessage)
)
assert app._goal_status == "active"
async def test_resume_notice_wording_when_goal_was_blocked(self) -> None:
"""Resuming a blocked goal announces a resume, not a plain continue.
`_reset_blocked_goal_for_user_turn` flips the status to active before the
notice check, so the wording must key off the reset signal rather than
the (already-mutated) status.
"""
app = DeepAgentsApp(agent=MagicMock())
async with app.run_test() as pilot:
await pilot.pause()
app._active_goal = "add refresh tokens"
app._active_rubric = "tests pass"
app._goal_status = "blocked"
app._goal_status_note = "waiting on provider credentials"
started = asyncio.Event()
def execute_stub(*_args: object, **_kwargs: object) -> SessionStats:
started.set()
return SessionStats()
with patch(
"deepagents_code.textual_adapter.execute_task_textual",
new=AsyncMock(side_effect=execute_stub),
):
await app._handle_user_message("Credentials are configured now")
await asyncio.wait_for(started.wait(), timeout=1)
assert app._goal_status == "active"
contents = [str(w._content) for w in app.query(AppMessage)]
assert "Resuming previously blocked goal: add refresh tokens" in contents
assert not any(c.startswith("Continuing active goal") for c in contents)
async def test_no_resume_notice_when_reset_persist_fails(self) -> None:
"""A rolled-back reset leaves the goal blocked, so no notice is shown."""
app = DeepAgentsApp(agent=MagicMock())
async with app.run_test() as pilot:
await pilot.pause()
app._agent = SimpleNamespace(
aupdate_state=AsyncMock(side_effect=RuntimeError("boom"))
)
app._lc_thread_id = "thread-1"
app._active_goal = "add refresh tokens"
app._active_rubric = "tests pass"
app._goal_status = "blocked"
app._goal_status_note = "waiting on provider credentials"
started = asyncio.Event()
def execute_stub(*_args: object, **_kwargs: object) -> SessionStats:
started.set()
return SessionStats()
with patch(
"deepagents_code.textual_adapter.execute_task_textual",
new=AsyncMock(side_effect=execute_stub),
):
await app._handle_user_message("Credentials are configured now")
await asyncio.wait_for(started.wait(), timeout=1)
assert app._goal_status == "blocked"
assert not any(
str(w._content).startswith(
("Resuming previously blocked goal", "Continuing active goal")
)
for w in app.query(AppMessage)
)
async def test_skill_invocation_resets_blocked_goal(self) -> None:
"""A skill send is the user acting on a blocked goal, so it resets it.
@@ -6475,6 +6761,252 @@ class TestRubricCommand:
assert "is empty" in errors
assert app._active_rubric is None
@pytest.mark.parametrize(
("raw", "expected", "err_substr"),
[
("10", 10, None),
("21", 21, None),
("1", 1, None),
("clear", None, None),
("default", None, None),
("0", None, "positive whole number"),
("-5", None, "positive whole number"),
("many", None, "clear' to reset"),
],
)
def test_parse_rubric_max_iterations(
self, raw: str, expected: int | None, err_substr: str | None
) -> None:
"""`/rubric max-iterations` accepts positive ints or clearing.
`err_substr` is `None` for accepted input and a substring of the
expected message when the input is rejected, so parse errors and
non-positive errors stay distinguishable to the user.
"""
value, error = _parse_rubric_max_iterations(raw)
assert value == expected
if err_substr is None:
assert error is None
else:
assert error is not None
assert err_substr in error
async def test_rubric_max_iterations_command_sets_before_owned_server_starts(
self,
) -> None:
"""The TUI command should stage the cap in owned server kwargs."""
app = DeepAgentsApp(agent=MagicMock())
async with app.run_test() as pilot:
await pilot.pause()
app._server_proc = None
app._server_kwargs = {}
await app._handle_command("/rubric max-iterations 21")
await pilot.pause()
assert app._rubric_max_iterations == 21
assert app._server_kwargs["rubric_max_iterations"] == 21
rendered = "\n".join(str(w._content) for w in app.query(AppMessage))
assert "Rubric max iterations set to 21" in rendered
async def test_rubric_max_iterations_command_rejects_non_positive(
self,
) -> None:
"""Non-positive max-iteration values should not mutate app state."""
app = DeepAgentsApp(agent=MagicMock())
async with app.run_test() as pilot:
await pilot.pause()
app._server_kwargs = {}
await app._handle_command("/rubric max-iterations 0")
await pilot.pause()
assert app._rubric_max_iterations is None
assert "rubric_max_iterations" not in app._server_kwargs
rendered = "\n".join(str(w._content) for w in app.query(ErrorMessage))
assert "positive whole number" in rendered
async def test_set_rubric_max_iterations_restarts_owned_server(self) -> None:
"""Changing the cap should update server env and respawn the graph."""
app = DeepAgentsApp(agent=MagicMock())
async with app.run_test() as pilot:
await pilot.pause()
app._server_proc = MagicMock()
app._server_kwargs = {}
with patch.object(
app,
"_respawn_server",
new_callable=AsyncMock,
return_value=True,
) as respawn:
await app._set_rubric_max_iterations(12)
await pilot.pause()
assert app._rubric_max_iterations == 12
assert app._server_kwargs["rubric_max_iterations"] == 12
app._server_proc.update_env.assert_called_once_with(
DEEPAGENTS_CODE_SERVER_RUBRIC_MAX_ITERATIONS="12",
)
app._server_proc.persist_env.assert_called_once_with(
DEEPAGENTS_CODE_SERVER_RUBRIC_MAX_ITERATIONS="12",
)
assert respawn.await_count == 1
rendered = "\n".join(str(w._content) for w in app.query(AppMessage))
assert "Rubric max iterations set to 12" in rendered
async def test_set_rubric_max_iterations_rolls_back_on_failed_respawn(self) -> None:
"""A failed respawn should restore the previous cap."""
app = DeepAgentsApp(agent=MagicMock())
async with app.run_test() as pilot:
await pilot.pause()
app._rubric_max_iterations = 10
app._server_kwargs = {"rubric_max_iterations": 10}
app._server_proc = MagicMock()
with patch.object(
app,
"_respawn_server",
new_callable=AsyncMock,
return_value=False,
):
await app._set_rubric_max_iterations(12)
assert app._rubric_max_iterations == 10
assert app._server_kwargs["rubric_max_iterations"] == 10
app._server_proc.persist_env.assert_not_called()
# The failed forward staging (12) must be re-staged back to the
# previous value (10) so a later restart cannot resurrect it.
assert app._server_proc.update_env.call_count == 2
assert app._server_proc.update_env.call_args_list[-1].kwargs == {
"DEEPAGENTS_CODE_SERVER_RUBRIC_MAX_ITERATIONS": "10",
}
async def test_set_rubric_max_iterations_defers_while_agent_running(self) -> None:
"""A cap change during a run is deferred, not applied immediately."""
app = DeepAgentsApp(agent=MagicMock())
async with app.run_test() as pilot:
await pilot.pause()
app._agent_running = True
app._server_kwargs = {}
with patch.object(app, "_defer_action") as defer:
await app._set_rubric_max_iterations(10)
defer.assert_called_once()
deferred = defer.call_args.args[0]
assert deferred.kind == "rubric_max_iterations_switch"
# The cap is untouched until the deferred action runs.
assert app._rubric_max_iterations is None
async def test_set_rubric_max_iterations_noop_when_value_matches(self) -> None:
"""Re-issuing the current cap should not respawn the server."""
app = DeepAgentsApp(agent=MagicMock())
async with app.run_test() as pilot:
await pilot.pause()
app._rubric_max_iterations = 10
app._server_kwargs = {"rubric_max_iterations": 10}
app._server_proc = MagicMock()
with patch.object(
app, "_respawn_server", new_callable=AsyncMock
) as respawn:
await app._set_rubric_max_iterations(10)
await pilot.pause()
respawn.assert_not_awaited()
app._server_proc.update_env.assert_not_called()
rendered = "\n".join(str(w._content) for w in app.query(AppMessage))
assert "already set to 10" in rendered
async def test_set_rubric_max_iterations_noop_when_already_default(self) -> None:
"""Clearing an already-default cap should not respawn the server."""
app = DeepAgentsApp(agent=MagicMock())
async with app.run_test() as pilot:
await pilot.pause()
app._server_kwargs = {}
app._server_proc = MagicMock()
with patch.object(
app, "_respawn_server", new_callable=AsyncMock
) as respawn:
await app._set_rubric_max_iterations(None)
await pilot.pause()
respawn.assert_not_awaited()
app._server_proc.update_env.assert_not_called()
rendered = "\n".join(str(w._content) for w in app.query(AppMessage))
assert "already use the SDK default" in rendered
async def test_rubric_max_iterations_command_clears_owned_server(self) -> None:
"""`/rubric max-iterations clear` resets the cap to the SDK default."""
app = DeepAgentsApp(agent=MagicMock())
async with app.run_test() as pilot:
await pilot.pause()
app._rubric_max_iterations = 10
app._server_kwargs = {"rubric_max_iterations": 10}
app._server_proc = MagicMock()
with patch.object(
app,
"_respawn_server",
new_callable=AsyncMock,
return_value=True,
):
await app._handle_command("/rubric max-iterations clear")
await pilot.pause()
assert app._rubric_max_iterations is None
assert app._server_kwargs["rubric_max_iterations"] is None
app._server_proc.update_env.assert_called_once_with(
DEEPAGENTS_CODE_SERVER_RUBRIC_MAX_ITERATIONS="",
)
rendered = "\n".join(str(w._content) for w in app.query(AppMessage))
assert "cleared; using the SDK default" in rendered
async def test_rubric_max_iterations_shown_in_state_and_usage(self) -> None:
"""A set cap should surface in `/rubric show` and bare `/rubric`."""
app = DeepAgentsApp(agent=MagicMock())
async with app.run_test() as pilot:
await pilot.pause()
app._rubric_max_iterations = 7
await app._handle_command("/rubric show")
await app._handle_command("/rubric")
await pilot.pause()
rendered = "\n".join(str(w._content) for w in app.query(AppMessage))
assert "Rubric max iterations: 7" in rendered
async def test_rubric_state_reports_sdk_default_when_cap_unset(self) -> None:
"""`/rubric show` labels an unset cap as the SDK default."""
app = DeepAgentsApp(agent=MagicMock())
async with app.run_test() as pilot:
await pilot.pause()
app._active_rubric = "tests pass"
await app._handle_command("/rubric show")
await pilot.pause()
rendered = "\n".join(str(w._content) for w in app.query(AppMessage))
assert "Rubric max iterations: SDK default" in rendered
async def test_set_rubric_max_iterations_rejects_without_owned_server(self) -> None:
"""External graph sessions cannot change construction-time rubric caps."""
app = DeepAgentsApp(agent=MagicMock())
async with app.run_test() as pilot:
await pilot.pause()
app._server_proc = None
app._server_kwargs = None
await app._set_rubric_max_iterations(10)
await pilot.pause()
assert app._rubric_max_iterations is None
rendered = "\n".join(str(w._content) for w in app.query(ErrorMessage))
assert "does not own a restartable server" in rendered
async def test_rubric_model_bare_opens_grader_model_selector(self) -> None:
"""Bare `/rubric model` should open the grader-model picker."""
app = DeepAgentsApp(agent=MagicMock())
@@ -6574,6 +7106,12 @@ class TestRubricCommand:
assert app._rubric_model == "anthropic:claude-sonnet-4-6"
assert app._server_kwargs["rubric_model"] == "anthropic:claude-sonnet-4-6"
# The failed forward staging must be re-staged back to the previous
# model so a later restart cannot resurrect the rolled-back value.
assert app._server_proc.update_env.call_count == 2
assert app._server_proc.update_env.call_args_list[-1].kwargs == {
"DEEPAGENTS_CODE_SERVER_RUBRIC_MODEL": "anthropic:claude-sonnet-4-6",
}
async def test_set_rubric_model_sets_before_owned_server_starts(self) -> None:
"""With owned server config, the grader model is staged and confirmed."""
@@ -13093,6 +13631,29 @@ class TestRestartServerForAgentSwap:
assert len(failures) == 1
assert failures[0].error is boom
async def test_failure_restages_previous_assistant_id(self) -> None:
"""A failed swap re-stages the previous assistant_id env override.
Otherwise the failed target stays staged in the server's one-shot env
overrides and a later, unrelated restart would silently resurrect it.
"""
app, server_proc = self._make_app()
server_proc.restart = AsyncMock(side_effect=RuntimeError("boom"))
posted: list[object] = []
with patch.object(app, "post_message", side_effect=posted.append):
async with app.run_test() as pilot:
await pilot.pause()
await app._restart_server_for_agent_swap("researcher")
# Forward staging (researcher) then rollback re-staging (coder).
assert server_proc.update_env.call_count == 2
assert server_proc.update_env.call_args_list[0].kwargs == {
"DEEPAGENTS_CODE_SERVER_ASSISTANT_ID": "researcher",
}
assert server_proc.update_env.call_args_list[-1].kwargs == {
"DEEPAGENTS_CODE_SERVER_ASSISTANT_ID": "coder",
}
class TestResolveResumeThread:
"""Resume-thread inference must not pollute the persisted default agent."""
libs/code/tests/unit_tests/test_command_registry.py
+24
View File
@@ -212,6 +212,30 @@ class TestMCPCommand:
assert "reconnect" in keywords
class TestGoalCommand:
"""Validate the `/goal` entry specifically.
`/goal` aliases the shared rubric grader controls (`model`,
`max-iterations`), so the entry must advertise them in the argument hint
and surface them via keyword search so goal-first users can discover
grader tuning without knowing about `/rubric`.
"""
def test_goal_argument_hint_advertises_grader_aliases(self) -> None:
goal_cmd = next(cmd for cmd in COMMANDS if cmd.name == "/goal")
assert "model" in goal_cmd.argument_hint
assert "max-iterations" in goal_cmd.argument_hint
def test_goal_hidden_keywords_cover_grader_search(self) -> None:
goal_cmd = next(cmd for cmd in COMMANDS if cmd.name == "/goal")
keywords = goal_cmd.hidden_keywords.split()
assert {"grader", "grading", "model", "iterations"} <= set(keywords)
def test_goal_hidden_keywords_retain_acceptance(self) -> None:
goal_cmd = next(cmd for cmd in COMMANDS if cmd.name == "/goal")
assert "acceptance" in goal_cmd.hidden_keywords.split()
class TestCopyCommand:
"""Validate the `/copy` entry specifically."""
libs/code/tests/unit_tests/test_rubric_cli.py
+15 -7
View File
@@ -198,7 +198,7 @@ class TestServerConfigRubric:
def test_defaults(self) -> None:
config = ServerConfig()
assert config.rubric_model is None
assert config.rubric_max_iterations == 3
assert config.rubric_max_iterations is None
def test_round_trip(self) -> None:
original = ServerConfig(
@@ -256,8 +256,8 @@ class TestHeaderIndicator:
assert "Goal" not in header.plain
def _render_event(data: dict) -> str:
state = StreamState()
def _render_event(data: dict, *, show_rubric_iterations: bool = False) -> str:
state = StreamState(show_rubric_iterations=show_rubric_iterations)
buf = io.StringIO()
console = Console(file=buf, width=200, highlight=False)
_process_rubric_event(data, state, console)
@@ -270,14 +270,22 @@ class TestProcessRubricEvent:
def test_start_event(self) -> None:
out = _render_event({"type": "rubric_evaluation_start", "iteration": 0})
assert "Grading against rubric" in out
assert "Checking acceptance criteria" in out
assert "iteration 1" not in out
def test_start_event_mentions_explicit_iteration(self) -> None:
out = _render_event(
{"type": "rubric_evaluation_start", "iteration": 0},
show_rubric_iterations=True,
)
assert "Checking acceptance criteria" in out
assert "iteration 1" in out
def test_satisfied(self) -> None:
out = _render_event(
{"type": "rubric_evaluation_end", "result": "satisfied", "criteria": []}
)
assert "Rubric satisfied" in out
assert "Acceptance criteria satisfied" in out
def test_needs_revision_with_criteria(self) -> None:
out = _render_event(
@@ -291,7 +299,7 @@ class TestProcessRubricEvent:
],
}
)
assert "needs revision" in out
assert "Changes need revision" in out
assert "tests missing" in out
assert "no coverage" in out
assert "style" not in out
@@ -300,7 +308,7 @@ class TestProcessRubricEvent:
out = _render_event(
{"type": "rubric_evaluation_end", "result": "max_iterations_reached"}
)
assert "max iterations reached" in out
assert "iteration limit reached" in out
def test_failed(self) -> None:
out = _render_event(
libs/code/tests/unit_tests/test_server.py
+103
View File
@@ -722,6 +722,109 @@ class TestServerProcess:
assert stop_thread_id is not None
assert stop_thread_id != loop_thread_id
async def test_persistent_env_applies_to_later_restarts(
self, tmp_path: Path
) -> None:
"""Persistent env overrides should apply without restaging."""
config_dir = tmp_path / "runtime"
config_dir.mkdir()
(config_dir / "langgraph.json").write_text("{}")
first_process = MagicMock()
first_process.pid = 1234
first_process.poll.return_value = None
second_process = MagicMock()
second_process.pid = 5678
second_process.poll.return_value = None
first_log_file = MagicMock()
first_log_file.name = str(tmp_path / "server-1.log")
second_log_file = MagicMock()
second_log_file.name = str(tmp_path / "server-2.log")
env_key = "DEEPAGENTS_CODE_SERVER_RUBRIC_MAX_ITERATIONS"
server = ServerProcess(config_dir=config_dir, owns_config_dir=False)
server.persist_env(**{env_key: "12"})
with (
patch("deepagents_code.server._find_free_port", return_value=12345),
patch("deepagents_code.server._port_in_use", return_value=False),
patch(
"deepagents_code.server.tempfile.NamedTemporaryFile",
side_effect=[first_log_file, second_log_file],
),
patch(
"deepagents_code.server.subprocess.Popen",
side_effect=[first_process, second_process],
) as popen,
patch(
"deepagents_code.server.wait_for_server_healthy",
new=AsyncMock(),
),
):
await server.start()
assert server._env_overrides == {}
await server.restart()
first_env = popen.call_args_list[0].kwargs["env"]
second_env = popen.call_args_list[1].kwargs["env"]
assert first_env[env_key] == "12"
assert second_env[env_key] == "12"
assert server._env_overrides == {}
async def test_one_shot_override_wins_over_persisted(self, tmp_path: Path) -> None:
"""A one-shot `update_env` must override a persisted default on restart.
Regression: persisting a value (via `persist_env`) and then staging a
different value (via `update_env`) before a restart must launch the
subprocess with the freshly staged value, not the stale persisted one.
"""
config_dir = tmp_path / "runtime"
config_dir.mkdir()
(config_dir / "langgraph.json").write_text("{}")
first_process = MagicMock()
first_process.pid = 1234
first_process.poll.return_value = None
second_process = MagicMock()
second_process.pid = 5678
second_process.poll.return_value = None
first_log_file = MagicMock()
first_log_file.name = str(tmp_path / "server-1.log")
second_log_file = MagicMock()
second_log_file.name = str(tmp_path / "server-2.log")
env_key = "DEEPAGENTS_CODE_SERVER_RUBRIC_MAX_ITERATIONS"
server = ServerProcess(config_dir=config_dir, owns_config_dir=False)
server.persist_env(**{env_key: "10"})
with (
patch("deepagents_code.server._find_free_port", return_value=12345),
patch("deepagents_code.server._port_in_use", return_value=False),
patch(
"deepagents_code.server.tempfile.NamedTemporaryFile",
side_effect=[first_log_file, second_log_file],
),
patch(
"deepagents_code.server.subprocess.Popen",
side_effect=[first_process, second_process],
) as popen,
patch(
"deepagents_code.server.wait_for_server_healthy",
new=AsyncMock(),
),
):
await server.start()
# Stage a different value for the next restart only.
server.update_env(**{env_key: "12"})
await server.restart()
# The restart that applies the staged override must use it, not the
# persisted default it temporarily supersedes.
second_env = popen.call_args_list[1].kwargs["env"]
assert second_env[env_key] == "12"
async def test_restart_rollback_on_failure(self, tmp_path: Path) -> None:
"""Env overrides are rolled back when restart fails."""
config_dir = tmp_path / "runtime"
libs/code/tests/unit_tests/test_server_config.py
+3 -3
View File
@@ -375,8 +375,8 @@ class TestServerConfigEdgeCases:
assert restored.enable_interpreter is False
def test_malformed_rubric_max_iterations_env_uses_default(self) -> None:
"""Bad optional rubric iteration config must not break graph startup."""
def test_malformed_rubric_max_iterations_env_uses_sdk_default(self) -> None:
"""Bad optional rubric iteration config must fall back to SDK default."""
with patch.dict(
os.environ,
{f"{SERVER_ENV_PREFIX}RUBRIC_MAX_ITERATIONS": "abc"},
@@ -384,7 +384,7 @@ class TestServerConfigEdgeCases:
):
restored = ServerConfig.from_env()
assert restored.rubric_max_iterations == 3
assert restored.rubric_max_iterations is None
def test_rubric_max_iterations_env_parses_int(self) -> None:
"""Valid rubric iteration config survives the server env boundary."""
libs/code/tests/unit_tests/test_server_graph.py
+1 -1
View File
@@ -206,7 +206,7 @@ class TestServerGraph:
enable_shell=True,
enable_interpreter=False,
rubric_model=None,
rubric_max_iterations=3,
rubric_max_iterations=None,
mcp_server_info=mcp_server_info,
cwd=None,
project_context=None,
libs/code/tests/unit_tests/test_textual_adapter.py
+27 -15
View File
@@ -1046,13 +1046,26 @@ class TestFormatRubricEvent:
):
yield
def test_start_event_mentions_iteration(self) -> None:
"""Start events should surface visible grading state."""
def test_start_event_omits_iteration_by_default(self) -> None:
"""Start events should avoid noisy iteration numbers by default."""
assert (
_format_rubric_event(
{"type": "rubric_evaluation_start", "iteration": 1},
)
== "⏳ Grading against rubric (iteration 2)…"
== "⏳ Checking acceptance criteria…"
)
def test_start_event_mentions_explicit_iteration(self) -> None:
"""Explicit iteration display should surface the 1-based count."""
assert (
_format_rubric_event(
{
"type": "rubric_evaluation_start",
"iteration": 1,
"show_iteration": True,
},
)
== "⏳ Checking acceptance criteria (iteration 2)…"
)
def test_needs_revision_includes_failed_criteria(self) -> None:
@@ -1069,7 +1082,7 @@ class TestFormatRubricEvent:
],
},
)
== "↻ Rubric needs revision: missing coverage\n ✗ tests pass — not run"
== "↻ Changes need revision: missing coverage\n ✗ tests pass — not run"
)
def test_satisfied_event(self) -> None:
@@ -1078,7 +1091,7 @@ class TestFormatRubricEvent:
_format_rubric_event(
{"type": "rubric_evaluation_end", "result": "satisfied"},
)
== "✓ Rubric satisfied"
== "✓ Acceptance criteria satisfied"
)
def test_start_event_without_int_iteration_omits_number(self) -> None:
@@ -1087,7 +1100,7 @@ class TestFormatRubricEvent:
_format_rubric_event(
{"type": "rubric_evaluation_start", "iteration": None},
)
== "⏳ Grading against rubric…"
== "⏳ Checking acceptance criteria…"
)
def test_max_iterations_reached_event(self) -> None:
@@ -1096,7 +1109,7 @@ class TestFormatRubricEvent:
_format_rubric_event(
{"type": "rubric_evaluation_end", "result": "max_iterations_reached"},
)
== "⚠ Rubric not satisfied (max iterations reached)"
== "⚠ Acceptance criteria not satisfied (iteration limit reached)"
)
def test_grader_failure_results_render_warning(self) -> None:
@@ -1159,14 +1172,12 @@ class TestFormatRubricEvent:
failed = _format_rubric_event(
{"type": "rubric_evaluation_end", "result": "failed"},
)
assert (
start == f"{ASCII_GLYPHS.hourglass} Grading against rubric (iteration 1)..."
)
assert start == f"{ASCII_GLYPHS.hourglass} Checking acceptance criteria..."
assert revision == (
f"{ASCII_GLYPHS.retry} Rubric needs revision\n"
f"{ASCII_GLYPHS.retry} Changes need revision\n"
f" {ASCII_GLYPHS.error} tests pass — not run"
)
assert satisfied == f"{ASCII_GLYPHS.checkmark} Rubric satisfied"
assert satisfied == f"{ASCII_GLYPHS.checkmark} Acceptance criteria satisfied"
assert failed == f"{ASCII_GLYPHS.warning} Rubric grader failed"
@@ -2649,10 +2660,10 @@ class TestExecuteTaskTextualRubricRevisionStreaming:
app_text = [str(widget._content) for widget in app_messages]
assert app_text == [
(
f"{UNICODE_GLYPHS.hourglass} Grading against rubric (iteration 1)"
f"{UNICODE_GLYPHS.hourglass} Checking acceptance criteria"
f"{UNICODE_GLYPHS.ellipsis}"
),
f"{UNICODE_GLYPHS.retry} Rubric needs revision: say yellow",
f"{UNICODE_GLYPHS.retry} Changes need revision: say yellow",
]
@@ -4001,7 +4012,8 @@ class TestExecuteTaskTextualRubricEvents:
rubric_msgs = [
m
for m in mounted
if isinstance(m, AppMessage) and "Rubric satisfied" in str(m._content)
if isinstance(m, AppMessage)
and "Acceptance criteria satisfied" in str(m._content)
]
assert len(rubric_msgs) == 1
libs/deepagents/deepagents/middleware/rubric.py
+5 -9
View File
@@ -108,9 +108,6 @@ when a single tool call returns a large blob (e.g. a file dump or test
log).
"""
_MAX_ITERATIONS_HARD_CAP = 20
"""Hard upper bound for `max_iterations`."""
_PAYLOAD_CLOSER_RE = re.compile(r"</(rubric|transcript)", re.IGNORECASE)
"""Matches a closing `rubric` or `transcript` tag in payload content."""
@@ -329,8 +326,8 @@ class RubricMiddleware(AgentMiddleware[RubricState, ContextT, ResponseT]):
`GraderResponse`.
With none, the grader reasons from the transcript alone.
max_iterations: Hard cap on grader iterations per rubric attempt;
hard-capped at 20.
max_iterations: Maximum grader iterations per rubric attempt; must be a
positive integer.
When the cap is reached without a `satisfied` verdict, the agent
terminates with status `'max_iterations_reached'` (see the
@@ -342,8 +339,7 @@ class RubricMiddleware(AgentMiddleware[RubricState, ContextT, ResponseT]):
suppressed; do not use this callback to enforce control flow.
Raises:
ValueError: If `max_iterations` is outside `[1, 20]`, or if `model`
is falsy.
ValueError: If `max_iterations` is less than 1, or if `model` is falsy.
TypeError: If `max_iterations` is not an `int`.
"""
@@ -364,8 +360,8 @@ class RubricMiddleware(AgentMiddleware[RubricState, ContextT, ResponseT]):
if not isinstance(max_iterations, int) or isinstance(max_iterations, bool):
msg = f"RubricMiddleware: `max_iterations` must be an int, got {type(max_iterations).__name__}."
raise TypeError(msg)
if not 1 <= max_iterations <= _MAX_ITERATIONS_HARD_CAP:
msg = f"RubricMiddleware: `max_iterations` must be in [1, {_MAX_ITERATIONS_HARD_CAP}], got {max_iterations}."
if max_iterations < 1:
msg = f"RubricMiddleware: `max_iterations` must be positive, got {max_iterations}."
raise ValueError(msg)
self.max_iterations = max_iterations
libs/deepagents/tests/unit_tests/middleware/test_rubric_middleware.py
+8 -3
View File
@@ -121,9 +121,14 @@ class TestConstruction:
with pytest.raises(ValueError, match="max_iterations"):
RubricMiddleware(model=_STUB_MODEL, max_iterations=0)
def test_max_iterations_upper_bound(self) -> None:
with pytest.raises(ValueError, match="max_iterations"):
RubricMiddleware(model=_STUB_MODEL, max_iterations=21)
def test_max_iterations_lower_bound_accepted(self) -> None:
# 1 is the smallest accepted value; the guard is `max_iterations < 1`.
mw = RubricMiddleware(model=_STUB_MODEL, max_iterations=1)
assert mw.max_iterations == 1
def test_max_iterations_above_previous_hard_cap_allowed(self) -> None:
mw = RubricMiddleware(model=_STUB_MODEL, max_iterations=21)
assert mw.max_iterations == 21
def test_max_iterations_bool_rejected(self) -> None:
# bool is a subclass of int; reject explicitly so True/False can't