mirror of
https://github.com/codewhale-hq/Codewhale.git
synced 2026-11-08 17:10:50 -10:00
Merge pull request #6717 from Hmbown/fix/fleet-role-route-failures
fix(fleet): run refused saved-profile pins on the parent route; name the real route source
This commit is contained in:
8 files changed
+1104
-47
No files matched your search
@@ -2738,6 +2738,11 @@ pub struct SubAgentRuntime {
|
||||
/// replacements`); every other route source, including exact Fleet
|
||||
/// bindings and task-level models, keeps it empty and stays exact.
|
||||
pub route_replacements: Vec<SubagentModelOverride>,
|
||||
/// Where this child's route came from, in operator words, and how to
|
||||
/// change it; for a saved-profile pin, also the parent route the child
|
||||
/// may take when that pin refuses authorization before any work. Set by
|
||||
/// the spawn that bound the route; descendants earn their own.
|
||||
pub(crate) route_origin: Option<Arc<SpawnRouteOrigin>>,
|
||||
pub context: ToolContext,
|
||||
pub allow_shell: bool,
|
||||
/// When true, Suggest-level file writes auto-accept for write-capable roles
|
||||
@@ -2903,6 +2908,7 @@ impl SubAgentRuntime {
|
||||
reasoning_effort_auto: false,
|
||||
role_models: HashMap::new(),
|
||||
route_replacements: Vec::new(),
|
||||
route_origin: None,
|
||||
context,
|
||||
allow_shell,
|
||||
accept_edits: false,
|
||||
@@ -3227,6 +3233,7 @@ impl SubAgentRuntime {
|
||||
role_models: self.role_models.clone(),
|
||||
// A descendant earns replacement authority only from its own pin.
|
||||
route_replacements: Vec::new(),
|
||||
route_origin: None,
|
||||
context: child_context,
|
||||
allow_shell: self.allow_shell,
|
||||
accept_edits: self.accept_edits,
|
||||
@@ -3757,6 +3764,15 @@ pub struct SubAgentManager {
|
||||
/// reported to the parent model as `needs_person`. A reported id keeps a
|
||||
/// later wait blocking instead of re-waking on the same request.
|
||||
reported_pending: HashSet<String>,
|
||||
/// Saved-profile routes (`provider/model`) whose first request this
|
||||
/// session refused authorization, with the short reason. In memory only:
|
||||
/// later spawns of the same pin go straight to the parent route instead
|
||||
/// of each failing first; a new session re-tries the pin.
|
||||
refused_routes: HashMap<String, String>,
|
||||
/// Children whose requests moved to another route before any work, by
|
||||
/// agent id, so the task's failure text blames the route that failed.
|
||||
/// Taken when the task settles.
|
||||
rerouted_requests: HashMap<String, ReroutedRequest>,
|
||||
}
|
||||
|
||||
/// One child request waiting on a person (approvals C2). The store is the
|
||||
@@ -4242,6 +4258,8 @@ impl SubAgentManager {
|
||||
worktree_cleanups: Arc::default(),
|
||||
child_approvals: HashMap::new(),
|
||||
reported_pending: HashSet::new(),
|
||||
refused_routes: HashMap::new(),
|
||||
rerouted_requests: HashMap::new(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6024,21 +6042,38 @@ impl SubAgentManager {
|
||||
}
|
||||
|
||||
/// Persist a pre-work route replacement in the worker's existing route
|
||||
/// receipt: the effective provider/model, `role.replacement` as the
|
||||
/// source, and a note naming the original route, reason and attempts.
|
||||
/// receipt: the effective provider/model, the replacement source
|
||||
/// (`role.replacement` for an approved route, `session.fallback` for a
|
||||
/// refused saved-profile pin), and a note naming the original route,
|
||||
/// reason and outcome.
|
||||
///
|
||||
/// `next` is the runtime the child's requests now use; its route origin
|
||||
/// is kept so a failure on that route names it.
|
||||
fn record_route_replacement(
|
||||
&mut self,
|
||||
worker_id: &str,
|
||||
provider_id: String,
|
||||
model_id: String,
|
||||
next: &SubAgentRuntime,
|
||||
source: SpawnRouteSource,
|
||||
note: String,
|
||||
) {
|
||||
let provider_id = runtime_provider_id(next);
|
||||
let model_id = next.model.clone();
|
||||
if let Some(origin) = next.route_origin.clone() {
|
||||
self.rerouted_requests.insert(
|
||||
worker_id.to_string(),
|
||||
ReroutedRequest {
|
||||
model: model_id.clone(),
|
||||
provider: next.client.api_provider(),
|
||||
origin,
|
||||
},
|
||||
);
|
||||
}
|
||||
if let Some(record) = self.worker_records.get_mut(worker_id) {
|
||||
record.spec.model.clone_from(&model_id);
|
||||
if let Some(route) = record.spec.child_route.as_mut() {
|
||||
route.provider_id = provider_id;
|
||||
route.model_id = model_id;
|
||||
route.route_source = SpawnRouteSource::RoleReplacement.as_str().to_string();
|
||||
route.route_source = source.as_str().to_string();
|
||||
route.fallback_note = Some(note);
|
||||
}
|
||||
record.updated_at_ms = epoch_millis_now();
|
||||
@@ -6046,6 +6081,17 @@ impl SubAgentManager {
|
||||
}
|
||||
}
|
||||
|
||||
/// Remember a saved-profile route that refused authorization this session.
|
||||
fn record_refused_route(&mut self, route: &str, reason: &str) {
|
||||
self.refused_routes
|
||||
.insert(route.to_string(), reason.chars().take(160).collect());
|
||||
}
|
||||
|
||||
/// Why this session already knows `route` refuses authorization, if it does.
|
||||
fn refused_route(&self, route: &str) -> Option<String> {
|
||||
self.refused_routes.get(route).cloned()
|
||||
}
|
||||
|
||||
fn mark_worker_unreported_usage(&mut self, worker_id: &str) {
|
||||
if let Some(record) = self.worker_records.get_mut(worker_id) {
|
||||
record.has_unreported_usage = true;
|
||||
@@ -7005,6 +7051,18 @@ impl SubAgentManager {
|
||||
)
|
||||
})?;
|
||||
}
|
||||
// Resume skips the spawn's route binding, so rebuild where the saved
|
||||
// route came from; a failure then names that source, not a guess.
|
||||
runtime.route_origin = child_route
|
||||
.as_ref()
|
||||
.and_then(route_origin_from_receipt)
|
||||
.map(Arc::new);
|
||||
// A child moved before any work (session fallback or approved
|
||||
// replacement) resumes on the route its receipt names; the launch
|
||||
// model would pair the old pin's model with the new provider.
|
||||
let model = child_route
|
||||
.as_ref()
|
||||
.map_or(model, |route| route.model_id.clone());
|
||||
let saved_manifest = self
|
||||
.worker_records
|
||||
.get(&agent_id)
|
||||
@@ -12647,17 +12705,35 @@ async fn run_subagent_task_inner(mut task: SubAgentTask) {
|
||||
};
|
||||
|
||||
let agent_id = task.agent_id.clone();
|
||||
// A request that moved before any work failed (if it failed) on its new
|
||||
// route: blame that route, not the one it left.
|
||||
let rerouted = task
|
||||
.manager_handle
|
||||
.write()
|
||||
.await
|
||||
.rerouted_requests
|
||||
.remove(&agent_id);
|
||||
let failure_error = result.as_ref().err().map(|err| {
|
||||
crate::logging::warn(format!(
|
||||
"sub-agent {} model request failed: {err:#}",
|
||||
task.agent_id
|
||||
));
|
||||
annotate_child_model_error(
|
||||
&subagent_failure_message(err),
|
||||
&task.runtime.model,
|
||||
task.runtime.client.api_provider(),
|
||||
&task.runtime.worker_profile.model,
|
||||
)
|
||||
match rerouted.as_ref() {
|
||||
Some(rerouted) => annotate_child_model_error_with_origin(
|
||||
&subagent_failure_message(err),
|
||||
&rerouted.model,
|
||||
rerouted.provider,
|
||||
&ModelRoute::Inherit,
|
||||
Some(&rerouted.origin),
|
||||
),
|
||||
None => annotate_child_model_error_with_origin(
|
||||
&subagent_failure_message(err),
|
||||
&task.runtime.model,
|
||||
task.runtime.client.api_provider(),
|
||||
&task.runtime.worker_profile.model,
|
||||
task.runtime.route_origin.as_deref(),
|
||||
),
|
||||
}
|
||||
});
|
||||
|
||||
// #5529: a wall-time task error must still name the work the child left
|
||||
@@ -14670,6 +14746,63 @@ async fn run_subagent(
|
||||
// request died with zero completed work —
|
||||
// fail plainly, exactly as before.
|
||||
if steps <= 1 {
|
||||
// A saved-profile pin that refuses
|
||||
// authorization before any work runs on
|
||||
// the parent route, visibly, once.
|
||||
if route_override.is_none()
|
||||
&& let Some(origin) = runtime.route_origin.as_deref()
|
||||
&& let Some(parent) = origin.parent.as_ref()
|
||||
&& let Some(why) = pin_refusal_reason(&err)
|
||||
{
|
||||
let detail: String = route_runtime
|
||||
.client
|
||||
.redact_model_bound_text(&format!("{err}"))
|
||||
.chars()
|
||||
.take(160)
|
||||
.collect();
|
||||
let note: String = format!(
|
||||
"{}, which failed authorization ({why}: {detail}); ran on {} instead",
|
||||
origin.source, parent.label
|
||||
)
|
||||
.chars()
|
||||
.take(480)
|
||||
.collect();
|
||||
crate::logging::warn(format!(
|
||||
"sub-agent {agent_id} route fallback: {note}"
|
||||
));
|
||||
let mut next = runtime.clone();
|
||||
parent.install(&mut next);
|
||||
next.route_origin = Some(Arc::new(spawn_route_origin(
|
||||
SpawnRouteSource::SessionFallback,
|
||||
runtime.worker_profile.role.as_str(),
|
||||
None,
|
||||
parent.label.clone(),
|
||||
Some(¬e),
|
||||
)));
|
||||
{
|
||||
let mut manager = runtime.manager.write().await;
|
||||
manager.record_refused_route(
|
||||
&origin.route,
|
||||
&format!("{why}: {detail}"),
|
||||
);
|
||||
manager.record_route_replacement(
|
||||
&agent_id,
|
||||
&next,
|
||||
SpawnRouteSource::SessionFallback,
|
||||
note.clone(),
|
||||
);
|
||||
}
|
||||
record_agent_progress(
|
||||
runtime,
|
||||
&agent_id,
|
||||
AgentProgressEventMeta::new(AgentWorkerStatus::Running)
|
||||
.with_step(0),
|
||||
format!("Route fell back: {note}"),
|
||||
);
|
||||
staged_route_override = Some(next);
|
||||
steps = 0;
|
||||
continue 'subagent;
|
||||
}
|
||||
// Nothing of this run has executed yet,
|
||||
// so an operator-approved route may take
|
||||
// the same request; never after work.
|
||||
@@ -14705,21 +14838,19 @@ async fn run_subagent(
|
||||
note.push_str(&skipped_replacements.join("; "));
|
||||
}
|
||||
let note: String = note.chars().take(480).collect();
|
||||
let provider_id = next
|
||||
.api_config
|
||||
.as_ref()
|
||||
.map(|config| {
|
||||
config.provider_identity_for(
|
||||
next.client.api_provider(),
|
||||
)
|
||||
})
|
||||
.unwrap_or_else(|| {
|
||||
next.client.api_provider().as_str().to_string()
|
||||
});
|
||||
let mut next = next;
|
||||
next.route_origin =
|
||||
Some(Arc::new(spawn_route_origin(
|
||||
SpawnRouteSource::RoleReplacement,
|
||||
runtime.worker_profile.role.as_str(),
|
||||
None,
|
||||
runtime_route_label(&next),
|
||||
Some(¬e),
|
||||
)));
|
||||
runtime.manager.write().await.record_route_replacement(
|
||||
&agent_id,
|
||||
provider_id,
|
||||
next.model.clone(),
|
||||
&next,
|
||||
SpawnRouteSource::RoleReplacement,
|
||||
note.clone(),
|
||||
);
|
||||
record_agent_progress(
|
||||
@@ -16690,6 +16821,173 @@ impl SpawnRouteSource {
|
||||
Self::RoleReplacement => "role.replacement",
|
||||
}
|
||||
}
|
||||
|
||||
fn parse(value: &str) -> Option<Self> {
|
||||
[
|
||||
Self::AgentProfileModel,
|
||||
Self::RolePin,
|
||||
Self::TaskModel,
|
||||
Self::TaskModelStrength,
|
||||
Self::RoleDefault,
|
||||
Self::RunModel,
|
||||
Self::SessionFallback,
|
||||
Self::RoleReplacement,
|
||||
]
|
||||
.into_iter()
|
||||
.find(|source| source.as_str() == value)
|
||||
}
|
||||
}
|
||||
|
||||
/// `provider/model` of the route a runtime would send its next request on,
|
||||
/// named the way route receipts name it.
|
||||
fn runtime_route_label(runtime: &SubAgentRuntime) -> String {
|
||||
format!("{}/{}", runtime_provider_id(runtime), runtime.model)
|
||||
}
|
||||
|
||||
fn runtime_provider_id(runtime: &SubAgentRuntime) -> String {
|
||||
runtime.api_config.as_deref().map_or_else(
|
||||
|| runtime.client.api_provider().as_str().to_string(),
|
||||
|config| config.provider_identity_for(runtime.client.api_provider()),
|
||||
)
|
||||
}
|
||||
|
||||
/// The parent's route, captured before a spawn binds its own, so a saved
|
||||
/// profile pin that refuses authorization can run where the conversation
|
||||
/// already runs. No new provider ever receives the task this way.
|
||||
#[derive(Clone)]
|
||||
pub(crate) struct ParentRoute {
|
||||
client: CodewhaleClient,
|
||||
api_config: Option<Arc<crate::config::Config>>,
|
||||
model: String,
|
||||
reasoning_effort: Option<String>,
|
||||
reasoning_effort_auto: bool,
|
||||
label: String,
|
||||
}
|
||||
|
||||
impl ParentRoute {
|
||||
fn capture(runtime: &SubAgentRuntime) -> Self {
|
||||
Self {
|
||||
client: runtime.client.clone(),
|
||||
api_config: runtime.api_config.clone(),
|
||||
model: runtime.model.clone(),
|
||||
reasoning_effort: runtime.reasoning_effort.clone(),
|
||||
reasoning_effort_auto: runtime.reasoning_effort_auto,
|
||||
label: runtime_route_label(runtime),
|
||||
}
|
||||
}
|
||||
|
||||
/// Move only the request route; role, grants, tools and budgets stay.
|
||||
fn install(&self, runtime: &mut SubAgentRuntime) {
|
||||
runtime.client = self.client.clone();
|
||||
runtime.api_config.clone_from(&self.api_config);
|
||||
runtime.model.clone_from(&self.model);
|
||||
runtime.reasoning_effort.clone_from(&self.reasoning_effort);
|
||||
runtime.reasoning_effort_auto = self.reasoning_effort_auto;
|
||||
}
|
||||
}
|
||||
|
||||
/// Where a child's route came from, in operator words, and how to change it.
|
||||
/// Failure text and fallback notes read it, so an error names the real
|
||||
/// source (a saved profile, a config pin, a task argument) instead of
|
||||
/// guessing from the resolved model.
|
||||
pub(crate) struct SpawnRouteOrigin {
|
||||
/// e.g. `saved agent profile "reviewer" pins xai/grok-4.6`.
|
||||
source: String,
|
||||
/// The exact way to change it.
|
||||
change: String,
|
||||
/// `provider/model` the spawn bound: the refused-route cache key.
|
||||
route: String,
|
||||
/// Present only when a refused first request may run on the parent route.
|
||||
parent: Option<ParentRoute>,
|
||||
}
|
||||
|
||||
fn saved_profile_change_hint(profile_id: &str, profile_file: Option<&std::path::Path>) -> String {
|
||||
let editor = format!("change it in /fleet members (Enter on \"{profile_id}\")");
|
||||
match profile_file.filter(|path| !path.as_os_str().is_empty()) {
|
||||
Some(path) => format!("{editor} or edit `provider`/`model` in {}", path.display()),
|
||||
None => editor,
|
||||
}
|
||||
}
|
||||
|
||||
/// The saved profile a route came from: its id and, when known, its file.
|
||||
type RouteProfile<'a> = (&'a str, Option<&'a std::path::Path>);
|
||||
|
||||
fn spawn_route_origin(
|
||||
source: SpawnRouteSource,
|
||||
role: &str,
|
||||
profile: Option<RouteProfile<'_>>,
|
||||
route: String,
|
||||
fallback_note: Option<&str>,
|
||||
) -> SpawnRouteOrigin {
|
||||
let (source, change) = match (source, profile) {
|
||||
(SpawnRouteSource::AgentProfileModel, Some((id, file))) => (
|
||||
format!("saved agent profile \"{id}\" pins {route}"),
|
||||
saved_profile_change_hint(id, file),
|
||||
),
|
||||
(SpawnRouteSource::RolePin, _) => (
|
||||
format!("config.toml role pin for \"{role}\" routes {route}"),
|
||||
format!(
|
||||
"edit `model` under [subagents.roles.{role}] in config.toml, or declare approved `replacements` there"
|
||||
),
|
||||
),
|
||||
(SpawnRouteSource::TaskModel, _) => (
|
||||
format!("explicit child model override (the task's `model` argument) routes {route}"),
|
||||
"omit `model` or choose a route listed by agent action=roster".to_string(),
|
||||
),
|
||||
(SpawnRouteSource::TaskModelStrength, _) => (
|
||||
format!("the task's `model_strength` routes {route}"),
|
||||
"omit `model_strength` to inherit the parent route".to_string(),
|
||||
),
|
||||
(SpawnRouteSource::RoleDefault, _) => (
|
||||
format!("the configured default model for role \"{role}\" routes {route}"),
|
||||
format!("edit the \"{role}\" default under [subagents] in config.toml"),
|
||||
),
|
||||
// The route the child moved to after its own was refused or
|
||||
// unusable: blame this route, and still point at the original.
|
||||
(SpawnRouteSource::SessionFallback | SpawnRouteSource::RoleReplacement, _) => (
|
||||
match fallback_note {
|
||||
Some(note) => format!("{route}, taken because {note}"),
|
||||
None => format!("fallback route {route}"),
|
||||
},
|
||||
format!(
|
||||
"{route} failed too; check that provider's sign-in, credits, or access, and fix the original route the note names"
|
||||
),
|
||||
),
|
||||
_ => (
|
||||
format!("inherited from the parent route {route}"),
|
||||
"the parent route itself was refused; check that provider's sign-in, credits, or access"
|
||||
.to_string(),
|
||||
),
|
||||
};
|
||||
SpawnRouteOrigin {
|
||||
source,
|
||||
change,
|
||||
route,
|
||||
parent: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// A resumed child's route origin, rebuilt from its saved route receipt: the
|
||||
/// resume does not re-run the spawn's route binding, but its failures must
|
||||
/// still name the real source instead of guessing from the resolved model.
|
||||
fn route_origin_from_receipt(receipt: &ChildRouteReceipt) -> Option<SpawnRouteOrigin> {
|
||||
let source = SpawnRouteSource::parse(&receipt.route_source)?;
|
||||
Some(spawn_route_origin(
|
||||
source,
|
||||
&receipt.canonical_role,
|
||||
receipt.resolved_profile_id.as_deref().map(|id| (id, None)),
|
||||
format!("{}/{}", receipt.provider_id, receipt.model_id),
|
||||
receipt.fallback_note.as_deref(),
|
||||
))
|
||||
}
|
||||
|
||||
/// The route a child's requests moved to before any work (a refused
|
||||
/// saved-profile pin on the parent route, or an approved replacement), so a
|
||||
/// failure there is reported against that route, not the one it left.
|
||||
pub(crate) struct ReroutedRequest {
|
||||
model: String,
|
||||
provider: crate::config::ApiProvider,
|
||||
origin: Arc<SpawnRouteOrigin>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
@@ -16877,6 +17175,8 @@ async fn bind_spawn_model_route(
|
||||
// Replacement authority belongs to one role pin, never to a descendant
|
||||
// that inherited this runtime or chose its route another way.
|
||||
runtime.route_replacements.clear();
|
||||
runtime.route_origin = None;
|
||||
let parent_route = ParentRoute::capture(runtime);
|
||||
match bind_profile_provider(runtime, member)? {
|
||||
MemberProviderBind::Bound => {}
|
||||
MemberProviderBind::Unavailable {
|
||||
@@ -17008,6 +17308,62 @@ async fn bind_spawn_model_route(
|
||||
} else {
|
||||
selection.source
|
||||
};
|
||||
let bound = runtime_route_label(runtime);
|
||||
let role = request
|
||||
.assignment
|
||||
.role
|
||||
.as_deref()
|
||||
.unwrap_or_else(|| request.agent_type.as_str());
|
||||
let mut origin = spawn_route_origin(
|
||||
source,
|
||||
role,
|
||||
member.map(|member| (member.id.as_str(), Some(member.source.as_path()))),
|
||||
bound.clone(),
|
||||
fallback_note.as_deref(),
|
||||
);
|
||||
if source == SpawnRouteSource::AgentProfileModel && bound != parent_route.label {
|
||||
// A saved-profile pin that refuses authorization before any work may
|
||||
// run on the parent route instead (the #5529 mode 2 rule, extended
|
||||
// to the first request). Exact-bound spawns and capability-bound
|
||||
// members stay exact, and the failure then says so.
|
||||
let known_refusal = runtime.manager.read().await.refused_route(&bound);
|
||||
if !allow_provider_fallback {
|
||||
origin
|
||||
.change
|
||||
.push_str("; this exact Fleet binding forbids falling back to the parent route");
|
||||
} else if member.is_some_and(|member| !member.requires.is_empty()) {
|
||||
origin
|
||||
.change
|
||||
.push_str("; a capability-bound member is never moved to another route");
|
||||
} else if let Some(reason) = known_refusal {
|
||||
// This session already saw the pin refuse: go straight to the
|
||||
// parent route instead of failing first again.
|
||||
let note: String = format!(
|
||||
"{}, which failed authorization earlier this session ({reason}); ran on {} instead",
|
||||
origin.source, parent_route.label
|
||||
)
|
||||
.chars()
|
||||
.take(480)
|
||||
.collect();
|
||||
crate::logging::warn(format!("sub-agent route fallback: {note}"));
|
||||
parent_route.install(runtime);
|
||||
runtime.route_origin = Some(Arc::new(spawn_route_origin(
|
||||
SpawnRouteSource::SessionFallback,
|
||||
role,
|
||||
None,
|
||||
parent_route.label.clone(),
|
||||
Some(¬e),
|
||||
)));
|
||||
return Ok((
|
||||
ModelRoute::Inherit,
|
||||
SpawnRouteSource::SessionFallback,
|
||||
Some(note),
|
||||
));
|
||||
} else {
|
||||
origin.parent = Some(parent_route);
|
||||
}
|
||||
}
|
||||
runtime.route_origin = Some(Arc::new(origin));
|
||||
Ok((route.model_route, source, fallback_note))
|
||||
}
|
||||
|
||||
@@ -17132,7 +17488,7 @@ async fn resolved_spawn_roster_entry(
|
||||
Err(error) => Err(error),
|
||||
};
|
||||
match resolved {
|
||||
Ok((_, source, _)) => {
|
||||
Ok((_, source, fallback_note)) => {
|
||||
let envelope = child
|
||||
.client
|
||||
.effective_route_envelope(&child.model, chrono::Utc::now())
|
||||
@@ -17174,6 +17530,11 @@ async fn resolved_spawn_roster_entry(
|
||||
"unpriced_reason": audit.unpriced_reason.map(|reason| reason.label()),
|
||||
"reachability": "unverified",
|
||||
});
|
||||
if let Some(note) = fallback_note {
|
||||
// A pin this session saw refuse authorization previews the
|
||||
// parent route it will actually run on, and says why.
|
||||
entry["route"]["fallback_note"] = json!(note);
|
||||
}
|
||||
}
|
||||
Err(error) => {
|
||||
entry["route"] = Value::Null;
|
||||
@@ -17624,6 +17985,19 @@ fn route_replacement_reason(error: &anyhow::Error) -> Option<&'static str> {
|
||||
}
|
||||
}
|
||||
|
||||
/// Why a saved-profile pin's first-request refusal may move the child to the
|
||||
/// parent route, or `None`. Only typed authorization and credit refusals
|
||||
/// qualify: transient failures keep the normal retry path, and content or
|
||||
/// model refusals are not a reason to change routes.
|
||||
fn pin_refusal_reason(error: &anyhow::Error) -> Option<&'static str> {
|
||||
match error.downcast_ref::<LlmError>()? {
|
||||
LlmError::QuotaExhausted(_)
|
||||
| LlmError::AuthenticationError(_)
|
||||
| LlmError::AuthorizationError(_) => route_replacement_reason(error),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Bind `base` to one approved replacement route. Only the request route
|
||||
/// changes: role, grants, tool scope, budgets and workspace stay `base`'s.
|
||||
fn replacement_route_runtime(
|
||||
@@ -20131,20 +20505,47 @@ fn route_source_label(route: &ModelRoute) -> String {
|
||||
/// route produced the failing model and how to recover (#2653, #4049) without
|
||||
/// re-classifying the underlying error. Errors unrelated to model availability
|
||||
/// pass through unchanged.
|
||||
#[cfg(test)]
|
||||
fn annotate_child_model_error(
|
||||
err: &str,
|
||||
model: &str,
|
||||
provider: crate::config::ApiProvider,
|
||||
route: &ModelRoute,
|
||||
) -> String {
|
||||
let hint = || {
|
||||
format!(
|
||||
annotate_child_model_error_with_origin(err, model, provider, route, None)
|
||||
}
|
||||
|
||||
/// [`annotate_child_model_error`] naming the route's real source when the
|
||||
/// spawn recorded one: a saved profile, a config pin, or a task argument,
|
||||
/// with the exact way to change it. Only a task `model` argument is ever
|
||||
/// called an explicit child model override.
|
||||
fn annotate_child_model_error_with_origin(
|
||||
err: &str,
|
||||
model: &str,
|
||||
provider: crate::config::ApiProvider,
|
||||
route: &ModelRoute,
|
||||
origin: Option<&SpawnRouteOrigin>,
|
||||
) -> String {
|
||||
let hint = || match origin {
|
||||
Some(origin) => format!(
|
||||
"{err}\n(provider `{}` · requested model `{model}` · route: {} — {})",
|
||||
provider_name_for_error(provider),
|
||||
origin.source,
|
||||
origin.change,
|
||||
),
|
||||
None => format!(
|
||||
"{err}\n(provider `{}` · requested model `{model}` · route: {} — \
|
||||
the model may be unavailable under the current access profile; remove the explicit \
|
||||
child model override or adjust child-agent model config before retrying)",
|
||||
the model may be unavailable under the current access profile; {} before retrying)",
|
||||
provider_name_for_error(provider),
|
||||
route_source_label(route),
|
||||
)
|
||||
if matches!(route, ModelRoute::Fixed(_)) {
|
||||
// Fixed covers task `model` arguments, saved profiles and
|
||||
// config pins alike; without an origin, name them all.
|
||||
"change the child's pinned model (the task's `model` argument, a saved agent profile in /fleet members, or a config.toml role pin) or adjust child-agent model config"
|
||||
} else {
|
||||
"check that provider's access or adjust child-agent model config"
|
||||
},
|
||||
),
|
||||
};
|
||||
let lower = err.to_ascii_lowercase();
|
||||
match crate::error_taxonomy::classify_error_message(err) {
|
||||
|
||||
@@ -12439,9 +12439,13 @@ fn annotate_child_model_error_adds_actionable_hint() {
|
||||
let auth = annotate_child_model_error("403 Forbidden", "kimi-k2", provider, &inherit);
|
||||
assert!(auth.contains("kimi-k2"), "names the model: {auth}");
|
||||
assert!(
|
||||
auth.contains("child model override"),
|
||||
auth.contains("child-agent model config"),
|
||||
"names the recovery path: {auth}"
|
||||
);
|
||||
assert!(
|
||||
!auth.contains("explicit child model override"),
|
||||
"an inherited route never blames an override nobody gave: {auth}"
|
||||
);
|
||||
assert!(
|
||||
auth.contains("403 Forbidden"),
|
||||
"preserves the original: {auth}"
|
||||
@@ -15032,6 +15036,7 @@ pub(crate) fn stub_runtime() -> SubAgentRuntime {
|
||||
reasoning_effort_auto: false,
|
||||
role_models: std::collections::HashMap::new(),
|
||||
route_replacements: Vec::new(),
|
||||
route_origin: None,
|
||||
context,
|
||||
allow_shell: true,
|
||||
accept_edits: false,
|
||||
|
||||
@@ -85,11 +85,31 @@ async fn reviewer_tool(
|
||||
SharedSubAgentManager,
|
||||
Arc<AtomicUsize>,
|
||||
Arc<AtomicUsize>,
|
||||
) {
|
||||
reviewer_tool_with(root, pin, false).await
|
||||
}
|
||||
|
||||
/// [`reviewer_tool`], optionally with the parent route refusing as well.
|
||||
async fn reviewer_tool_with(
|
||||
root: &std::path::Path,
|
||||
pin: &str,
|
||||
parent_refuses: bool,
|
||||
) -> (
|
||||
AgentTool,
|
||||
ToolContext,
|
||||
SharedSubAgentManager,
|
||||
Arc<AtomicUsize>,
|
||||
Arc<AtomicUsize>,
|
||||
) {
|
||||
let (backup, backup_calls, _) = delayed_chat_client(Duration::ZERO, "review done").await;
|
||||
let (pin_url, pin_calls) = refusing_chat_server().await;
|
||||
let config_path = root.join("config.toml");
|
||||
write_replacement_config(&config_path, &pin_url, backup.base_url(), pin).await;
|
||||
let parent_url = if parent_refuses {
|
||||
pin_url.as_str()
|
||||
} else {
|
||||
backup.base_url()
|
||||
};
|
||||
write_replacement_config(&config_path, &pin_url, parent_url, pin).await;
|
||||
let config = crate::config::Config::load(Some(config_path), None).unwrap();
|
||||
let client = CodewhaleClient::new(&config).unwrap();
|
||||
let manager = new_shared_subagent_manager(root.to_path_buf(), 2);
|
||||
@@ -274,6 +294,300 @@ replacements = ["fixture-backup-model"]
|
||||
);
|
||||
}
|
||||
|
||||
/// The founder's shape: a saved agent profile (not a config pin) routes the
|
||||
/// reviewer role to a provider whose account refuses authorization, while
|
||||
/// the parent route works.
|
||||
async fn write_refusing_reviewer_profile(root: &std::path::Path) {
|
||||
let dir = root.join(".codewhale/agents");
|
||||
tokio::fs::create_dir_all(&dir).await.unwrap();
|
||||
tokio::fs::write(
|
||||
dir.join("reviewer.toml"),
|
||||
"id = \"reviewer\"\ndisplay_name = \"reviewer\"\nprovider = \"PinRoute\"\nmodel = \"fixture-pin-model\"\nrole_hint = \"reviewer\"\n",
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
struct ProjectProfiles(bool);
|
||||
impl ProjectProfiles {
|
||||
fn enabled() -> Self {
|
||||
let previous = crate::fleet::roster::project_agent_profiles_enabled();
|
||||
crate::fleet::roster::set_project_agent_profiles_enabled(true);
|
||||
Self(previous)
|
||||
}
|
||||
}
|
||||
impl Drop for ProjectProfiles {
|
||||
fn drop(&mut self) {
|
||||
crate::fleet::roster::set_project_agent_profiles_enabled(self.0);
|
||||
}
|
||||
}
|
||||
|
||||
async fn settle(manager: &SharedSubAgentManager, id: &str) -> SubAgentResult {
|
||||
tokio::time::timeout(Duration::from_secs(10), async {
|
||||
loop {
|
||||
let result = manager.read().await.get_result(id).expect("registered");
|
||||
if result.status != SubAgentStatus::Running {
|
||||
return result;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(10)).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("child settles")
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn refused_saved_profile_pin_runs_on_the_parent_route_visibly_and_once() {
|
||||
let _env = crate::test_support::lock_test_env();
|
||||
let _profiles = ProjectProfiles::enabled();
|
||||
let root = tempdir().unwrap();
|
||||
let _home = crate::test_support::EnvVarGuard::set("CODEWHALE_HOME", root.path().join("state"));
|
||||
let _provider = crate::test_support::EnvVarGuard::set("CODEWHALE_PROVIDER", "deepseek");
|
||||
let _model = crate::test_support::EnvVarGuard::set("CODEWHALE_MODEL", "deepseek-v4-flash");
|
||||
write_refusing_reviewer_profile(root.path()).await;
|
||||
let (tool, context, manager, pin_calls, backup_calls) = reviewer_tool(root.path(), "").await;
|
||||
|
||||
// First worker: the pin is asked once, refuses, and the same request runs
|
||||
// on the parent route with the reason on the receipt.
|
||||
let started = tool
|
||||
.execute(
|
||||
json!({"type": "reviewer", "prompt": "Review the change."}),
|
||||
&context,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let meta = started.metadata.clone().unwrap();
|
||||
assert_eq!(meta["child_route"]["route_source"], "agent_profile.model");
|
||||
assert_eq!(meta["child_route"]["provider_id"], "PinRoute");
|
||||
let first = settle(&manager, meta["agent_id"].as_str().unwrap()).await;
|
||||
assert_eq!(
|
||||
first.status,
|
||||
SubAgentStatus::Completed,
|
||||
"{:?}",
|
||||
first.status
|
||||
);
|
||||
assert_eq!(first.result.as_deref(), Some("review done"));
|
||||
assert_eq!(pin_calls.load(Ordering::SeqCst), 1, "the pin is asked once");
|
||||
assert!(
|
||||
backup_calls.load(Ordering::SeqCst) >= 1,
|
||||
"the parent route ran it"
|
||||
);
|
||||
let route = first.child_route.expect("route receipt");
|
||||
assert_eq!(route.route_source, "session.fallback");
|
||||
assert_eq!(route.provider_id, "deepseek");
|
||||
assert_eq!(route.model_id, "deepseek-v4-flash");
|
||||
let note = route.fallback_note.expect("fallback note");
|
||||
for fact in [
|
||||
"saved agent profile \"reviewer\" pins PinRoute/fixture-pin-model",
|
||||
"failed authorization",
|
||||
"run out of credits",
|
||||
"ran on deepseek/deepseek-v4-flash instead",
|
||||
] {
|
||||
assert!(note.contains(fact), "{fact} missing from {note}");
|
||||
}
|
||||
assert!(!note.contains("fixture-pin-key") && !note.contains("fixture-backup-key"));
|
||||
|
||||
// Second worker in the same session goes straight to the parent route:
|
||||
// the known-bad pin is not asked again.
|
||||
let started = tool
|
||||
.execute(
|
||||
json!({"type": "reviewer", "prompt": "Review the other change."}),
|
||||
&context,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let meta = started.metadata.clone().unwrap();
|
||||
assert_eq!(meta["child_route"]["route_source"], "session.fallback");
|
||||
assert_eq!(meta["child_route"]["provider_id"], "deepseek");
|
||||
let note = meta["child_route"]["fallback_note"].as_str().unwrap();
|
||||
assert!(note.contains("earlier this session"), "{note}");
|
||||
assert!(note.contains("PinRoute/fixture-pin-model"), "{note}");
|
||||
let second = settle(&manager, meta["agent_id"].as_str().unwrap()).await;
|
||||
assert_eq!(
|
||||
second.status,
|
||||
SubAgentStatus::Completed,
|
||||
"{:?}",
|
||||
second.status
|
||||
);
|
||||
assert_eq!(pin_calls.load(Ordering::SeqCst), 1, "no second refusal");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn strict_saved_profile_pin_stays_exact_and_names_its_source() {
|
||||
let _env = crate::test_support::lock_test_env();
|
||||
let _profiles = ProjectProfiles::enabled();
|
||||
let root = tempdir().unwrap();
|
||||
let _home = crate::test_support::EnvVarGuard::set("CODEWHALE_HOME", root.path().join("state"));
|
||||
write_refusing_reviewer_profile(root.path()).await;
|
||||
let (tool, _context, _manager, _pin, _backup) = reviewer_tool(root.path(), "").await;
|
||||
let runtime = tool.runtime.clone();
|
||||
let bind = |allow_fallback: bool| {
|
||||
let runtime = runtime.clone();
|
||||
async move {
|
||||
let roster = spawn_roster(&runtime);
|
||||
let mut request =
|
||||
parse_spawn_request(&json!({"type": "reviewer", "prompt": "Review."})).unwrap();
|
||||
let member = resolve_spawn_route_profile(&runtime, &mut request, &roster).unwrap();
|
||||
let mut child = runtime.child_runtime();
|
||||
bind_spawn_model_route(&mut child, &request, member.as_ref(), true, allow_fallback)
|
||||
.await
|
||||
.unwrap();
|
||||
child
|
||||
}
|
||||
};
|
||||
|
||||
let fallback = bind(true).await;
|
||||
let origin = fallback.route_origin.as_deref().expect("origin");
|
||||
assert_eq!(
|
||||
origin.parent.as_ref().map(|parent| parent.label.as_str()),
|
||||
Some("deepseek/deepseek-v4-flash")
|
||||
);
|
||||
|
||||
// An exact Fleet binding forbids the substitution: no parent route is
|
||||
// armed, and the failure names the saved profile and how to change it.
|
||||
let strict = bind(false).await;
|
||||
let origin = strict.route_origin.as_deref().expect("origin");
|
||||
assert!(origin.parent.is_none());
|
||||
let refusal = anyhow::Error::new(LlmError::from_http_response(403, REFUSAL));
|
||||
let message = annotate_child_model_error_with_origin(
|
||||
&subagent_failure_message(&refusal),
|
||||
&strict.model,
|
||||
strict.client.api_provider(),
|
||||
&ModelRoute::Fixed(strict.model.clone()),
|
||||
Some(origin),
|
||||
);
|
||||
for fact in [
|
||||
"run out of credits",
|
||||
"saved agent profile \"reviewer\" pins PinRoute/fixture-pin-model",
|
||||
"/fleet members",
|
||||
"reviewer.toml",
|
||||
"forbids falling back to the parent route",
|
||||
] {
|
||||
assert!(message.contains(fact), "{fact} missing from {message}");
|
||||
}
|
||||
assert!(
|
||||
!message.contains("explicit child model override"),
|
||||
"no override was given: {message}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn only_typed_auth_and_credit_refusals_move_a_saved_pin() {
|
||||
let refusal = |status| anyhow::Error::new(LlmError::from_http_response(status, REFUSAL));
|
||||
assert!(pin_refusal_reason(&refusal(403)).is_some());
|
||||
assert!(pin_refusal_reason(&refusal(401)).is_some());
|
||||
// Transient failures keep the ordinary retry path; no route change.
|
||||
for transient in [
|
||||
LlmError::RateLimited {
|
||||
message: "slow down".into(),
|
||||
retry_after: None,
|
||||
},
|
||||
LlmError::NetworkError("connection reset".into()),
|
||||
LlmError::ServerError {
|
||||
status: 503,
|
||||
message: "unavailable".into(),
|
||||
},
|
||||
LlmError::ModelError("no such model".into()),
|
||||
] {
|
||||
assert_eq!(pin_refusal_reason(&anyhow::Error::new(transient)), None);
|
||||
}
|
||||
assert_eq!(pin_refusal_reason(&anyhow!(REFUSAL)), None);
|
||||
}
|
||||
|
||||
/// Finding on #6717: after the fallback, a failure on the parent route was
|
||||
/// still reported against the refused pin ("change it in /fleet members").
|
||||
#[tokio::test]
|
||||
async fn a_failure_after_the_fallback_names_the_route_that_failed() {
|
||||
let _env = crate::test_support::lock_test_env();
|
||||
let _profiles = ProjectProfiles::enabled();
|
||||
let root = tempdir().unwrap();
|
||||
let _home = crate::test_support::EnvVarGuard::set("CODEWHALE_HOME", root.path().join("state"));
|
||||
let _provider = crate::test_support::EnvVarGuard::set("CODEWHALE_PROVIDER", "deepseek");
|
||||
let _model = crate::test_support::EnvVarGuard::set("CODEWHALE_MODEL", "deepseek-v4-flash");
|
||||
write_refusing_reviewer_profile(root.path()).await;
|
||||
let (tool, context, manager, calls, _backup) =
|
||||
reviewer_tool_with(root.path(), "", /* parent_refuses */ true).await;
|
||||
let started = tool
|
||||
.execute(
|
||||
json!({"type": "reviewer", "prompt": "Review the change."}),
|
||||
&context,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let meta = started.metadata.clone().unwrap();
|
||||
let result = settle(&manager, meta["agent_id"].as_str().unwrap()).await;
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 2, "pin once, parent once");
|
||||
let SubAgentStatus::Failed(error) = &result.status else {
|
||||
panic!("both routes refused: {:?}", result.status);
|
||||
};
|
||||
for fact in [
|
||||
"requested model `deepseek-v4-flash`",
|
||||
"deepseek/deepseek-v4-flash failed too",
|
||||
"saved agent profile \"reviewer\" pins PinRoute/fixture-pin-model",
|
||||
] {
|
||||
assert!(error.contains(fact), "{fact} missing from {error}");
|
||||
}
|
||||
assert!(
|
||||
!error.contains("requested model `fixture-pin-model`"),
|
||||
"the refused pin did not make the failing request: {error}"
|
||||
);
|
||||
assert!(
|
||||
manager.read().await.rerouted_requests.is_empty(),
|
||||
"taken when the task settles"
|
||||
);
|
||||
}
|
||||
|
||||
/// Finding on #6717: a resumed child skips the spawn's route binding, so its
|
||||
/// origin comes from the saved receipt instead of a guess from the model.
|
||||
#[test]
|
||||
fn a_resumed_child_names_its_saved_route_source() {
|
||||
let receipt: ChildRouteReceipt = serde_json::from_value(json!({
|
||||
"requested_type": "reviewer",
|
||||
"requested_profile": null,
|
||||
"resolved_profile_id": "reviewer",
|
||||
"profile_origin": null,
|
||||
"canonical_role": "reviewer",
|
||||
"provider_id": "PinRoute",
|
||||
"model_id": "fixture-pin-model",
|
||||
"route_source": "agent_profile.model",
|
||||
"fallback_note": null,
|
||||
"requested_reasoning": "auto",
|
||||
"effective_reasoning": null,
|
||||
"runtime_version": "0",
|
||||
"runtime_build_sha": "0"
|
||||
}))
|
||||
.expect("receipt");
|
||||
let origin = route_origin_from_receipt(&receipt).expect("known source");
|
||||
let refusal = anyhow::Error::new(LlmError::from_http_response(403, REFUSAL));
|
||||
let message = annotate_child_model_error_with_origin(
|
||||
&subagent_failure_message(&refusal),
|
||||
"fixture-pin-model",
|
||||
crate::config::ApiProvider::Deepseek,
|
||||
&ModelRoute::Fixed("fixture-pin-model".into()),
|
||||
Some(&origin),
|
||||
);
|
||||
assert!(
|
||||
message.contains("saved agent profile \"reviewer\" pins PinRoute/fixture-pin-model"),
|
||||
"{message}"
|
||||
);
|
||||
assert!(message.contains("/fleet members"), "{message}");
|
||||
assert!(
|
||||
!message.contains("explicit child model override"),
|
||||
"no override was given: {message}"
|
||||
);
|
||||
// Without any origin, a fixed route no longer claims an override either.
|
||||
let message = annotate_child_model_error(
|
||||
&subagent_failure_message(&refusal),
|
||||
"fixture-pin-model",
|
||||
crate::config::ApiProvider::Deepseek,
|
||||
&ModelRoute::Fixed("fixture-pin-model".into()),
|
||||
);
|
||||
assert!(
|
||||
!message.contains("explicit child model override"),
|
||||
"{message}"
|
||||
);
|
||||
}
|
||||
|
||||
/// The approved replacement owns the response grammar. Its distinct Responses
|
||||
/// item IDs do not make a repeated call_id safe to dispatch under Chat rules.
|
||||
#[tokio::test]
|
||||
|
||||
@@ -2732,10 +2732,12 @@ fn workflow_source(input: &Value, context: &ToolContext) -> Result<WorkflowSourc
|
||||
|
||||
match (script, source_path, plan) {
|
||||
(Some(source), None, None) if !source.trim().is_empty() => {
|
||||
workflow_source_from_raw(source, None)
|
||||
workflow_source_from_raw(source, None, Some(&context.workspace))
|
||||
}
|
||||
(None, Some(path), None) => read_workflow_source_path(path, context),
|
||||
(None, None, Some(plan_value)) => workflow_source_from_plan(plan_value),
|
||||
(None, None, Some(plan_value)) => {
|
||||
workflow_source_from_plan_in(plan_value, Some(&context.workspace))
|
||||
}
|
||||
_ => Err(ToolError::missing_field("script")),
|
||||
}
|
||||
}
|
||||
@@ -2745,9 +2747,17 @@ fn workflow_source(input: &Value, context: &ToolContext) -> Result<WorkflowSourc
|
||||
/// Accepts product-shaped plans (`goal` + `phases`/`children`) or IR-shaped
|
||||
/// plans (`goal` + `nodes`), validates them, and lowers to imperative JS that
|
||||
/// uses `parallel()` (partial success) rather than raw `Promise.all()`.
|
||||
#[cfg(test)]
|
||||
fn workflow_source_from_plan(plan_value: &Value) -> Result<WorkflowSource, ToolError> {
|
||||
workflow_source_from_plan_in(plan_value, None)
|
||||
}
|
||||
|
||||
fn workflow_source_from_plan_in(
|
||||
plan_value: &Value,
|
||||
workspace: Option<&Path>,
|
||||
) -> Result<WorkflowSource, ToolError> {
|
||||
let spec = structured_plan_to_workflow_spec(plan_value)?;
|
||||
let lowered = lower_declarative_workflow_to_imperative_js(&spec)?;
|
||||
let lowered = lower_declarative_workflow_to_imperative_js_in(&spec, workspace)?;
|
||||
Ok(WorkflowSource {
|
||||
source: lowered,
|
||||
path: None,
|
||||
@@ -3209,14 +3219,15 @@ fn read_workflow_source_path(
|
||||
canonical.display()
|
||||
))
|
||||
})?;
|
||||
workflow_source_from_raw(source, Some(canonical))
|
||||
workflow_source_from_raw(source, Some(canonical), Some(&context.workspace))
|
||||
}
|
||||
|
||||
fn workflow_source_from_raw(
|
||||
source: String,
|
||||
path: Option<PathBuf>,
|
||||
workspace: Option<&Path>,
|
||||
) -> Result<WorkflowSource, ToolError> {
|
||||
let adapted = adapt_workflow_source(&source, path.as_deref())?;
|
||||
let adapted = adapt_workflow_source_in(&source, path.as_deref(), workspace)?;
|
||||
Ok(WorkflowSource {
|
||||
source: adapted.source,
|
||||
path,
|
||||
@@ -3229,9 +3240,18 @@ struct AdaptedWorkflowSource {
|
||||
spec: Option<WorkflowSpec>,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
fn adapt_workflow_source(
|
||||
source: &str,
|
||||
path: Option<&Path>,
|
||||
) -> Result<AdaptedWorkflowSource, ToolError> {
|
||||
adapt_workflow_source_in(source, path, None)
|
||||
}
|
||||
|
||||
fn adapt_workflow_source_in(
|
||||
source: &str,
|
||||
path: Option<&Path>,
|
||||
workspace: Option<&Path>,
|
||||
) -> Result<AdaptedWorkflowSource, ToolError> {
|
||||
if !looks_like_declarative_workflow(source) {
|
||||
return Ok(AdaptedWorkflowSource {
|
||||
@@ -3258,7 +3278,7 @@ fn adapt_workflow_source(
|
||||
))
|
||||
})?;
|
||||
|
||||
let lowered = lower_declarative_workflow_to_imperative_js(&spec)?;
|
||||
let lowered = lower_declarative_workflow_to_imperative_js_in(&spec, workspace)?;
|
||||
Ok(AdaptedWorkflowSource {
|
||||
source: lowered,
|
||||
spec: Some(spec),
|
||||
@@ -3276,8 +3296,19 @@ fn looks_like_declarative_workflow(source: &str) -> bool {
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
fn lower_declarative_workflow_to_imperative_js(spec: &WorkflowSpec) -> Result<String, ToolError> {
|
||||
let mut lowerer = DeclarativeWorkflowLowerer::default();
|
||||
lower_declarative_workflow_to_imperative_js_in(spec, None)
|
||||
}
|
||||
|
||||
fn lower_declarative_workflow_to_imperative_js_in(
|
||||
spec: &WorkflowSpec,
|
||||
workspace: Option<&Path>,
|
||||
) -> Result<String, ToolError> {
|
||||
let mut lowerer = DeclarativeWorkflowLowerer {
|
||||
workspace: workspace.map(Path::to_path_buf),
|
||||
..DeclarativeWorkflowLowerer::default()
|
||||
};
|
||||
lowerer.line("\"use strict\";");
|
||||
lowerer.line("const __results = Object.create(null);");
|
||||
lowerer.line(format!(
|
||||
@@ -3295,6 +3326,9 @@ fn lower_declarative_workflow_to_imperative_js(spec: &WorkflowSpec) -> Result<St
|
||||
struct DeclarativeWorkflowLowerer {
|
||||
source: String,
|
||||
next_var: usize,
|
||||
/// Workspace root, so a leaf's absolute `cwd` inside it lowers to the
|
||||
/// repo-relative form `task()` accepts.
|
||||
workspace: Option<PathBuf>,
|
||||
}
|
||||
|
||||
impl DeclarativeWorkflowLowerer {
|
||||
@@ -3335,7 +3369,7 @@ impl DeclarativeWorkflowLowerer {
|
||||
self.line(format!(
|
||||
"__results[{}] = await task({});",
|
||||
js_string(&spec.id),
|
||||
leaf_task_options_expression(spec, phase, parallel)?
|
||||
leaf_task_options_expression(spec, phase, parallel, self.workspace.as_deref())?
|
||||
));
|
||||
Ok(())
|
||||
}
|
||||
@@ -3362,7 +3396,12 @@ impl DeclarativeWorkflowLowerer {
|
||||
// (#4120) unless the plan explicitly sets isolation: shared.
|
||||
self.line(format!(
|
||||
" () => task({}),",
|
||||
leaf_task_options_expression(leaf, Some(&spec.id), /* parallel */ true)?
|
||||
leaf_task_options_expression(
|
||||
leaf,
|
||||
Some(&spec.id),
|
||||
/* parallel */ true,
|
||||
self.workspace.as_deref(),
|
||||
)?
|
||||
));
|
||||
}
|
||||
self.line("]);");
|
||||
@@ -3472,6 +3511,7 @@ fn leaf_task_options_expression(
|
||||
spec: &LeafSpec,
|
||||
phase: Option<&str>,
|
||||
parallel: bool,
|
||||
workspace: Option<&Path>,
|
||||
) -> Result<String, ToolError> {
|
||||
validate_leaf_runtime_contract(spec)?;
|
||||
// Reject invalid plans before accepting a background run, using the same
|
||||
@@ -3479,7 +3519,7 @@ fn leaf_task_options_expression(
|
||||
let cwd = spec
|
||||
.cwd
|
||||
.as_deref()
|
||||
.map(codewhale_workflow_js::normalize_task_cwd)
|
||||
.map(|cwd| codewhale_workflow_js::normalize_task_cwd_in(cwd, workspace))
|
||||
.transpose()
|
||||
.map_err(|error| {
|
||||
ToolError::invalid_input(format!("Workflow leaf '{}': {error}", spec.id))
|
||||
@@ -4926,6 +4966,10 @@ impl WorkflowDriver for SubAgentWorkflowDriver {
|
||||
snapshot
|
||||
}
|
||||
|
||||
fn workspace_root(&self) -> Option<PathBuf> {
|
||||
Some(self.runtime.context.workspace.clone())
|
||||
}
|
||||
|
||||
fn progress(&self, event: ProgressEvent) {
|
||||
let mut schema_error = None;
|
||||
let mut schema_repair = None;
|
||||
@@ -5439,12 +5483,39 @@ fn spawn_completion_pump(
|
||||
let (task_completion, usage) =
|
||||
completion_from_manager(driver.manager.clone(), &agent_id, completion.payload)
|
||||
.await;
|
||||
// A worker that ran somewhere other than its requested route
|
||||
// says so in the run itself, not only on its receipt.
|
||||
if let Some(note) = rerouted_task_note(&driver.manager, &agent_id).await {
|
||||
driver.record_run_event(WorkflowUiEvent::new(
|
||||
&driver.owner_session_id,
|
||||
WorkflowUiEventKind::Log {
|
||||
message: format!("Route fallback ({agent_id}): {note}"),
|
||||
},
|
||||
));
|
||||
}
|
||||
driver.deliver_completion(agent_id, task_completion, usage);
|
||||
}
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
/// The fallback note of a child that ran on a replacement or parent route
|
||||
/// instead of the one it requested, if it did.
|
||||
async fn rerouted_task_note(manager: &SharedSubAgentManager, agent_id: &str) -> Option<String> {
|
||||
let route = manager
|
||||
.read()
|
||||
.await
|
||||
.get_result(agent_id)
|
||||
.ok()?
|
||||
.child_route?;
|
||||
matches!(
|
||||
route.route_source.as_str(),
|
||||
"session.fallback" | "role.replacement"
|
||||
)
|
||||
.then_some(route.fallback_note)
|
||||
.flatten()
|
||||
}
|
||||
|
||||
/// Resolve a child's terminal completion from the manager, reading it exactly
|
||||
/// once.
|
||||
///
|
||||
@@ -8116,6 +8187,43 @@ export default workflow({
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn leaf_absolute_cwd_inside_the_workspace_lowers_repo_relative() {
|
||||
// Founder run: a plan leaf with cwd "/Volumes/VIXinSSD/CW/codewhale"
|
||||
// was refused although it named the workspace's own checkout.
|
||||
// Platform-absolute paths: `/Volumes/…` is not absolute on Windows.
|
||||
let workspace = std::env::temp_dir().join("cw-leaf-fixture");
|
||||
let leaf: LeafSpec = serde_json::from_value(json!({
|
||||
"id": "engine-readiness",
|
||||
"prompt": "Audit release readiness",
|
||||
"agent_type": "review",
|
||||
"mode": "read_only",
|
||||
"cwd": workspace.join("codewhale").to_str().unwrap()
|
||||
}))
|
||||
.expect("leaf");
|
||||
let source = leaf_task_options_expression(&leaf, None, false, Some(&workspace)).unwrap();
|
||||
assert!(source.contains("cwd: \"codewhale\""), "{source}");
|
||||
assert!(!source.contains("cw-leaf-fixture"), "{source}");
|
||||
|
||||
let mut outside = leaf.clone();
|
||||
outside.cwd = Some(
|
||||
std::env::temp_dir()
|
||||
.join("cw-leaf-fixture-other")
|
||||
.to_str()
|
||||
.unwrap()
|
||||
.to_string(),
|
||||
);
|
||||
let error = leaf_task_options_expression(&outside, None, false, Some(&workspace))
|
||||
.unwrap_err()
|
||||
.to_string();
|
||||
assert!(error.contains("outside the workspace"), "{error}");
|
||||
// Without a known workspace the old refusal stands.
|
||||
let error = leaf_task_options_expression(&leaf, None, false, None)
|
||||
.unwrap_err()
|
||||
.to_string();
|
||||
assert!(error.contains("bounded repo-relative paths"), "{error}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn read_only_workflow_leaves_use_the_runtime_grant_unless_explicitly_narrowed() {
|
||||
for agent_type in ["explore", "review", "verifier", "general"] {
|
||||
@@ -8127,7 +8235,7 @@ export default workflow({
|
||||
}))
|
||||
.expect("read-only leaf");
|
||||
assert_eq!(leaf_allowed_tools(&leaf).unwrap(), None);
|
||||
let source = leaf_task_options_expression(&leaf, None, false).unwrap();
|
||||
let source = leaf_task_options_expression(&leaf, None, false, None).unwrap();
|
||||
assert!(source.contains("writeAuthority: \"read_only\""), "{source}");
|
||||
assert!(!source.contains("allowedTools:"), "{source}");
|
||||
|
||||
|
||||
@@ -286,6 +286,12 @@ pub trait WorkflowDriver: Send + Sync {
|
||||
|
||||
/// Receive a script progress event (ordered, synchronous).
|
||||
fn progress(&self, event: ProgressEvent);
|
||||
|
||||
/// The run's workspace root. When known, `task()` accepts an absolute
|
||||
/// `cwd` inside it and normalizes it to the repo-relative form.
|
||||
fn workspace_root(&self) -> Option<std::path::PathBuf> {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
/// One `tools.call()` invocation: a tool name plus its JSON arguments.
|
||||
|
||||
@@ -72,7 +72,7 @@ pub use driver::{
|
||||
};
|
||||
pub use error::{DriverError, TaskErrorKind, WorkflowJsError};
|
||||
pub use schema::{SCHEMA_RAW_CARRY_CHARS, SCHEMA_RAW_PREVIEW_CHARS, SCHEMA_REPAIR_MAX_ATTEMPTS};
|
||||
pub use vm::{VmLimits, WorkflowRunCancel, WorkflowVm, normalize_task_cwd};
|
||||
pub use vm::{VmLimits, WorkflowRunCancel, WorkflowVm, normalize_task_cwd, normalize_task_cwd_in};
|
||||
|
||||
/// Maximum `task()` spawn attempts per run (design §4.3). Counted in the VM
|
||||
/// before the driver is consulted, so a runaway `loop-until-dry` terminates
|
||||
|
||||
@@ -946,7 +946,8 @@ async fn task_host_inner(
|
||||
spawned: Rc<Cell<u64>>,
|
||||
) -> Result<serde_json::Value, TaskError> {
|
||||
let admission = |message: String| TaskError::new(TaskErrorKind::Admission, message);
|
||||
let request = parse_task_options(&opts_json)
|
||||
let workspace = driver.workspace_root();
|
||||
let request = parse_task_options(&opts_json, workspace.as_deref())
|
||||
.map_err(|message| admission(reject_task(&driver, &opts_json, message)))?;
|
||||
// Compile the schema before spawning so a malformed one fails fast
|
||||
// instead of burning a subagent.
|
||||
@@ -1180,7 +1181,10 @@ struct TaskOptions {
|
||||
phase: Option<String>,
|
||||
}
|
||||
|
||||
fn parse_task_options(opts_json: &str) -> Result<TaskRequest, String> {
|
||||
fn parse_task_options(
|
||||
opts_json: &str,
|
||||
workspace: Option<&std::path::Path>,
|
||||
) -> Result<TaskRequest, String> {
|
||||
let mut options: TaskOptions =
|
||||
serde_json::from_str(opts_json).map_err(|err| format!("task(): invalid options: {err}"))?;
|
||||
if let Some(policy) = options.workspace_policy.take() {
|
||||
@@ -1220,7 +1224,11 @@ fn parse_task_options(opts_json: &str) -> Result<TaskRequest, String> {
|
||||
.map_err(|err| format!("task(): {err}"))?;
|
||||
options.write_roots = normalize_task_paths("writeRoots", options.write_roots, 32)?;
|
||||
options.exact_files = normalize_task_paths("exactFiles", options.exact_files, 32)?;
|
||||
let cwd = options.cwd.as_deref().map(normalize_task_cwd).transpose()?;
|
||||
let cwd = options
|
||||
.cwd
|
||||
.as_deref()
|
||||
.map(|cwd| normalize_task_cwd_in(cwd, workspace))
|
||||
.transpose()?;
|
||||
options.coordination_contracts =
|
||||
normalize_task_string_list("coordinationContracts", options.coordination_contracts, 16)?;
|
||||
options.dependencies = normalize_task_string_list("dependencies", options.dependencies, 8)?;
|
||||
@@ -1325,6 +1333,122 @@ pub fn normalize_task_cwd(value: &str) -> Result<String, String> {
|
||||
normalize_task_paths("cwd", vec![value.to_owned()], 1).map(|mut paths| paths.remove(0))
|
||||
}
|
||||
|
||||
/// [`normalize_task_cwd`] for a run that knows its workspace root. An
|
||||
/// absolute path that lies inside `workspace` is rewritten to the same
|
||||
/// bounded repo-relative form; an absolute path outside it, or one that
|
||||
/// escapes through `..`, is still rejected. Without a workspace every
|
||||
/// absolute path is rejected, exactly as before.
|
||||
///
|
||||
/// The comparison is lexical and component-wise (no filesystem access), so
|
||||
/// `/ws-other` is never inside `/ws`. On Windows a drive letter or UNC share
|
||||
/// matches with or without the verbatim `\\?\` prefix, `/` and `\` are both
|
||||
/// separators, and components compare case-insensitively.
|
||||
pub fn normalize_task_cwd_in(
|
||||
value: &str,
|
||||
workspace: Option<&std::path::Path>,
|
||||
) -> Result<String, String> {
|
||||
let raw = value.trim();
|
||||
let path = std::path::Path::new(raw);
|
||||
let Some(workspace) = workspace.filter(|_| path.is_absolute()) else {
|
||||
return normalize_task_cwd(value);
|
||||
};
|
||||
let outside = || {
|
||||
format!(
|
||||
"task(): cwd {raw:?} is outside the workspace {}; cwd entries must be bounded repo-relative paths or absolute paths inside the workspace",
|
||||
workspace.display()
|
||||
)
|
||||
};
|
||||
let mut remainder = path_keys(path).ok_or_else(outside)?.into_iter();
|
||||
for expected in path_keys(workspace).ok_or_else(outside)? {
|
||||
match remainder.next() {
|
||||
Some(actual) if actual.matches(&expected) => {}
|
||||
// `..` right after the workspace prefix is an escape, not a
|
||||
// different root: name it as traversal.
|
||||
Some(PathKey::Parent) => {
|
||||
return Err("task(): cwd paths cannot contain parent traversal".to_string());
|
||||
}
|
||||
_ => return Err(outside()),
|
||||
}
|
||||
}
|
||||
let mut segments = Vec::new();
|
||||
for key in remainder {
|
||||
match key {
|
||||
PathKey::Name { original, .. } => segments.push(original),
|
||||
PathKey::Parent => {
|
||||
return Err("task(): cwd paths cannot contain parent traversal".to_string());
|
||||
}
|
||||
PathKey::Root => return Err(outside()),
|
||||
}
|
||||
}
|
||||
if segments.is_empty() {
|
||||
return Ok(".".to_string());
|
||||
}
|
||||
normalize_task_cwd(&segments.join("/"))
|
||||
}
|
||||
|
||||
/// One lexical path component, keyed for comparison: a drive or UNC prefix
|
||||
/// with its verbatim marker dropped, and names case-folded on Windows.
|
||||
#[derive(Debug)]
|
||||
enum PathKey {
|
||||
Root,
|
||||
Parent,
|
||||
Name { key: String, original: String },
|
||||
}
|
||||
|
||||
impl PathKey {
|
||||
fn matches(&self, other: &Self) -> bool {
|
||||
match (self, other) {
|
||||
(Self::Root, Self::Root) | (Self::Parent, Self::Parent) => true,
|
||||
(Self::Name { key: left, .. }, Self::Name { key: right, .. }) => left == right,
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn path_keys(path: &std::path::Path) -> Option<Vec<PathKey>> {
|
||||
use std::path::{Component, Prefix};
|
||||
let fold = |value: &str| {
|
||||
if cfg!(windows) {
|
||||
value.to_lowercase()
|
||||
} else {
|
||||
value.to_string()
|
||||
}
|
||||
};
|
||||
let mut keys = Vec::new();
|
||||
for component in path.components() {
|
||||
match component {
|
||||
Component::Prefix(prefix) => {
|
||||
let key = match prefix.kind() {
|
||||
Prefix::Disk(drive) | Prefix::VerbatimDisk(drive) => {
|
||||
format!("{}:", drive.to_ascii_lowercase() as char)
|
||||
}
|
||||
Prefix::UNC(server, share) | Prefix::VerbatimUNC(server, share) => format!(
|
||||
"//{}/{}",
|
||||
server.to_str()?.to_lowercase(),
|
||||
share.to_str()?.to_lowercase()
|
||||
),
|
||||
_ => prefix.as_os_str().to_str()?.to_lowercase(),
|
||||
};
|
||||
keys.push(PathKey::Name {
|
||||
original: key.clone(),
|
||||
key,
|
||||
});
|
||||
}
|
||||
Component::RootDir => keys.push(PathKey::Root),
|
||||
Component::CurDir => {}
|
||||
Component::ParentDir => keys.push(PathKey::Parent),
|
||||
Component::Normal(name) => {
|
||||
let original = name.to_str()?.to_string();
|
||||
keys.push(PathKey::Name {
|
||||
key: fold(&original),
|
||||
original,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
Some(keys)
|
||||
}
|
||||
|
||||
fn normalize_task_paths(
|
||||
field: &str,
|
||||
values: Vec<String>,
|
||||
|
||||
@@ -2546,3 +2546,102 @@ async fn tools_call_cap_rejects_runaway_loops() {
|
||||
"unexpected: {message}"
|
||||
);
|
||||
}
|
||||
|
||||
/// A platform-native absolute path for `unix_path` (`/a/b`): unchanged on
|
||||
/// Unix, `C:\a\b` on Windows, where `/a/b` has no drive and is not absolute.
|
||||
fn native_abs(unix_path: &str) -> String {
|
||||
if cfg!(windows) {
|
||||
format!("C:{}", unix_path.replace('/', "\\"))
|
||||
} else {
|
||||
unix_path.to_string()
|
||||
}
|
||||
}
|
||||
|
||||
/// An absolute `cwd` inside the run's workspace normalizes to the
|
||||
/// repo-relative form; outside it, through `..`, or with no known workspace
|
||||
/// it is still refused (the trust boundary is unchanged).
|
||||
#[test]
|
||||
fn absolute_cwd_inside_the_workspace_normalizes_and_outside_is_refused() {
|
||||
use codewhale_workflow_js::{normalize_task_cwd, normalize_task_cwd_in};
|
||||
use std::path::PathBuf;
|
||||
let workspace = PathBuf::from(native_abs("/Volumes/VIXinSSD/CW"));
|
||||
let workspace = Some(workspace.as_path());
|
||||
assert_eq!(
|
||||
normalize_task_cwd_in(&native_abs("/Volumes/VIXinSSD/CW/codewhale"), workspace).unwrap(),
|
||||
"codewhale"
|
||||
);
|
||||
assert_eq!(
|
||||
normalize_task_cwd_in(&native_abs("/Volumes/VIXinSSD/CW/./crates/tui"), workspace).unwrap(),
|
||||
"crates/tui"
|
||||
);
|
||||
assert_eq!(
|
||||
normalize_task_cwd_in(&native_abs("/Volumes/VIXinSSD/CW/"), workspace).unwrap(),
|
||||
"."
|
||||
);
|
||||
assert_eq!(
|
||||
normalize_task_cwd_in("crates/tui", workspace).unwrap(),
|
||||
normalize_task_cwd("crates/tui").unwrap()
|
||||
);
|
||||
for outside in [
|
||||
native_abs("/Volumes/VIXinSSD/CW-other/codewhale"),
|
||||
native_abs("/Volumes/VIXinSSD"),
|
||||
native_abs("/etc"),
|
||||
native_abs("/Volumes/VIXinSSD/CW/../secrets"),
|
||||
native_abs("/Volumes/VIXinSSD/CW/codewhale/../../secrets"),
|
||||
native_abs("/Volumes/VIXinSSD/X/../CW/codewhale"),
|
||||
] {
|
||||
let error = normalize_task_cwd_in(&outside, workspace).unwrap_err();
|
||||
assert!(
|
||||
error.contains("outside the workspace") || error.contains("parent traversal"),
|
||||
"{outside}: {error}"
|
||||
);
|
||||
}
|
||||
let error =
|
||||
normalize_task_cwd_in(&native_abs("/Volumes/VIXinSSD/CW/codewhale"), None).unwrap_err();
|
||||
assert!(error.contains("bounded repo-relative paths"), "{error}");
|
||||
}
|
||||
|
||||
/// Windows spellings of the same workspace path: a verbatim `\\?\` prefix on
|
||||
/// either side, forward slashes, and a different case all match; another
|
||||
/// drive or a sibling directory does not.
|
||||
#[cfg(windows)]
|
||||
#[test]
|
||||
fn absolute_cwd_matches_windows_spellings_of_the_workspace() {
|
||||
use codewhale_workflow_js::normalize_task_cwd_in;
|
||||
use std::path::Path;
|
||||
let plain = Path::new(r"C:\Users\dev\CW");
|
||||
let verbatim = Path::new(r"\\?\C:\Users\dev\CW");
|
||||
for workspace in [plain, verbatim] {
|
||||
for inside in [
|
||||
r"C:\Users\dev\CW\codewhale",
|
||||
"C:/Users/dev/CW/codewhale",
|
||||
r"c:\users\DEV\cw\codewhale",
|
||||
r"\\?\C:\Users\dev\CW\codewhale",
|
||||
] {
|
||||
assert_eq!(
|
||||
normalize_task_cwd_in(inside, Some(workspace)).unwrap(),
|
||||
"codewhale",
|
||||
"{inside} in {}",
|
||||
workspace.display()
|
||||
);
|
||||
}
|
||||
for outside in [
|
||||
r"D:\Users\dev\CW\codewhale",
|
||||
r"C:\Users\dev\CW-other",
|
||||
r"C:\Users\dev\CW\..\secrets",
|
||||
r"\\server\share\Users\dev\CW",
|
||||
] {
|
||||
let error = normalize_task_cwd_in(outside, Some(workspace)).unwrap_err();
|
||||
assert!(
|
||||
error.contains("outside the workspace") || error.contains("parent traversal"),
|
||||
"{outside}: {error}"
|
||||
);
|
||||
}
|
||||
}
|
||||
// A UNC workspace matches its own share, case-insensitively.
|
||||
let unc = Path::new(r"\\Server\Share\CW");
|
||||
assert_eq!(
|
||||
normalize_task_cwd_in(r"\\server\share\cw\codewhale", Some(unc)).unwrap(),
|
||||
"codewhale"
|
||||
);
|
||||
}
|
||||
Reference in new issue
Block a user