feat(tui): show model capability badges in Fleet setup and roster (#5069)

* feat(tui): show model capability badges in Fleet setup and roster

Fleet setup asked users to assign models to roles without showing the
facts needed to choose. Add one shared resolver,
fleet::capability_badges::resolve_route_capability_badges, that answers
"what can this model do" for a (provider, model) route:

- exact merged Models.dev catalog offerings first (provider-aware,
  bundled/live/override layers via provider_lake — no second catalog),
- seeded model_registry facts for ids without a catalog row,
- None for completely unknown models, so absence renders as absence and
  never blocks selection.

Badges are concise and provenance-labelled (`1M ctx · 384K out · tools ·
reasoning · no vision (bundled catalog)`): three-state catalog facts keep
explicit "no tools"/"no vision", unknowns are omitted rather than guessed.

Rendered in the Fleet setup Model step's word-wrapped detail pane (list
stays narrow-terminal friendly) and in the roster detail pane for pinned
member models and the operator's session model.

model_registry.rs loses its module-level #![allow(dead_code)]: every item
now has production consumers (model_profile, model_picker, and this
resolver).

Tests cover known-catalog resolution, registry fallback for custom
routes, graceful absence for unknown models, explicit-unsupported badge
text, auto->model display routes, and both views' rendering paths.

Implemented with agent assistance (Claude).

Refs #5038

* fix(ci): keep fleet roster below structure ceiling

* fix(fleet): keep capability facts provider-aware

Signed-off-by: Hmbown <[email protected]>

---------

Signed-off-by: Hmbown <[email protected]>
This commit is contained in:
Hunter Bown authored and GitHub committed 2026-08-03 05:00:54 -07:00
1 parent 83d42a8551
commit a94c5b5239
7 files changed
+441 -13

No files matched your search

+255
View File
@@ -0,0 +1,255 @@
//! Concise model-capability badges for Fleet configuration UI (#5038).
//!
//! One resolver answers "what can this model do" for both the Fleet setup
//! model-selection step and the roster detail pane. Facts come from the
//! existing owners — the merged Models.dev catalog (via
//! [`crate::provider_lake::catalog_offering_for_model`], provider-aware, with
//! bundled/live/override layers) first, then the seeded
//! [`crate::model_registry`] facts for ids no catalog row covers (custom
//! providers, local models). No second model catalog is introduced here.
//!
//! Honesty rules: unknown facts are omitted rather than guessed, an explicit
//! catalog "unsupported" renders as `no <badge>`, provenance is always named,
//! and a completely unknown model resolves to `None` so callers can skip the
//! line entirely instead of blocking selection or fabricating capabilities.
use codewhale_config::catalog::CatalogSource;
use codewhale_config::route::{CapabilityState, RouteCapabilities, RouteLimits};
use crate::config::ApiProvider;
use crate::model_registry::{self, ModelMetadata};
use crate::tui::model_picker::format_picker_context_window;
/// Resolved capability badges for one Fleet route.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct RouteCapabilityBadges {
/// Ordered, concise badges, e.g. `["1M ctx", "384K out", "tools",
/// "reasoning", "no vision"]`. Never empty when resolution succeeds.
pub badges: Vec<String>,
/// Where the facts came from: `bundled catalog`, `live catalog`,
/// `override`, or `registry`.
pub provenance: &'static str,
}
impl RouteCapabilityBadges {
/// One-line summary with provenance, sized for narrow detail panes:
/// `1M ctx · 384K out · tools · reasoning · no vision (bundled catalog)`.
#[must_use]
pub fn summary(&self) -> String {
format!("{} ({})", self.badges.join(" · "), self.provenance)
}
}
/// Resolve capability badges for one `(provider, model)` Fleet route.
///
/// `provider_id` is the exact configured route key when known (canonical
/// built-in id or named custom table key); pass `None` when the route has no
/// resolvable provider (e.g. a pinned model that inherits the provider). Only
/// exact built-in ids reach the provider-scoped catalog; every other id falls
/// back to provider-agnostic registry facts rather than guessing a route.
///
/// The session `auto -> model` display form is accepted and resolved against
/// the effective model. Returns `None` when nothing is known about the model,
/// so absence renders as absence.
#[must_use]
pub fn resolve_route_capability_badges(
provider_id: Option<&str>,
model: &str,
) -> Option<RouteCapabilityBadges> {
let model = effective_model(model)?;
if let Some(provider) = provider_id.and_then(exact_builtin_provider)
&& let Some(offering) = crate::provider_lake::catalog_offering_for_model(provider, model)
{
let route = offering.to_offering();
let badges = badges_from_route(&route.limits, &route.capabilities);
if !badges.is_empty() {
return Some(RouteCapabilityBadges {
badges,
provenance: catalog_provenance(&offering.source),
});
}
}
let meta = model_registry::lookup(model)?;
let badges = badges_from_registry(&meta);
(!badges.is_empty()).then_some(RouteCapabilityBadges {
badges,
provenance: "registry",
})
}
/// Strip the session `auto -> model` display form down to the effective model.
/// A bare `auto` (nothing resolved yet) or empty id yields `None`.
fn effective_model(model: &str) -> Option<&str> {
let model = model
.split_once("->")
.map_or(model, |(_, effective)| effective)
.trim();
(!model.is_empty() && !model.eq_ignore_ascii_case("auto")).then_some(model)
}
/// Accept only exact canonical built-in provider ids. Display labels and named
/// custom table keys must not inherit a built-in catalog by similarity.
fn exact_builtin_provider(provider_id: &str) -> Option<ApiProvider> {
ApiProvider::parse(provider_id).filter(|provider| provider.as_str() == provider_id)
}
const fn catalog_provenance(source: &CatalogSource) -> &'static str {
match source {
CatalogSource::Bundled => "bundled catalog",
CatalogSource::Live { .. } => "live catalog",
CatalogSource::UserOverride => "override",
}
}
/// Badges from exact provider-offering facts. Three-state facts keep their
/// explicit `Unsupported` (`no tools` / `no vision`); `Unknown` is omitted.
fn badges_from_route(limits: &RouteLimits, capabilities: &RouteCapabilities) -> Vec<String> {
let mut badges = Vec::new();
if let Some(context) = limits.context_tokens {
badges.push(format!("{} ctx", format_picker_context_window(context)));
}
if let Some(output) = limits.output_tokens {
badges.push(format!("{} out", format_picker_context_window(output)));
}
push_state_badge(&mut badges, capabilities.native_tool_calls, "tools");
push_state_badge(&mut badges, capabilities.reasoning, "reasoning");
push_state_badge(&mut badges, capabilities.image_input, "vision");
badges
}
/// Badges from seeded registry facts. The registry has no tool/vision facts,
/// and its `supports_reasoning: false` is a heuristic default rather than a
/// sourced denial, so only a positive reasoning fact is shown.
fn badges_from_registry(meta: &ModelMetadata) -> Vec<String> {
let mut badges = Vec::new();
if let Some(context) = meta.context_window {
badges.push(format!(
"{} ctx",
format_picker_context_window(u64::from(context))
));
}
if let Some(output) = meta.max_output {
badges.push(format!(
"{} out",
format_picker_context_window(u64::from(output))
));
}
if meta.supports_reasoning {
badges.push("reasoning".to_string());
}
badges
}
fn push_state_badge(badges: &mut Vec<String>, state: CapabilityState, name: &str) {
match state {
CapabilityState::Supported => badges.push(name.to_string()),
CapabilityState::Unsupported => badges.push(format!("no {name}")),
CapabilityState::Unknown => {}
}
}
#[cfg(test)]
mod tests {
use super::*;
use codewhale_config::catalog::CatalogOffering;
use codewhale_config::models_dev::{ModelsDevLimit, ModelsDevModalities};
#[test]
fn known_catalog_model_resolves_provider_aware_badges() {
let badges = resolve_route_capability_badges(Some("deepseek"), "deepseek-v4-pro")
.expect("bundled catalog knows deepseek-v4-pro");
assert!(
badges.badges.contains(&"1M ctx".to_string()),
"missing context badge: {:?}",
badges.badges
);
assert!(badges.badges.contains(&"384K out".to_string()));
assert!(badges.badges.contains(&"tools".to_string()));
assert!(badges.badges.contains(&"reasoning".to_string()));
// Text-only modalities are an explicit sourced fact, not an unknown.
assert!(badges.badges.contains(&"no vision".to_string()));
assert!(
badges.provenance.contains("catalog"),
"catalog facts must carry catalog provenance, got {}",
badges.provenance
);
assert!(badges.summary().contains(" · "));
}
#[test]
fn unknown_model_resolves_to_graceful_absence() {
assert_eq!(
resolve_route_capability_badges(Some("deepseek"), "totally-made-up-model-xyz"),
None
);
assert_eq!(resolve_route_capability_badges(None, ""), None);
assert_eq!(resolve_route_capability_badges(None, "auto"), None);
}
#[test]
fn registry_fallback_covers_routes_without_catalog_rows() {
// A named custom route key never inherits a built-in catalog; the
// provider-agnostic registry still answers for the model id.
let badges = resolve_route_capability_badges(Some("my-custom-endpoint"), "claude-fable-5")
.expect("registry knows claude-fable-5");
assert_eq!(badges.provenance, "registry");
assert!(badges.badges.contains(&"1M ctx".to_string()));
assert!(badges.badges.contains(&"reasoning".to_string()));
// Tool/vision facts are unknown here — omitted, never fabricated.
assert!(
!badges
.badges
.iter()
.any(|badge| badge.contains("tools") || badge.contains("vision")),
"unsourced facts must not appear: {:?}",
badges.badges
);
}
#[test]
fn auto_display_route_resolves_the_effective_model() {
let badges = resolve_route_capability_badges(None, "auto -> deepseek-v4-pro")
.expect("effective model resolves");
assert!(badges.badges.contains(&"1M ctx".to_string()));
}
#[test]
fn explicit_unsupported_catalog_facts_render_as_no_badges() {
let offering = CatalogOffering {
provider: "deepseek".to_string(),
wire_model_id: "fixture-model".to_string(),
endpoint_key: "chat".to_string(),
limit: Some(ModelsDevLimit {
context: Some(131_072),
input: None,
output: None,
}),
reasoning: Some(false),
tool_call: Some(false),
modalities: Some(ModelsDevModalities {
input: vec!["text".to_string(), "image".to_string()],
output: vec!["text".to_string()],
}),
..CatalogOffering::default()
};
let route = offering.to_offering();
let badges = badges_from_route(&route.limits, &route.capabilities);
assert_eq!(
badges,
vec![
"131K ctx".to_string(),
"no tools".to_string(),
"no reasoning".to_string(),
"vision".to_string(),
]
);
}
#[test]
fn token_labels_match_picker_vocabulary() {
assert_eq!(format_picker_context_window(1_000_000), "1M");
assert_eq!(format_picker_context_window(1_050_000), "1.05M");
assert_eq!(format_picker_context_window(262_144), "262K");
assert_eq!(format_picker_context_window(500), "500");
}
}
+1
View File
@@ -1,6 +1,7 @@
//! Agent Fleet control plane — local-first manager, ledger, and workers.
pub mod alerts;
pub mod capability_badges;
pub mod control;
pub mod exact;
pub mod executor;
+3 -4
View File
@@ -27,10 +27,9 @@
//! re-asserts the equivalence for a sample so that if a future change replaces
//! a seed with a hard-coded literal, CI catches the drift immediately.
//!
//! NOTE: the public surface here is intentionally not yet consumed by
//! production call sites (consumers are wired in a later pass), so
//! `dead_code` is allowed at the module level until then.
#![allow(dead_code)]
//! Production consumers: [`crate::model_profile`] (capability bridge),
//! `crate::tui::model_picker` (picker hints), and
//! [`crate::fleet::capability_badges`] (Fleet setup/roster badges, #5038).
use std::collections::BTreeMap;
use std::sync::OnceLock;
+8 -5
View File
@@ -1688,7 +1688,7 @@ fn render_picker_model_hint(
if provider == Some(ApiProvider::OpenaiCodex) {
parts.push(format!(
"{} ctx · ChatGPT route",
format_picker_context_window(context_window)
format_picker_context_window(u64::from(context_window))
));
} else if provider == Some(ApiProvider::Moonshot)
&& id.trim().eq_ignore_ascii_case("k3")
@@ -1700,18 +1700,21 @@ fn render_picker_model_hint(
// `context_window` setting when the plan includes 1M.
parts.push(format!(
"{} ctx (plan floor; raise via context_window)",
format_picker_context_window(context_window)
format_picker_context_window(u64::from(context_window))
));
} else {
parts.push(format!(
"{} ctx",
format_picker_context_window(context_window)
format_picker_context_window(u64::from(context_window))
));
}
}
if let Some(max_output) = metadata.max_output {
parts.push(format!("{} out", format_picker_context_window(max_output)));
parts.push(format!(
"{} out",
format_picker_context_window(u64::from(max_output))
));
}
match metadata.tool_calls {
@@ -1755,7 +1758,7 @@ fn render_picker_model_hint(
}
}
fn format_picker_context_window(tokens: u32) -> String {
pub(crate) fn format_picker_context_window(tokens: u64) -> String {
if tokens >= 1_000_000 {
if tokens.is_multiple_of(1_000_000) {
format!("{}M", tokens / 1_000_000)
+53 -3
View File
@@ -43,6 +43,9 @@ use crate::worker_profile::{ShellPolicy, WorkerRuntimeProfile};
#[derive(Debug, Clone)]
struct OperatorInfo {
provider: String,
/// Exact canonical route key, kept separate from the display label so
/// capability lookup can use provider-scoped catalog facts.
provider_id: String,
model: String,
reasoning: String,
}
@@ -57,13 +60,32 @@ impl OperatorInfo {
} else {
app.model.clone()
};
let provider = if app.api_provider == crate::config::ApiProvider::Custom {
app.provider_identity_for_persistence().to_string()
let route_provider = if app.auto_model {
app.last_effective_provider.unwrap_or(app.api_provider)
} else {
app.api_provider.display_name().to_string()
app.api_provider
};
let provider_id = if app.auto_model {
app.last_effective_provider_identity
.clone()
.unwrap_or_else(|| {
if route_provider == crate::config::ApiProvider::Custom {
app.provider_identity_for_persistence().to_string()
} else {
route_provider.as_str().to_string()
}
})
} else {
app.provider_identity_for_persistence().to_string()
};
let provider = if route_provider == crate::config::ApiProvider::Custom {
provider_id.clone()
} else {
route_provider.display_name().to_string()
};
Self {
provider,
provider_id,
model,
reasoning: app.reasoning_effort_display_label(),
}
@@ -439,6 +461,15 @@ fn operator_detail_lines(operator: &OperatorInfo) -> Vec<Line<'static>> {
detail_field(&mut lines, "Posture", "full session authority".to_string());
detail_field(&mut lines, "Provider", operator.provider.clone());
detail_field(&mut lines, "Model", operator.model.clone());
// Session-route capability badges (#5038). Use the exact route key rather
// than the display label so built-in routes get provider-scoped catalog
// facts; custom routes still fall back conservatively to registry facts.
if let Some(badges) = crate::fleet::capability_badges::resolve_route_capability_badges(
Some(&operator.provider_id),
&operator.model,
) {
detail_field(&mut lines, "Capabilities", badges.summary());
}
detail_field(&mut lines, "Reasoning", operator.reasoning.clone());
detail_field(
&mut lines,
@@ -545,6 +576,22 @@ fn member_detail_lines_with_session(
member_routing_with_session(member, session_model),
);
// Capability badges for a pinned model, from the shared Fleet resolver
// (#5038). Unknown models omit the field rather than fabricating facts.
if let Some(model) = member
.profile
.model
.as_deref()
.map(str::trim)
.filter(|model| !model.is_empty())
&& let Some(badges) = crate::fleet::capability_badges::resolve_route_capability_badges(
member.profile.provider.as_deref(),
model,
)
{
detail_field(&mut lines, "Capabilities", badges.summary());
}
let delegation = &member.profile.delegation;
if delegation.max_spawn_depth.is_some() || delegation.max_concurrency.is_some() {
let mut bounds: Vec<String> = Vec::new();
@@ -605,6 +652,7 @@ mod tests {
fn operator() -> OperatorInfo {
OperatorInfo {
provider: "DeepSeek".to_string(),
provider_id: "deepseek".to_string(),
model: "deepseek-v4-pro".to_string(),
reasoning: "Auto".to_string(),
}
@@ -841,6 +889,8 @@ mod tests {
assert_eq!(member_routing(&pinned), "model glm-5.2 (pinned)");
}
include!("fleet_roster_capability_tests.rs");
#[test]
fn detail_lines_carry_overlay_source_for_project_members() {
let view = view_with_overrides();
@@ -0,0 +1,62 @@
/// #5038: a pinned model surfaces capability badges from the shared Fleet
/// resolver; members inheriting the session route omit the field, and an
/// unknown pinned model degrades to honest absence, never fabricated facts.
#[test]
fn detail_shows_capability_badges_for_pinned_models_only() {
fn detail_text(member: &AgentProfile) -> String {
member_detail_lines_with_session(member, None)
.iter()
.map(|line| {
line.spans
.iter()
.map(|span| span.content.clone().into_owned())
.collect::<String>()
})
.collect::<Vec<_>>()
.join("\n")
}
// Pinned known model (glm-5.2, no provider): registry facts.
let view = view_with_overrides();
let reviewer = view.members.iter().find(|m| m.id == "reviewer").unwrap();
let text = detail_text(reviewer);
assert!(text.contains("Capabilities"), "{text}");
assert!(text.contains("1M ctx"), "{text}");
assert!(text.contains("(registry)"), "{text}");
// Inheriting member: no pinned model, no capabilities field.
let scout = FleetRoster::built_ins_only().get("scout").unwrap().clone();
assert!(!detail_text(&scout).contains("Capabilities"));
// Unknown pinned model: the field is omitted rather than guessed.
let mut unknown = reviewer.clone();
unknown.profile.model = Some("totally-made-up-model-xyz".to_string());
let text = detail_text(&unknown);
assert!(!text.contains("Capabilities"), "{text}");
assert!(
text.contains("model totally-made-up-model-xyz (pinned)"),
"{text}"
);
}
/// #5038: the pinned operator row names its session model's capabilities.
#[test]
fn operator_detail_surfaces_session_model_capabilities() {
let text = operator_detail_lines(&operator())
.iter()
.map(|line| {
line.spans
.iter()
.map(|span| span.content.clone().into_owned())
.collect::<String>()
})
.collect::<Vec<_>>()
.join("\n");
assert!(text.contains("Capabilities"), "{text}");
assert!(text.contains("1M ctx"), "{text}");
assert!(text.contains("tools"), "{text}");
assert!(
text.contains("catalog"),
"operator should use provider-scoped catalog facts: {text}"
);
}
+59 -1
View File
@@ -559,13 +559,23 @@ impl FleetSetupView {
|| readiness.label().into_owned(),
|detail| format!("{}: {detail}", readiness.label()),
);
// Capability badges from the existing catalog/registry owners
// (#5038): shown in the word-wrapped detail pane so the picker
// list stays narrow-terminal friendly. Unknown models honestly
// omit the sentence instead of blocking selection.
let capability_note = crate::fleet::capability_badges::resolve_route_capability_badges(
Some(provider),
model,
)
.map(|badges| format!(" Capabilities: {}.", badges.summary()))
.unwrap_or_default();
model_choices.push(Choice {
label: Cow::Owned(model.clone()),
summary: Cow::Owned(format!(
"Pin this model ({provider_label}) · {readiness_summary}"
)),
description: Cow::Owned(format!(
"Route this worker to {model} on {provider_label} instead of inheriting the session route."
"Route this worker to {model} on {provider_label} instead of inheriting the session route.{capability_note}"
)),
});
// Canonical provider id (not the display label above) — this is
@@ -1667,6 +1677,54 @@ mod tests {
draft
}
/// #5038: the Model step's detail pane carries capability badges for
/// known catalog models and honestly omits them for unknown models, so
/// stale/absent data never blocks selection.
#[test]
fn model_step_detail_shows_capability_badges_for_known_models_only() {
let mut snap = snapshot();
snap.available_models.push((
"deepseek".to_string(),
"totally-made-up-model-xyz".to_string(),
crate::provider_readiness::ResolvedProviderReadiness::SavedUnchecked,
));
let view = FleetSetupView::from_snapshot(snap);
let known = view
.model_choices
.iter()
.find(|choice| choice.label == "deepseek-v4-pro")
.expect("known catalog model row");
assert!(
known.description.contains("Capabilities:"),
"{}",
known.description
);
assert!(
known.description.contains("1M ctx"),
"{}",
known.description
);
assert!(
known.description.contains("catalog"),
"catalog-backed rows must name catalog provenance: {}",
known.description
);
let unknown = view.model_choices.last().expect("appended unknown row");
assert_eq!(unknown.label, "totally-made-up-model-xyz");
assert!(
!unknown.description.contains("Capabilities:"),
"{}",
unknown.description
);
// The unknown row stays selectable; absence of data is not a block.
assert_eq!(
view.model_row_states.last(),
Some(&FleetModelRowState::Ready)
);
}
#[test]
fn provider_display_label_preserves_case_colliding_custom_ids() {
assert_eq!(provider_display_label("deepseek"), "DeepSeek");