fix(fleet): say why a sub-agent is queued and how much budget it has left (F5)

The rate-limit governor could shrink or pause launches with no visible
trace: GovernorSnapshot was test-only, and a queued row only said
"waiting for a launch slot" while its wall budget quietly ran down.

- GovernorSnapshot is now a real surface with status_line(): "launch
  slots throttled to 4/8 after 2 provider rate limit(s) in the last
  60s" or "launches paused after 4 ...", and None at full concurrency.
- A queued child's reason carries that line plus its remaining wall
  budget: "... (29m of wall budget left; it keeps running while
  queued)". The recovery probe re-publishes the row when the governor
  state behind it changes.
- Wall-clock choice: the budget keeps starting at spawn (shared with
  the permit wait so saturation cannot stretch a child past it, #6277);
  the row now shows the remainder instead of hiding it.

Not done here (other lanes' files): the one-line governor header in
/subagents (tui/views/mod.rs), the agent-runs payload (runtime_api.rs
reads persisted records, which hold no governor state), and recording
the wall-clock choice in docs/SUBAGENTS.md.

No-Issue: 0.10.1 addendum F5 (fleet-4)

Checks (targeted, local):
- cargo test -p codewhale-tui --lib -- tools::subagent:: cancel_sub:
  711 passed; 0 failed (tree held F4 + F5).
- New: governor status_line_names_throttle_and_pause_only_when_active.
- npm test / npm run check:web: not run (targeted-tests-only lane).

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
CodeWhale BotandClaude Opus 5.5 committed 2026-09-22 17:18:57 -07:00
1 parent 84442f3058
commit 9e7e5e7f55
2 files changed
+95 -19

No files matched your search

+46 -6
View File
@@ -462,10 +462,8 @@ impl RateLimitGovernor {
state.paused
}
/// Observability snapshot: `(gate capacity, window limit events, paused)`.
/// (Unit-test/diagnostics surface; wired into status events by the parent
/// repo follow-up.)
#[cfg(test)]
/// Observability snapshot: gate capacity, window limit events, paused.
/// Feeds the fleet throttling line (addendum F5) and tests.
pub(crate) fn snapshot(&self, now: Instant) -> GovernorSnapshot {
let mut state = self.state.lock().expect("rate limit governor poisoned");
Self::prune(&mut state, now);
@@ -479,8 +477,7 @@ impl RateLimitGovernor {
}
}
/// Point-in-time view of the governor for tests and diagnostics.
#[cfg(test)]
/// Point-in-time view of the governor for status surfaces and tests.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) struct GovernorSnapshot {
pub(crate) launch_capacity: usize,
@@ -490,6 +487,27 @@ pub(crate) struct GovernorSnapshot {
pub(crate) paused: bool,
}
impl GovernorSnapshot {
/// One line for the fleet header and queued rows (addendum F5), or `None`
/// while launches run at the full configured concurrency.
#[must_use]
pub(crate) fn status_line(&self) -> Option<String> {
let window = RATE_LIMIT_WINDOW.as_secs();
if self.paused {
return Some(format!(
"launches paused after {} provider rate limit(s) in the last {window}s",
self.window_limited
));
}
(self.launch_capacity < self.max_capacity).then(|| {
format!(
"launch slots throttled to {}/{} after {} provider rate limit(s) in the last {window}s",
self.launch_capacity, self.max_capacity, self.window_limited
)
})
}
}
#[cfg(test)]
mod tests {
use super::*;
@@ -540,6 +558,28 @@ mod tests {
assert!(snap.paused);
}
#[test]
fn status_line_names_throttle_and_pause_only_when_active() {
let (governor, _gate) = RateLimitGovernor::new(8);
let t0 = Instant::now();
assert_eq!(governor.snapshot(t0).status_line(), None);
for i in 0..2 {
governor.record_attempt(t0 + ms(i));
governor.record_rate_limited(t0 + ms(i));
}
let throttled = governor
.snapshot(t0 + ms(2))
.status_line()
.expect("throttled");
assert!(throttled.contains("throttled to 4/8"), "{throttled}");
for i in 2..4 {
governor.record_attempt(t0 + ms(i));
governor.record_rate_limited(t0 + ms(i));
}
let paused = governor.snapshot(t0 + ms(4)).status_line().expect("paused");
assert!(paused.starts_with("launches paused"), "{paused}");
}
#[test]
fn ratio_threshold_triggers_decrease_even_with_few_events() {
let (governor, _gate) = RateLimitGovernor::new(8);
+49 -13
View File
@@ -11663,7 +11663,7 @@ async fn run_subagent_task_inner(mut task: SubAgentTask) {
None => {
match tokio::time::timeout_at(
deadline.into(),
acquire_queued_launch_permit(&task, Arc::clone(gate)),
acquire_queued_launch_permit(&task, Arc::clone(gate), deadline),
)
.await
{
@@ -11801,24 +11801,53 @@ async fn run_subagent_task_inner(mut task: SubAgentTask) {
}
}
/// Queued-row reason (addendum F5): why the child waits — a free slot, or the
/// rate-limit governor's pause/throttle — and how much of its wall budget is
/// left. The wall clock starts at spawn and keeps running while queued (it is
/// shared with the permit wait so saturation cannot stretch a child past its
/// budget, #6277); the row says so instead of hiding it.
fn queued_launch_reason(task: &SubAgentTask, deadline: Instant) -> String {
let now = Instant::now();
let governor_line = task
.runtime
.governor
.as_ref()
.and_then(|governor| governor.snapshot(now).status_line());
let base = match governor_line {
Some(line)
if task
.runtime
.governor
.as_ref()
.is_some_and(|governor| governor.is_paused(now)) =>
{
format!("{SUBAGENT_QUEUED_RATE_LIMIT_REASON} — {line}")
}
Some(line) => format!("{SUBAGENT_QUEUED_LAUNCH_REASON} — {line}"),
None => SUBAGENT_QUEUED_LAUNCH_REASON.to_string(),
};
let remaining = deadline.saturating_duration_since(now);
format!(
"{base} ({} of wall budget left; it keeps running while queued)",
crate::elapsed::format_elapsed_secs(remaining.as_secs())
)
}
/// The part of a queued reason that changes with governor state, not time.
fn queued_reason_cause(reason: &str) -> &str {
reason.split(" (").next().unwrap_or(reason)
}
async fn acquire_queued_launch_permit(
task: &SubAgentTask,
gate: Arc<governor::DynamicGate>,
deadline: Instant,
) -> Option<governor::DynamicGatePermit> {
// When the governor has paused launches over sustained provider 429s,
// surface the reason in the queued status instead of the generic
// "waiting for a launch slot" message.
let paused_for_rate_limit = task
.runtime
.governor
.as_ref()
.is_some_and(|governor| governor.is_paused(Instant::now()));
let queued_reason = if paused_for_rate_limit {
SUBAGENT_QUEUED_RATE_LIMIT_REASON
} else {
SUBAGENT_QUEUED_LAUNCH_REASON
};
record_queued_launch_progress(task, queued_reason).await;
let mut queued_reason = queued_launch_reason(task, deadline);
record_queued_launch_progress(task, &queued_reason).await;
// While queued, periodically probe the governor: if a rate-limit pause
// outlives its window (the in-flight fleet finished before any success
// could lift the pause), the probe resumes launches instead of leaving
@@ -11847,6 +11876,13 @@ async fn acquire_queued_launch_permit(
if let Some(governor) = task.runtime.governor.as_ref() {
governor.recover_if_window_drained(Instant::now());
}
// F5: re-publish when the governor state behind the queue
// changed (paused, throttled, recovered).
let reason = queued_launch_reason(task, deadline);
if queued_reason_cause(&reason) != queued_reason_cause(&queued_reason) {
record_queued_launch_progress(task, &reason).await;
queued_reason = reason;
}
// If the probe lifted a pause it raised the gate capacity,
// which grants queued waiters; the pinned `acquire_permit`
// below observes the grant on the next poll.
@@ -11858,7 +11894,7 @@ async fn acquire_queued_launch_permit(
}
}
async fn record_queued_launch_progress(task: &SubAgentTask, queued_reason: &'static str) {
async fn record_queued_launch_progress(task: &SubAgentTask, queued_reason: &str) {
{
let mut manager = task.runtime.manager.write().await;
manager.touch(&task.agent_id);