mirror of
https://github.com/codewhale-hq/Codewhale.git
synced 2026-11-11 01:51:34 -10:00
fix(fleet): say why a sub-agent is queued and how much budget it has left (F5)
The rate-limit governor could shrink or pause launches with no visible trace: GovernorSnapshot was test-only, and a queued row only said "waiting for a launch slot" while its wall budget quietly ran down. - GovernorSnapshot is now a real surface with status_line(): "launch slots throttled to 4/8 after 2 provider rate limit(s) in the last 60s" or "launches paused after 4 ...", and None at full concurrency. - A queued child's reason carries that line plus its remaining wall budget: "... (29m of wall budget left; it keeps running while queued)". The recovery probe re-publishes the row when the governor state behind it changes. - Wall-clock choice: the budget keeps starting at spawn (shared with the permit wait so saturation cannot stretch a child past it, #6277); the row now shows the remainder instead of hiding it. Not done here (other lanes' files): the one-line governor header in /subagents (tui/views/mod.rs), the agent-runs payload (runtime_api.rs reads persisted records, which hold no governor state), and recording the wall-clock choice in docs/SUBAGENTS.md. No-Issue: 0.10.1 addendum F5 (fleet-4) Checks (targeted, local): - cargo test -p codewhale-tui --lib -- tools::subagent:: cancel_sub: 711 passed; 0 failed (tree held F4 + F5). - New: governor status_line_names_throttle_and_pause_only_when_active. - npm test / npm run check:web: not run (targeted-tests-only lane). Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
1 parent
84442f3058
commit
9e7e5e7f55
2 files changed
+95
-19
No files matched your search
@@ -462,10 +462,8 @@ impl RateLimitGovernor {
|
||||
state.paused
|
||||
}
|
||||
|
||||
/// Observability snapshot: `(gate capacity, window limit events, paused)`.
|
||||
/// (Unit-test/diagnostics surface; wired into status events by the parent
|
||||
/// repo follow-up.)
|
||||
#[cfg(test)]
|
||||
/// Observability snapshot: gate capacity, window limit events, paused.
|
||||
/// Feeds the fleet throttling line (addendum F5) and tests.
|
||||
pub(crate) fn snapshot(&self, now: Instant) -> GovernorSnapshot {
|
||||
let mut state = self.state.lock().expect("rate limit governor poisoned");
|
||||
Self::prune(&mut state, now);
|
||||
@@ -479,8 +477,7 @@ impl RateLimitGovernor {
|
||||
}
|
||||
}
|
||||
|
||||
/// Point-in-time view of the governor for tests and diagnostics.
|
||||
#[cfg(test)]
|
||||
/// Point-in-time view of the governor for status surfaces and tests.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub(crate) struct GovernorSnapshot {
|
||||
pub(crate) launch_capacity: usize,
|
||||
@@ -490,6 +487,27 @@ pub(crate) struct GovernorSnapshot {
|
||||
pub(crate) paused: bool,
|
||||
}
|
||||
|
||||
impl GovernorSnapshot {
|
||||
/// One line for the fleet header and queued rows (addendum F5), or `None`
|
||||
/// while launches run at the full configured concurrency.
|
||||
#[must_use]
|
||||
pub(crate) fn status_line(&self) -> Option<String> {
|
||||
let window = RATE_LIMIT_WINDOW.as_secs();
|
||||
if self.paused {
|
||||
return Some(format!(
|
||||
"launches paused after {} provider rate limit(s) in the last {window}s",
|
||||
self.window_limited
|
||||
));
|
||||
}
|
||||
(self.launch_capacity < self.max_capacity).then(|| {
|
||||
format!(
|
||||
"launch slots throttled to {}/{} after {} provider rate limit(s) in the last {window}s",
|
||||
self.launch_capacity, self.max_capacity, self.window_limited
|
||||
)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
@@ -540,6 +558,28 @@ mod tests {
|
||||
assert!(snap.paused);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn status_line_names_throttle_and_pause_only_when_active() {
|
||||
let (governor, _gate) = RateLimitGovernor::new(8);
|
||||
let t0 = Instant::now();
|
||||
assert_eq!(governor.snapshot(t0).status_line(), None);
|
||||
for i in 0..2 {
|
||||
governor.record_attempt(t0 + ms(i));
|
||||
governor.record_rate_limited(t0 + ms(i));
|
||||
}
|
||||
let throttled = governor
|
||||
.snapshot(t0 + ms(2))
|
||||
.status_line()
|
||||
.expect("throttled");
|
||||
assert!(throttled.contains("throttled to 4/8"), "{throttled}");
|
||||
for i in 2..4 {
|
||||
governor.record_attempt(t0 + ms(i));
|
||||
governor.record_rate_limited(t0 + ms(i));
|
||||
}
|
||||
let paused = governor.snapshot(t0 + ms(4)).status_line().expect("paused");
|
||||
assert!(paused.starts_with("launches paused"), "{paused}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ratio_threshold_triggers_decrease_even_with_few_events() {
|
||||
let (governor, _gate) = RateLimitGovernor::new(8);
|
||||
|
||||
@@ -11663,7 +11663,7 @@ async fn run_subagent_task_inner(mut task: SubAgentTask) {
|
||||
None => {
|
||||
match tokio::time::timeout_at(
|
||||
deadline.into(),
|
||||
acquire_queued_launch_permit(&task, Arc::clone(gate)),
|
||||
acquire_queued_launch_permit(&task, Arc::clone(gate), deadline),
|
||||
)
|
||||
.await
|
||||
{
|
||||
@@ -11801,24 +11801,53 @@ async fn run_subagent_task_inner(mut task: SubAgentTask) {
|
||||
}
|
||||
}
|
||||
|
||||
/// Queued-row reason (addendum F5): why the child waits — a free slot, or the
|
||||
/// rate-limit governor's pause/throttle — and how much of its wall budget is
|
||||
/// left. The wall clock starts at spawn and keeps running while queued (it is
|
||||
/// shared with the permit wait so saturation cannot stretch a child past its
|
||||
/// budget, #6277); the row says so instead of hiding it.
|
||||
fn queued_launch_reason(task: &SubAgentTask, deadline: Instant) -> String {
|
||||
let now = Instant::now();
|
||||
let governor_line = task
|
||||
.runtime
|
||||
.governor
|
||||
.as_ref()
|
||||
.and_then(|governor| governor.snapshot(now).status_line());
|
||||
let base = match governor_line {
|
||||
Some(line)
|
||||
if task
|
||||
.runtime
|
||||
.governor
|
||||
.as_ref()
|
||||
.is_some_and(|governor| governor.is_paused(now)) =>
|
||||
{
|
||||
format!("{SUBAGENT_QUEUED_RATE_LIMIT_REASON} — {line}")
|
||||
}
|
||||
Some(line) => format!("{SUBAGENT_QUEUED_LAUNCH_REASON} — {line}"),
|
||||
None => SUBAGENT_QUEUED_LAUNCH_REASON.to_string(),
|
||||
};
|
||||
let remaining = deadline.saturating_duration_since(now);
|
||||
format!(
|
||||
"{base} ({} of wall budget left; it keeps running while queued)",
|
||||
crate::elapsed::format_elapsed_secs(remaining.as_secs())
|
||||
)
|
||||
}
|
||||
|
||||
/// The part of a queued reason that changes with governor state, not time.
|
||||
fn queued_reason_cause(reason: &str) -> &str {
|
||||
reason.split(" (").next().unwrap_or(reason)
|
||||
}
|
||||
|
||||
async fn acquire_queued_launch_permit(
|
||||
task: &SubAgentTask,
|
||||
gate: Arc<governor::DynamicGate>,
|
||||
deadline: Instant,
|
||||
) -> Option<governor::DynamicGatePermit> {
|
||||
// When the governor has paused launches over sustained provider 429s,
|
||||
// surface the reason in the queued status instead of the generic
|
||||
// "waiting for a launch slot" message.
|
||||
let paused_for_rate_limit = task
|
||||
.runtime
|
||||
.governor
|
||||
.as_ref()
|
||||
.is_some_and(|governor| governor.is_paused(Instant::now()));
|
||||
let queued_reason = if paused_for_rate_limit {
|
||||
SUBAGENT_QUEUED_RATE_LIMIT_REASON
|
||||
} else {
|
||||
SUBAGENT_QUEUED_LAUNCH_REASON
|
||||
};
|
||||
record_queued_launch_progress(task, queued_reason).await;
|
||||
let mut queued_reason = queued_launch_reason(task, deadline);
|
||||
record_queued_launch_progress(task, &queued_reason).await;
|
||||
// While queued, periodically probe the governor: if a rate-limit pause
|
||||
// outlives its window (the in-flight fleet finished before any success
|
||||
// could lift the pause), the probe resumes launches instead of leaving
|
||||
@@ -11847,6 +11876,13 @@ async fn acquire_queued_launch_permit(
|
||||
if let Some(governor) = task.runtime.governor.as_ref() {
|
||||
governor.recover_if_window_drained(Instant::now());
|
||||
}
|
||||
// F5: re-publish when the governor state behind the queue
|
||||
// changed (paused, throttled, recovered).
|
||||
let reason = queued_launch_reason(task, deadline);
|
||||
if queued_reason_cause(&reason) != queued_reason_cause(&queued_reason) {
|
||||
record_queued_launch_progress(task, &reason).await;
|
||||
queued_reason = reason;
|
||||
}
|
||||
// If the probe lifted a pause it raised the gate capacity,
|
||||
// which grants queued waiters; the pinned `acquire_permit`
|
||||
// below observes the grant on the next poll.
|
||||
@@ -11858,7 +11894,7 @@ async fn acquire_queued_launch_permit(
|
||||
}
|
||||
}
|
||||
|
||||
async fn record_queued_launch_progress(task: &SubAgentTask, queued_reason: &'static str) {
|
||||
async fn record_queued_launch_progress(task: &SubAgentTask, queued_reason: &str) {
|
||||
{
|
||||
let mut manager = task.runtime.manager.write().await;
|
||||
manager.touch(&task.agent_id);
|
||||
|
||||
Reference in new issue
Block a user