wip: runtime audit slice follow-ups (compile fixes pending test run)

Checkpoint only; to be folded into the per-finding commits once green.

Co-Authored-By: Claude Opus 5.5 <[email protected]>
Signed-off-by: CodeWhale Bot <[email protected]>
This commit is contained in:
CodeWhale BotandClaude Opus 5.5 committed 2026-09-30 01:18:53 -07:00
1 parent f6c9f1081e
commit c875f69491
13 files changed
+252 -46

No files matched your search

+2 -3
View File
@@ -3779,8 +3779,7 @@ impl CodewhaleClient {
// shortened deadline) must release requests that are
// already waiting instead of stranding them for the whole
// original window.
while let Some(delay) = crate::retry_status::rate_limit_remaining(pause_scope)
{
while let Some(delay) = crate::retry_status::rate_limit_remaining(pause_scope) {
tokio::time::sleep(delay.min(RATE_LIMIT_PAUSE_RECHECK_INTERVAL)).await;
}
self.wait_for_rate_limit().await;
@@ -3848,7 +3847,7 @@ impl CodewhaleClient {
/// identity plus the host it reaches. A 429 from one provider pauses only
/// requests that would hit the same limit, never another provider, a
/// local runtime, or a sub-agent on a different route.
fn rate_limit_scope(&self) -> String {
pub(crate) fn rate_limit_scope(&self) -> String {
let route = if self.provider_identity.is_empty() {
self.api_provider.as_str()
} else {
+1 -1
View File
@@ -34,10 +34,10 @@ use crate::logging;
use crate::tools::schema_sanitize;
use codewhale_models::{ContentBlock, MessageRequest, MessageResponse, StreamEvent, Usage};
use super::CodewhaleClient;
use super::prepared::WireDialect;
use super::role_placement::{RolePlacement, role_placement};
use super::wire::{extract_sse_data_value, next_sse_line, push_sse_event_data};
use super::CodewhaleClient;
/// Maximum `cache_control` breakpoints Anthropic accepts per request.
const MAX_CACHE_BREAKPOINTS: usize = 4;
+5 -3
View File
@@ -279,9 +279,11 @@ async fn responses_stream_eof_without_a_terminal_event_is_an_error() {
"a truncated stream must not report MessageStop: {events:?}"
);
assert!(
events.last().is_some_and(|event| event
.as_ref()
.is_err_and(|error| error.to_string().contains("closed before response.completed"))),
events
.last()
.is_some_and(|event| event.as_ref().is_err_and(|error| error
.to_string()
.contains("closed before response.completed"))),
"{events:?}"
);
}
+9 -2
View File
@@ -3237,13 +3237,20 @@ mod tests {
.unwrap()
.join()
.unwrap();
assert!(!outcome, "the phase save must see the cancel and stand down");
assert!(
!outcome,
"the phase save must see the cancel and stand down"
);
assert_eq!(
store.load(&job.id).unwrap().status,
CloudJobStatus::Canceled
);
assert!(
!temp.path().join("jobs").join(format!("{}.json.tmp", job.id)).exists(),
!temp
.path()
.join("jobs")
.join(format!("{}.json.tmp", job.id))
.exists(),
"records are written through unique temporaries"
);
}
+25 -7
View File
@@ -11320,7 +11320,10 @@ fn run_sandbox_command(args: SandboxArgs) -> Result<()> {
}
}
let stderr = stderr_tail.lock().map(|tail| tail.clone()).unwrap_or_default();
let stderr = stderr_tail
.lock()
.map(|tail| tail.clone())
.unwrap_or_default();
let stderr_str = String::from_utf8_lossy(&stderr);
let exit_code = status.code().unwrap_or(-1);
let sandbox_type = exec_env.sandbox_type;
@@ -18877,14 +18880,28 @@ api_key = "test-only-key"
fn scorecard_rejects_a_threshold_that_cannot_gate() {
for bad in ["NaN", "inf", "-inf"] {
assert!(
Cli::try_parse_from(["codewhale", "scorecard", "--input", "t.json", "--threshold", bad])
.is_err(),
Cli::try_parse_from([
"codewhale",
"scorecard",
"--input",
"t.json",
"--threshold",
bad
])
.is_err(),
"--threshold {bad} must be refused"
);
}
assert!(
Cli::try_parse_from(["codewhale", "scorecard", "--input", "t.json", "--threshold", "2.5"])
.is_ok()
Cli::try_parse_from([
"codewhale",
"scorecard",
"--input",
"t.json",
"--threshold",
"2.5"
])
.is_ok()
);
}
@@ -20147,8 +20164,9 @@ mod project_config_tests {
..Config::default()
};
let error = merge_project_config_with_approval_baseline(&mut config, workspace.path(), None)
.expect_err("a symlinked primary project config must stop the launch");
let error =
merge_project_config_with_approval_baseline(&mut config, workspace.path(), None)
.expect_err("a symlinked primary project config must stop the launch");
assert!(error.to_string().contains("--no-project-config"), "{error}");
assert_eq!(
+1 -4
View File
@@ -1954,10 +1954,7 @@ async fn plugin_stdio_does_not_surface_reviewed_child_stderr() {
async fn stdio_shutdown_also_terminates_the_servers_grandchildren() {
let mut config = test_server_config();
config.command = Some("sh".to_string());
config.args = vec![
"-c".to_string(),
"sleep 300 & echo $!; wait".to_string(),
];
config.args = vec!["-c".to_string(), "sleep 300 & echo $!; wait".to_string()];
let mut transport = StdioTransport::spawn(
"grandparent",
"sh",
+52 -14
View File
@@ -64,6 +64,9 @@ pub(crate) enum AutoRouterSetupIssue {
Incomplete,
/// The route is complete but its credential is missing.
MissingKey,
/// `thinking` is not a reasoning tier; the classifier call would carry
/// an effort the provider rejects or silently reinterprets.
InvalidThinking,
}
impl AutoRouterSetupIssue {
@@ -76,6 +79,9 @@ impl AutoRouterSetupIssue {
}
Self::Incomplete => "[auto.router] needs a known provider and a model",
Self::MissingKey => "no API key for the router route",
Self::InvalidThinking => {
"[auto.router] thinking must be auto, off, minimal, low, medium, high, xhigh, ultra, or max"
}
}
}
}
@@ -322,20 +328,28 @@ impl ModelInventory {
router_setup_issue = Some(AutoRouterSetupIssue::UnknownKind);
None
}
Some(AutoRouterKind::Chat) => router_provider_setting
.and_then(ApiProvider::parse)
.zip(router_model_setting)
.map(|(provider, model)| {
(
provider,
model.to_string(),
router_table
.and_then(|router| router.thinking.as_deref())
.map(str::trim)
.filter(|t| !t.is_empty())
.map(str::to_string),
)
}),
Some(AutoRouterKind::Chat) => {
let thinking = router_table
.and_then(|router| router.thinking.as_deref())
.map(str::trim)
.filter(|t| !t.is_empty());
// A typo here would otherwise ride every classifier request
// as an effort string the provider rejects, failing Auto back
// to the local fallback on every turn.
if thinking.is_some_and(|thinking| {
crate::reasoning_preference::ReasoningEffort::parse_strict(thinking).is_err()
}) {
router_setup_issue = Some(AutoRouterSetupIssue::InvalidThinking);
None
} else {
router_provider_setting
.and_then(ApiProvider::parse)
.zip(router_model_setting)
.map(|(provider, model)| {
(provider, model.to_string(), thinking.map(str::to_string))
})
}
}
Some(AutoRouterKind::Decision) => {
match router_provider_setting.map(DecisionRouterRoute::parse) {
Some(None) => {
@@ -1507,6 +1521,30 @@ mod decision_router_inventory_tests {
);
}
#[test]
fn a_chat_router_with_an_unknown_thinking_tier_is_not_configured() {
let _env = hermetic();
let chat = |thinking: &str| crate::config::AutoRouterConfig {
kind: Some("chat".to_string()),
provider: Some("openrouter".to_string()),
model: Some("openai/gpt-5-mini".to_string()),
thinking: Some(thinking.to_string()),
..Default::default()
};
let typo = ModelInventory::from_config(&with_router(chat("hgih"), true));
assert!(!typo.router_configured);
assert!(!typo.router_available);
assert_eq!(
typo.router_setup_issue,
Some(AutoRouterSetupIssue::InvalidThinking)
);
let valid = ModelInventory::from_config(&with_router(chat("low"), true));
assert!(valid.router_available);
assert_eq!(valid.router_setup_issue, None);
assert_eq!(valid.router_thinking.as_deref(), Some("low"));
}
#[test]
fn a_router_table_with_only_tuning_keys_is_declared_and_incomplete() {
let _env = hermetic();
+2 -1
View File
@@ -891,7 +891,8 @@ mod tests {
let dir = tempfile::tempdir().expect("tempdir");
let path = dir.path().join("huge.json");
let file = std::fs::File::create(&path).expect("create");
file.set_len(MAX_CATALOG_BYTES as u64 + 1).expect("sparse size");
file.set_len(MAX_CATALOG_BYTES as u64 + 1)
.expect("sparse size");
let rt = tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
+7 -4
View File
@@ -1296,7 +1296,11 @@ pub fn bundled_catalog_offering_for_model(
if needle.is_empty() {
return None;
}
find_wire_model(bundled_snapshot().offerings_for_provider(catalog_id), needle).cloned()
find_wire_model(
bundled_snapshot().offerings_for_provider(catalog_id),
needle,
)
.cloned()
}
/// Count of merged-catalog models for one provider (catalog view / dashboard).
@@ -1908,9 +1912,8 @@ mod tests {
..Default::default()
};
let rows = vec![row("org/Model-A"), row("org/model-a"), row("org/Model-B")];
let found = |needle: &str| {
find_wire_model(&rows, needle).map(|row| row.wire_model_id.clone())
};
let found =
|needle: &str| find_wire_model(&rows, needle).map(|row| row.wire_model_id.clone());
assert_eq!(found("org/model-a").as_deref(), Some("org/model-a"));
assert_eq!(found("org/Model-A").as_deref(), Some("org/Model-A"));
assert_eq!(found("ORG/MODEL-A"), None, "two distinct ids fold together");
+96
View File
@@ -3290,6 +3290,9 @@ async fn relay_worker(
sync_tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
let mut runtime_retry_delay = RUNTIME_UPLOAD_RETRY_INTERVAL;
let mut runtime_retry_not_before = Instant::now();
// One credential refresh per rejection streak (see
// `RUNTIME_REFRESHED_CREDENTIAL_REFUSED`).
let mut runtime_refreshed_since_accept = false;
loop {
tokio::select! {
@@ -3412,11 +3415,15 @@ async fn relay_worker(
RuntimeFlushOutcome::Accepted { run_id, cursor } => {
runtime_retry_delay = RUNTIME_UPLOAD_RETRY_INTERVAL;
runtime_retry_not_before = Instant::now();
runtime_refreshed_since_accept = false;
event_tx
.send(RemoteEvent::RuntimeCursor { run_id, cursor })
.map_err(|_| "The terminal remote-control owner stopped.".to_string())?;
}
RuntimeFlushOutcome::AccessTokenExpired => {
if std::mem::replace(&mut runtime_refreshed_since_accept, true) {
return Err(RUNTIME_REFRESHED_CREDENTIAL_REFUSED.to_string());
}
refresh_enrollment_and_reconnect(
&client,
&mut enrollment,
@@ -3694,6 +3701,12 @@ impl RuntimeTransportOutbox {
}
}
/// A runtime event refused (401/403) again right after a successful
/// credential refresh is a terminal rejection, not an expired token: a
/// revoked runner or a run this runner may not write. Refreshing again would
/// loop at the upload retry interval forever.
const RUNTIME_REFRESHED_CREDENTIAL_REFUSED: &str = "The remote-control server refused runtime events again with a freshly refreshed credential; stopping instead of retrying.";
/// Flushes every queued runtime envelope through the server-confirmed cursor
/// before a stop may be acknowledged. Emits `RuntimeCursor` events so the
/// controller compacts its journal as acknowledgements land. Failing to drain
@@ -3710,6 +3723,7 @@ async fn drain_runtime_outbox_for_stop(
deadline: Instant,
) -> Result<(), String> {
let mut delay = RUNTIME_UPLOAD_RETRY_INTERVAL;
let mut refreshed_since_accept = false;
while !outbox.events.is_empty() {
if Instant::now() >= deadline {
return Err(
@@ -3721,6 +3735,7 @@ async fn drain_runtime_outbox_for_stop(
RuntimeFlushOutcome::Idle => break,
RuntimeFlushOutcome::Accepted { run_id, cursor } => {
delay = RUNTIME_UPLOAD_RETRY_INTERVAL;
refreshed_since_accept = false;
let _ = event_tx.send(RemoteEvent::RuntimeCursor { run_id, cursor });
}
RuntimeFlushOutcome::Retryable => {
@@ -3728,6 +3743,9 @@ async fn drain_runtime_outbox_for_stop(
delay = delay.saturating_mul(2).min(RUNTIME_UPLOAD_MAX_BACKOFF);
}
RuntimeFlushOutcome::AccessTokenExpired => {
if std::mem::replace(&mut refreshed_since_accept, true) {
return Err(RUNTIME_REFRESHED_CREDENTIAL_REFUSED.to_string());
}
refresh_enrollment_and_reconnect(client, enrollment, runner_id, start, event_tx)
.await?;
}
@@ -7938,6 +7956,84 @@ mod tests {
assert_eq!(bodies[0], bodies[1], "retries must be byte-identical");
}
#[tokio::test]
async fn stop_drain_stops_when_a_fresh_credential_is_refused_again() {
let _env = crate::test_support::lock_test_env();
let secrets_root = tempfile::tempdir().expect("isolated remote-control secrets");
let _codewhale_home =
crate::test_support::EnvVarGuard::set("CODEWHALE_HOME", secrets_root.path());
let server = MockServer::start().await;
// A terminal refusal: the runner may not write this run, and a new
// access token does not change that.
Mock::given(method("POST"))
.and(path(
"/api/local-runners/runner_fixture/runs/run_fixture/events",
))
.respond_with(ResponseTemplate::new(403))
.expect(2)
.mount(&server)
.await;
let access_token = crate::test_support::future_test_jwt(&"a".repeat(40));
Mock::given(method("POST"))
.and(path("/api/runner/enrollments/token"))
.respond_with(ResponseTemplate::new(200).set_body_json(json!({
"enrollment": {
"id": "enrollment_fixture",
"userId": "account_fixture",
"deviceId": "device_fixture",
"runtimeVersion": "0.9.6",
"runtimeCommit": "a".repeat(40),
"capabilities": CAPABILITIES,
},
"credential": { "accessToken": access_token },
})))
.expect(1)
.mount(&server)
.await;
Mock::given(method("POST"))
.and(path("/api/local-runners/connect"))
.respond_with(ResponseTemplate::new(200).set_body_json(fixture_connection_response()))
.mount(&server)
.await;
let mut enrollment = fixture_enrollment(&format!("{}/", server.uri()));
let mut runner_id = "runner_fixture".to_string();
let client = crate::tls::reqwest_client_builder()
.redirect(reqwest::redirect::Policy::none())
.build()
.expect("fixture client");
let (event_tx, _event_rx) = mpsc::unbounded_channel();
let mut outbox = RuntimeTransportOutbox::default();
outbox
.enqueue(
"run_fixture",
runtime_envelope(
1,
"turn.completed",
Some("turn_fixture"),
"2026-08-08T12:00:00Z".to_string(),
json!({ "turn": { "status": "completed", "usage": {} } }),
),
)
.expect("queue terminal envelope");
let error = drain_runtime_outbox_for_stop(
&client,
&mut enrollment,
&mut runner_id,
&fixture_start(),
&event_tx,
&mut outbox,
Instant::now() + Duration::from_secs(5),
)
.await
.expect_err("a refusal that survives a refresh must end the drain");
assert_eq!(error, RUNTIME_REFRESHED_CREDENTIAL_REFUSED);
assert!(
!outbox.events.is_empty(),
"the refused envelope stays queued for a later, authorized resend"
);
}
#[tokio::test]
async fn stop_drain_deadline_failure_refuses_to_confirm_stop() {
let server = MockServer::start().await;
+7 -2
View File
@@ -570,12 +570,17 @@ mod tests {
};
// `open-port` has no source restriction: it admits the whole internet.
assert!(
steps.iter().all(|s| !s.args.iter().any(|a| a == "open-port")),
steps
.iter()
.all(|s| !s.args.iter().any(|a| a == "open-port")),
"Azure plan must not open a port to every source"
);
let create = steps
.iter()
.find(|s| s.args.starts_with(&["vm".to_string(), "create".to_string()]))
.find(|s| {
s.args
.starts_with(&["vm".to_string(), "create".to_string()])
})
.expect("vm create step");
assert_eq!(arg_after(create, "--nsg-rule").as_deref(), Some("NONE"));
let ssh_rules: Vec<_> = steps
+3 -1
View File
@@ -11277,7 +11277,9 @@ async fn spawn_subagent_from_input(
)));
}
if let Some(remaining) = crate::retry_status::rate_limit_remaining() {
if let Some(remaining) =
crate::retry_status::rate_limit_remaining(&runtime.client.rate_limit_scope())
{
let seconds = remaining.as_secs() + u64::from(remaining.subsec_nanos() > 0);
return Err(ToolError::execution_failed(format!(
"Provider is rate-limiting; sub-agent spawning is paused for {seconds}s. \
+42 -4
View File
@@ -12605,11 +12605,14 @@ async fn rate_limit_pause_blocks_subagent_spawn() {
let _clear = ClearRateLimitOnDrop;
crate::retry_status::clear();
crate::retry_status::clear_rate_limit();
crate::retry_status::note_rate_limit(Duration::from_secs(30));
let tmp = tempdir().expect("tempdir");
let mut runtime = stub_runtime();
runtime.context = ToolContext::new(tmp.path().to_path_buf());
crate::retry_status::note_rate_limit(
&runtime.client.rate_limit_scope(),
Duration::from_secs(30),
);
let manager = new_shared_subagent_manager(tmp.path().to_path_buf(), 2);
let err = spawn_subagent_from_input(
@@ -18453,6 +18456,32 @@ async fn spawn_budget_capped_worker(
String,
Arc<AtomicUsize>,
tokio::task::JoinHandle<()>,
) {
spawn_budget_capped_worker_with_pause(
workspace,
prompt_tokens,
completion_tokens,
max_steps,
wall_time,
None,
)
.await
}
/// As [`spawn_budget_capped_worker`], optionally opening a rate-limit pause
/// on the worker's own route before it starts.
async fn spawn_budget_capped_worker_with_pause(
workspace: &Path,
prompt_tokens: u64,
completion_tokens: u64,
max_steps: u32,
wall_time: Duration,
pause: Option<Duration>,
) -> (
Arc<RwLock<SubAgentManager>>,
String,
Arc<AtomicUsize>,
tokio::task::JoinHandle<()>,
) {
let manager = Arc::new(RwLock::new(SubAgentManager::new(
workspace.to_path_buf(),
@@ -18502,6 +18531,9 @@ async fn spawn_budget_capped_worker(
launch_gate: None,
_foreground_child_registration: None,
};
if let Some(pause) = pause {
crate::retry_status::note_rate_limit(&runtime.client.rate_limit_scope(), pause);
}
let task_handle = tokio::spawn(run_subagent_task(task));
(manager, agent_id, calls, task_handle)
}
@@ -18781,11 +18813,17 @@ async fn worker_is_not_stranded_by_transient_global_rate_limit_window() {
// in-flight requests promptly.
let _guard = crate::retry_status::test_guard();
let _clear = ClearRateLimitOnDrop;
crate::retry_status::note_rate_limit(Duration::from_secs(30));
let tmp = tempdir().expect("tempdir");
let (manager, agent_id, _calls, task_handle) =
spawn_budget_capped_worker(tmp.path(), 60, 40, 4, DEFAULT_CHILD_WALL_TIME).await;
let (manager, agent_id, _calls, task_handle) = spawn_budget_capped_worker_with_pause(
tmp.path(),
60,
40,
4,
DEFAULT_CHILD_WALL_TIME,
Some(Duration::from_secs(30)),
)
.await;
// Simulate the concurrent test finishing: the window closes shortly
// after the worker's first request has already observed it.