Merge main: #6870 #6863 #6854 #6855 #6856 into 0.10.1

Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
CodeWhale BotandClaude Opus 5.5 committed 2026-10-05 10:36:30 -07:00
commit ed26af95fe
14 files changed
+718 -92

No files matched your search

+43 -1
View File
@@ -28,6 +28,21 @@ use serde_json::Value;
use super::AppState;
// ── Upstream deadlines ─────────────────────────────────────────────────
/// Connect budget for the upstream forward. Matches the 10s connect bound
/// used by the TUI client's non-streaming requests (vision, the shared
/// retry client).
const UPSTREAM_CONNECT_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10);
/// Total budget for one upstream forward, connect through body end. The
/// handler rejects streaming (`stream: true`) and reads the full upstream
/// body, so without a client-level total a provider that accepts the
/// connection and stalls — or trickles the body — wedges this handler (and
/// the caller's connection) indefinitely. 1800s mirrors the TUI client's
/// non-streaming envelope for the same request class.
const UPSTREAM_TOTAL_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(1800);
// ── Resolved endpoint ──────────────────────────────────────────────────
/// Everything needed to forward a single chat-completions request upstream.
@@ -397,8 +412,12 @@ pub(crate) async fn chat_completions_handler(
.into_response();
}
// Build upstream request.
// Build upstream request. The shared platform builder sets no timeouts,
// so the proxy would hang forever on an accept-and-stall upstream;
// bound both the connect and the whole non-streaming round trip.
let upstream_req = codewhale_release::platform_http_client_builder()
.connect_timeout(UPSTREAM_CONNECT_TIMEOUT)
.timeout(UPSTREAM_TOTAL_TIMEOUT)
.build()
.map_err(|e| {
(
@@ -498,6 +517,29 @@ mod tests {
crate::install_test_crypto_provider();
}
// The proxy forwards with a client built per request, and reqwest gives
// no accessor for a built client's budgets, so a behavioral test would
// need an accept-and-stall upstream and a multi-second (connect) or
// half-hour (total) wait. Pin the values instead: if the handler stops
// applying them this test cannot see it, but a silent constant change
// or an inversion of the connect/total ordering cannot slip through.
#[test]
fn upstream_deadlines_are_bounded_and_ordered() {
assert_eq!(UPSTREAM_CONNECT_TIMEOUT, std::time::Duration::from_secs(10));
assert_eq!(UPSTREAM_TOTAL_TIMEOUT, std::time::Duration::from_secs(1800));
assert!(
UPSTREAM_TOTAL_TIMEOUT > UPSTREAM_CONNECT_TIMEOUT,
"the total forward budget must leave room beyond the connect budget"
);
// The shared platform builder must accept both bounds; the handler
// chains them onto this builder.
codewhale_release::platform_http_client_builder()
.connect_timeout(UPSTREAM_CONNECT_TIMEOUT)
.timeout(UPSTREAM_TOTAL_TIMEOUT)
.build()
.expect("platform builder accepts the proxy deadlines");
}
/// Start a minimal upstream mock server that echoes back what it received.
async fn start_mock_upstream() -> (String, tokio::task::JoinHandle<()>) {
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
+151 -14
View File
@@ -222,10 +222,13 @@ pub fn effective_muse_wire_id(model: &str) -> &str {
#[must_use]
pub fn model_is_openai_reasoning_family(model: &str) -> bool {
let lower = model.to_ascii_lowercase();
codewhale_config::catalog::reviewed::bundled_reviewed()
.openai_reasoning_ids
.contains_key(&lower)
|| reviewed_snapshot_model(&lower).is_some()
let reviewed = codewhale_config::catalog::reviewed::bundled_reviewed();
reviewed.openai_reasoning_ids.contains_key(&lower)
// Snapshot resolution spans every reviewed family; the resolved
// target must itself be an OpenAI reasoning row, or a DeepSeek
// snapshot would be relabelled as one.
|| reviewed_snapshot_model(&lower)
.is_some_and(|target| reviewed.openai_reasoning_ids.contains_key(target))
}
pub fn is_openai_gpt_56_api_model(model_lower: &str) -> bool {
@@ -249,6 +252,102 @@ pub fn has_date_snapshot_suffix(model_lower: &str, prefix: &str) -> bool {
.all(|(idx, byte)| idx == 4 || idx == 7 || byte.is_ascii_digit())
}
/// Resolve a snapshot or variant model id to the reviewed intrinsic row it
/// denotes, without inventing facts for names the catalog does not own.
///
/// Two documented contracts, applied in order:
/// - the authored `snapshot_prefixes` rows (`gpt-5.5-<YYYY-MM-DD>`);
/// - a trailing date stamp (`-YYYY-MM-DD` or `-MMDD`) or an explicit variant
/// marker (`-vision-exp`, `-vision`, `-exp`), accepted only when stripping
/// it leaves the exact id of a reviewed intrinsic row
/// (`deepseek-v4-pro-0813` denotes `deepseek-v4-pro`;
/// `deepseek-v4-flash-vision` denotes `deepseek-v4-flash`).
/// The compact `-YYYYMMDD` shape is deliberately not inferred: the
/// sibling-metadata contract rejects it.
///
/// Every layer is resolved against the bundled reviewed catalog only. An
/// unrecognized name still returns `None`: the unknown stays observable.
fn reviewed_snapshot_model(model: &str) -> Option<&'static str> {
let reviewed = codewhale_config::catalog::reviewed::bundled_reviewed();
let via_authored_contract = |candidate: &str| {
reviewed
.snapshot_prefixes
.iter()
.find_map(|(prefix, target)| {
has_date_snapshot_suffix(candidate, prefix).then_some(target.as_str())
})
};
let mut candidate = model.to_ascii_lowercase();
for _ in 0..3 {
if let Some(target) = via_authored_contract(&candidate) {
return Some(target);
}
let stripped = strip_snapshot_or_variant_suffix(&candidate)?;
if let Some((key, _)) = reviewed.intrinsic.get_key_value(&stripped) {
return Some(key.as_str());
}
candidate = stripped;
}
None
}
/// One snapshot/variant normalization layer, longest marker first. The
/// caller re-resolves the shortened id against the reviewed catalog and
/// never accepts a shortening on its own.
fn strip_snapshot_or_variant_suffix(id: &str) -> Option<String> {
for marker in ["-vision-exp", "-vision", "-exp"] {
if let Some(base) = id.strip_suffix(marker)
&& !base.is_empty()
{
return Some(base.to_string());
}
}
strip_date_stamp(id)
}
/// Strip one trailing date stamp: `-YYYY-MM-DD` or `-MMDD` (the reviewed
/// snapshot conventions). The compact `-YYYYMMDD` form is deliberately not
/// inferred: the sibling-metadata contract rejects that shape (see
/// `unrecognized_deepseek_models_do_not_inherit_sibling_metadata`).
fn strip_date_stamp(id: &str) -> Option<String> {
// `-YYYY-MM-DD`: a ten-character tail preceded by its own dash. The split
// index is a byte offset, so it must land on a char boundary: a multi-byte
// model id would panic in `split_at` otherwise.
if id.len() > 11 && id.is_char_boundary(id.len() - 10) {
let (head, tail) = id.split_at(id.len() - 10);
if let Some(head) = head.strip_suffix('-')
&& !head.is_empty()
&& has_date_snapshot_suffix(tail, "")
{
return Some(head.to_string());
}
}
let (head, tail) = id.rsplit_once('-')?;
if head.is_empty() || tail.is_empty() {
return None;
}
let digits = tail.as_bytes();
if digits.len() == 4
&& digits.iter().all(u8::is_ascii_digit)
&& valid_month_day(&digits[0..2], &digits[2..4])
{
return Some(head.to_string());
}
None
}
/// Validate a two-digit month/day pair (ranges only, no calendar math).
fn valid_month_day(month_digits: &[u8], day_digits: &[u8]) -> bool {
let two = |bytes: &[u8]| -> Option<u32> {
(bytes.len() == 2 && bytes.iter().all(u8::is_ascii_digit))
.then(|| u32::from(bytes[0] - b'0') * 10 + u32::from(bytes[1] - b'0'))
};
match (two(month_digits), two(day_digits)) {
(Some(month), Some(day)) => (1..=12).contains(&month) && (1..=31).contains(&day),
_ => false,
}
}
/// The context window a model name's `_Nk` suffix advertises, when the
/// catalog does not already describe the model (#5441).
///
@@ -257,16 +356,6 @@ pub fn has_date_snapshot_suffix(model_lower: &str, prefix: &str) -> bool {
/// from the name* — a naming convention the serving engine may ignore is not
/// a fact about the route, and every surface that shows such a window must
/// mark it unverified.
fn reviewed_snapshot_model(model: &str) -> Option<&'static str> {
let lower = model.to_ascii_lowercase();
codewhale_config::catalog::reviewed::bundled_reviewed()
.snapshot_prefixes
.iter()
.find_map(|(prefix, target)| {
has_date_snapshot_suffix(&lower, prefix).then_some(target.as_str())
})
}
#[must_use]
pub fn name_suffix_context_window_hint(model: &str) -> Option<u32> {
if codewhale_config::catalog::reviewed::intrinsic_model(model)
@@ -1179,6 +1268,54 @@ mod tests {
assert_eq!(context_window_for_model("deepseek-v3.2-2k-preview"), None);
}
/// 2026-10-05: custom gateways serve V4 under snapshot and variant ids
/// the reviewed catalog does not enumerate (`deepseek-v4-pro-0813`,
/// `deepseek-v4-flash-vision`). They denote the same rows as their base
/// ids and must inherit the base facts instead of falling back to the
/// 128K unknown shape.
#[test]
fn deepseek_v4_snapshot_and_variant_ids_inherit_reviewed_facts() {
for model in [
"deepseek-v4-pro-0813",
"DeepSeek-V4-Pro-0813",
"deepseek-v4-pro-2025-08-13",
"deepseek-v4-flash-vision",
"deepseek-v4-flash-vision-0813",
] {
assert_eq!(context_window_for_model(model), Some(1_000_000), "{model}");
assert_eq!(max_output_tokens_for_model(model), Some(384_000), "{model}");
assert!(model_supports_reasoning(model), "{model}");
}
// A trailing number that is not a recognized date stamp, an
// eight-digit compact stamp, or an unknown base, stays unknown: the
// resolver never invents a row.
for model in [
"not-a-model-0813",
"deepseek-v4-pro-9913",
"deepseek-v4-pro-08130",
"deepseek-v4-pro-x0813",
"deepseek-v4-pro-20250813",
] {
assert_eq!(context_window_for_model(model), None, "{model}");
assert_eq!(model_reasoning_capability(model), None, "{model}");
}
// Snapshot resolution must not relabel a DeepSeek row as an OpenAI
// reasoning model, while the OpenAI snapshot contract keeps working.
assert!(!model_is_openai_reasoning_family("deepseek-v4-pro-0813"));
assert!(model_is_openai_reasoning_family("gpt-5.5-2026-06-01"));
}
/// Multi-byte model ids must not panic the byte-indexed date scan: the
/// `-YYYY-MM-DD` layer splits at `len - 10`, which need not be a UTF-8
/// char boundary (review finding from the PR #6 review).
#[test]
fn non_ascii_ids_do_not_panic_the_snapshot_scan() {
for model in ["模型模型模型模型", "ローカルモデル-2026", "モデル-0813"] {
assert_eq!(context_window_for_model(model), None, "{model}");
assert_eq!(model_reasoning_capability(model), None, "{model}");
}
}
#[test]
fn compaction_threshold_scales_with_context_window() {
assert_eq!(
+162 -25
View File
@@ -390,6 +390,37 @@ fn client_user_agent(_api_provider: ProviderKind) -> &'static str {
/// of committing to the full remaining window up front.
const RATE_LIMIT_PAUSE_RECHECK_INTERVAL: Duration = Duration::from_millis(250);
/// Total budget for one non-streaming request. Two layers use it: each
/// attempt carries it as a reqwest per-request total (connect through body
/// end, so a trickling body cannot extend forever), and the retry loop
/// through `send_with_retry` is wrapped in one outer envelope of the same
/// length (all attempts, backoff, and honored Retry-After included). The
/// shared client intentionally has no client-level total timeout, so without
/// these nothing bounds a non-streaming completion: a provider that accepts
/// the connection and then stalls — or a gateway answering 429 +
/// `Retry-After: 3600` forever — wedged the caller indefinitely.
///
/// Streaming paths never carry it: their opens go through
/// `send_stream_open_with_retry`, which sets no per-request total (a total
/// would ride on the returned body and hard-cut a live stream), so a stream
/// stays bounded by its open cap and per-chunk idle checks only.
pub(super) const NON_STREAMING_REQUEST_ENVELOPE: Duration = Duration::from_secs(1800);
#[cfg(test)]
static TEST_NON_STREAMING_ENVELOPE_MS: std::sync::atomic::AtomicU64 =
std::sync::atomic::AtomicU64::new(0);
fn non_streaming_request_envelope() -> Duration {
#[cfg(test)]
{
let ms = TEST_NON_STREAMING_ENVELOPE_MS.load(std::sync::atomic::Ordering::SeqCst);
if ms > 0 {
return Duration::from_millis(ms);
}
}
NON_STREAMING_REQUEST_ENVELOPE
}
pub(super) const SSE_BACKPRESSURE_HIGH_WATERMARK: usize = 1024 * 1024; // 1 MB
pub(super) const SSE_BACKPRESSURE_SLEEP_MS: u64 = 10;
pub(super) const SSE_MAX_LINES_PER_CHUNK: usize = 256;
@@ -3267,9 +3298,16 @@ impl CodewhaleClient {
let disclosure = ErrorBodyDisclosure::Guarded {
request_secrets: request_query_secret_values(&url),
};
self.send_with_retry_error_body(build, &disclosure)
.await
.map_err(ModelsFetchError::Interactive)?
// The pinned 30s per-attempt total survives: the retry
// loop's shared envelope is not allowed to overwrite a
// caller's own budget.
self.send_with_retry_total_error_body(
NON_STREAMING_HTTP_TIMEOUT,
build,
&disclosure,
)
.await
.map_err(ModelsFetchError::Interactive)?
}
ModelsRequestMode::Refresh => build()
.send()
@@ -3381,8 +3419,10 @@ impl CodewhaleClient {
/// Activated for model-list authorities that are not satisfied by the
/// cross-provider Models.dev snapshot: OpenRouter, named live gateways,
/// and Baseten's account-scoped endpoint (no static snapshot can serve a
/// per-credential roster). Every other custom host is an ordinary
/// provider served by Models.dev plus its configured models (#6289).
/// per-credential roster). Custom OpenAI-compatible hosts are included
/// too: a private relay is not in the Models.dev snapshot, so without a
/// probe its `/model` picker stays empty even though the chat route
/// already talks to the same endpoint (#6289 widened).
/// The refresh is non-fatal: on failure, persisted prior rows and static
/// seeds remain available with a typed failed receipt.
pub fn spawn_active_provider_catalog_refresh(config: &Config) {
@@ -3395,21 +3435,11 @@ impl CodewhaleClient {
return;
};
let provider = identity.provider;
let is_baseten_endpoint = provider == ProviderKind::Custom
&& codewhale_config::catalog::endpoint_is_baseten(
&config.base_url_for_route(&identity),
);
if !matches!(
provider,
ProviderKind::Openrouter
| ProviderKind::Telecomjs
| ProviderKind::Edenai
| ProviderKind::Zenmux
| ProviderKind::Concentrate
| ProviderKind::Codewhale
| ProviderKind::Ollama
) && !is_baseten_endpoint
{
// Custom hosts include Baseten (its `/models` dialect is detected
// at fetch time) and every other custom host. A private route is
// the only place its roster exists, and a failed probe stays
// non-fatal.
if !crate::provider_catalog_live::provider_owns_live_catalog(provider) {
return;
}
@@ -3698,7 +3728,7 @@ impl CodewhaleClient {
/// much of the body reaches retry logs, state updates and the user.
async fn send_with_retry_error_body<F>(
&self,
mut build: F,
build: F,
disclosure: &ErrorBodyDisclosure,
) -> Result<reqwest::Response>
where
@@ -3707,13 +3737,79 @@ impl CodewhaleClient {
if self.isolated_request_state {
return self.send_with_isolated_retry(build, disclosure).await;
}
self.send_retry_loop(build, Some(non_streaming_request_envelope()), disclosure)
.await
}
/// [`Self::send_with_retry_error_body`] with a caller-pinned per-attempt
/// total (connect through body end). `list_models` pins its own 30s: the
/// plain variant would otherwise stretch that pinned budget out to the
/// shared envelope, because `.timeout()` on the builder is a pure
/// overwrite.
async fn send_with_retry_total_error_body<F>(
&self,
total: Duration,
build: F,
disclosure: &ErrorBodyDisclosure,
) -> Result<reqwest::Response>
where
F: FnMut() -> reqwest::RequestBuilder,
{
if self.isolated_request_state {
return self.send_with_isolated_retry(build, disclosure).await;
}
self.send_retry_loop(build, Some(total), disclosure).await
}
/// The streaming-open twin of [`Self::send_with_retry`]: the same retry
/// and rate-limit handling with no total deadline anywhere. reqwest's
/// per-request timeout wraps the response *body*, so a total set on the
/// open would ride along inside the returned body and hard-cut a live
/// stream mid-generation. Stream opens stay bounded by the caller's
/// `stream_open_timeout` around the open and per-chunk idle checks on
/// the returned body instead.
pub(super) async fn send_stream_open_with_retry<F>(&self, build: F) -> Result<reqwest::Response>
where
F: FnMut() -> reqwest::RequestBuilder,
{
if self.isolated_request_state {
return self
.send_with_isolated_retry(build, &ErrorBodyDisclosure::Full)
.await;
}
self.send_retry_loop(build, None, &ErrorBodyDisclosure::Full)
.await
}
async fn send_retry_loop<F>(
&self,
mut build: F,
attempt_total: Option<Duration>,
disclosure: &ErrorBodyDisclosure,
) -> Result<reqwest::Response>
where
F: FnMut() -> reqwest::RequestBuilder,
{
let retry_cfg: LlmRetryConfig = self.retry.clone().into();
let pause_scope = self.rate_limit_scope();
let callback_scope = pause_scope.clone();
let request_result = with_retry(
// Two bounded layers around a non-streaming completion
// (`attempt_total` = `Some`): the per-attempt request total (connect
// through body end) and an envelope around the whole retry loop (all
// attempts + backoff + honored Retry-After). Streaming opens
// (`None`) set no deadline at all: any total here would be
// inherited by the returned body and truncate the stream, so those
// calls keep only the caller's own open budget.
let retry_future = with_retry(
&retry_cfg,
|| {
let request = build();
// Per-attempt total: unlike the loop envelope below,
// reqwest's per-request timeout also covers the response
// body, so a slow-drip body cannot outlive the budget.
let request = match attempt_total {
Some(total) => build().timeout(total),
None => build(),
};
let pause_scope = pause_scope.as_str();
async move {
// Sleep in bounded slices rather than the full remaining
@@ -3753,8 +3849,30 @@ impl CodewhaleClient {
}
crate::retry_status::start(attempt + 1, delay, human_reason);
})),
)
.await;
);
let request_result = if let Some(total) = attempt_total {
// The loop envelope must dominate the per-attempt total it
// wraps: a caller-pinned budget (list_models' 30s) may exceed
// the shared envelope, and the envelope must never strangle
// its own attempts.
let loop_envelope = total.max(non_streaming_request_envelope());
match tokio::time::timeout(loop_envelope, retry_future).await {
Ok(result) => result,
Err(_elapsed) => {
let last = LlmError::Timeout(loop_envelope);
logging::warn(format!(
"non-streaming request envelope exceeded ({loop_envelope:?}); retry loop aborted"
));
crate::retry_status::failed(last.to_string());
self.mark_request_failure("non-streaming request envelope exceeded")
.await;
self.maybe_probe_recovery().await;
return Err(anyhow::Error::new(last));
}
}
} else {
retry_future.await
};
match request_result {
Ok(response) => {
@@ -3863,6 +3981,25 @@ impl CodewhaleClient {
})
.await
}
/// JSON POST through the streaming-open retry path: no total deadline,
/// because the response body outlives the open (see
/// [`Self::send_stream_open_with_retry`]).
pub(super) async fn open_stream_json_with_retry(
&self,
url: &str,
body: &serde_json::Value,
) -> Result<reqwest::Response> {
let request_body =
serde_json::to_vec(body).context("Failed to serialize JSON request body")?;
self.send_stream_open_with_retry(|| {
self.http_client
.post(url)
.header(CONTENT_TYPE, "application/json")
.body(request_body.clone())
})
.await
}
}
/// Record that a request was routed to `provider` and came back with `status`.
+7 -6
View File
@@ -236,11 +236,12 @@ impl CodewhaleClient {
/// Open the streaming Messages request through the shared stream-entry
/// transport policy: bounded header wait, dual-client selection, and at
/// most one HTTP/1.1 fallback retry on a classified H2 header stall.
/// Inside each open attempt the provider retry loop (`send_with_retry`)
/// handles rate limits and transient upstream failures before any stream
/// body exists, as the Chat and Responses adapters do. Wire-specific
/// request construction (headers, endpoint, body) stays here at the
/// adapter edge.
/// Inside each open attempt the provider retry loop
/// (`send_stream_open_with_retry`) handles rate limits and transient
/// upstream failures before any stream body exists — with no total
/// deadline, which would ride on the returned body — as the Chat and
/// Responses adapters do. Wire-specific request construction (headers,
/// endpoint, body) stays here at the adapter edge.
async fn open_anthropic_stream_response(
&self,
url: &str,
@@ -259,7 +260,7 @@ impl CodewhaleClient {
self.http1_fallback_client(),
policy,
);
self.send_with_retry(|| {
self.send_stream_open_with_retry(|| {
client
.post(&url)
.header(reqwest::header::CONTENT_TYPE, "application/json")
+4 -1
View File
@@ -1351,7 +1351,10 @@ impl CodewhaleClient {
.await?)
}
super::stream_entry::StreamHttpPolicy::DualWithH1Fallback => {
self.send_json_with_retry(url, body).await
// Stream open, not a JSON retry: the response body outlives
// the open, so this path must not carry any total deadline
// (`open_stream_json_with_retry`, not `send_json_with_retry`).
self.open_stream_json_with_retry(url, body).await
}
}
})
+4 -1
View File
@@ -200,7 +200,10 @@ impl CodewhaleClient {
self.http1_fallback_client(),
policy,
);
self.send_with_retry(|| {
// Stream open: the same retry and rate-limit handling with no
// total deadline — a per-request total would ride on the
// returned SSE body and hard-cut the live stream.
self.send_stream_open_with_retry(|| {
client
.post(&url)
.header("Content-Type", "application/json")
+159
View File
@@ -526,6 +526,165 @@
client
}
/// Swap in a short non-streaming envelope for the test's duration,
/// restoring the previous value on drop.
struct NonStreamingEnvelopeGuard(u64);
impl NonStreamingEnvelopeGuard {
fn millis(ms: u64) -> Self {
Self(TEST_NON_STREAMING_ENVELOPE_MS.swap(ms, std::sync::atomic::Ordering::SeqCst))
}
}
impl Drop for NonStreamingEnvelopeGuard {
fn drop(&mut self) {
TEST_NON_STREAMING_ENVELOPE_MS.store(self.0, std::sync::atomic::Ordering::SeqCst);
}
}
/// The provider accepts the connection but stalls far past the budgeted
/// envelope before answering: the non-streaming request must be cut off
/// with a timeout instead of wedging the caller (mid-turn compaction,
/// translate, provider-native search) indefinitely.
#[tokio::test]
async fn non_streaming_envelope_bounds_a_stalled_provider() {
// The injected budget is process-global; serialize against other tests
// (which may issue non-streaming requests with their own timing
// assumptions) through the shared test-env lock.
let _env_lock = crate::test_support::lock_test_env();
let _envelope = NonStreamingEnvelopeGuard::millis(2000);
let server = MockServer::start().await;
Mock::given(method("POST"))
.respond_with(
ResponseTemplate::new(200)
.set_body_json(json!({
"id": "chatcmpl_envelope",
"object": "chat.completion",
"model": "deepseek-v4-pro",
"choices": [{
"index": 0,
"message": {"role": "assistant", "content": "late"},
"finish_reason": "stop"
}],
"usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}
}))
.set_delay(Duration::from_secs(5)),
)
.mount(&server)
.await;
let client = deepseek_request_boundary_client(&server.uri(), server.uri());
let err = client
.create_message(k3_request_fixture("deepseek-v4-pro", Some("off"), false))
.await
.expect_err("a provider that never answers must hit the envelope");
assert!(
err.to_string().to_lowercase().contains("timed out"),
"envelope timeout must be reported as such; got {err:#}"
);
}
/// Every attempt answers 429 with an hour-long Retry-After. Honoring the
/// header must not hand the total budget to the server: the retry loop is
/// capped by the envelope, so a gateway answering 429 + Retry-After: 3600
/// forever fails the turn in bounded time instead of wedging it for hours.
#[tokio::test]
async fn retry_after_honoring_cannot_extend_the_non_streaming_envelope() {
let _env_lock = crate::test_support::lock_test_env();
let _envelope = NonStreamingEnvelopeGuard::millis(2000);
let server = MockServer::start().await;
Mock::given(method("POST"))
.respond_with(
ResponseTemplate::new(429)
.insert_header("retry-after", "3600")
.set_body_string("rate limited"),
)
.mount(&server)
.await;
let client = deepseek_request_boundary_client(&server.uri(), server.uri());
let started = std::time::Instant::now();
let err = client
.create_message(k3_request_fixture("deepseek-v4-pro", Some("off"), false))
.await
.expect_err("unbounded Retry-After honoring must still hit the envelope");
assert!(
err.to_string().to_lowercase().contains("timed out"),
"the envelope must cut off the Retry-After wait; got {err:#}"
);
assert!(
started.elapsed() < Duration::from_secs(30),
"a 3600s Retry-After must not run past the envelope; took {:?}",
started.elapsed()
);
crate::retry_status::clear();
crate::retry_status::clear_rate_limit();
}
/// The open answers past the injected non-streaming envelope. The
/// streaming-open path must not inherit any total: reqwest's per-request
/// timeout wraps the response body, so a total set on the open would ride
/// on the returned body and hard-cut a live stream mid-generation.
#[tokio::test]
async fn stream_open_retry_path_sets_no_total_deadline() {
let _env_lock = crate::test_support::lock_test_env();
let _envelope = NonStreamingEnvelopeGuard::millis(2000);
let server = MockServer::start().await;
Mock::given(method("POST"))
.respond_with(
ResponseTemplate::new(200)
.set_body_string("data: [DONE]\n\n")
.set_delay(Duration::from_millis(2500)),
)
.mount(&server)
.await;
let client = deepseek_request_boundary_client(&server.uri(), server.uri());
let response = client
.send_stream_open_with_retry(|| {
client
.http_client
.post(format!("{}/chat/completions", server.uri()))
.header(reqwest::header::CONTENT_TYPE, "application/json")
.body("{}".to_string())
})
.await
.expect("stream open must not carry the non-streaming envelope");
assert!(response.status().is_success());
let text = response.text().await.expect("read stream-open body");
assert_eq!(text, "data: [DONE]\n\n");
}
/// A caller that pins its own, larger per-attempt total (`list_models`
/// pins 30s) must keep it: `.timeout()` on the builder is a pure
/// overwrite, so an unconditional envelope would silently replace the
/// pinned budget with the injected 2s and fail this 2.5s-late response.
#[tokio::test]
async fn pinned_request_total_survives_the_shared_retry_envelope() {
let _env_lock = crate::test_support::lock_test_env();
let _envelope = NonStreamingEnvelopeGuard::millis(2000);
let server = MockServer::start().await;
Mock::given(method("GET"))
.respond_with(
ResponseTemplate::new(200)
.set_body_string("{}")
.set_delay(Duration::from_millis(2500)),
)
.mount(&server)
.await;
let client = deepseek_request_boundary_client(&server.uri(), server.uri());
let response = client
.send_with_retry_total_error_body(
Duration::from_secs(5),
|| client.http_client.get(format!("{}/models", server.uri())),
&ErrorBodyDisclosure::Full,
)
.await
.expect("caller-pinned total must not be overwritten by the envelope");
assert!(response.status().is_success());
}
/// The per-chunk line cap is backpressure relief, not a data budget. When
/// one transport chunk carries more SSE lines than the cap, the drain loop
/// stops mid-buffer and the outer loop waits for the *next* chunk before
+18
View File
@@ -511,6 +511,24 @@ fn storage_provider(kind: ProviderKind, identity: &str) -> String {
format!("{}:{}", kind.as_str(), identity.trim())
}
/// Providers whose model list is owned by their own `/v1/models` roster
/// rather than the cross-provider Models.dev snapshot: the named live
/// gateways, plus custom hosts whose private roster no snapshot can serve
/// (#6289 widened). The active-provider refresh and the picker's freshness
/// receipt both gate on this one predicate, so they cannot drift apart.
pub(crate) fn provider_owns_live_catalog(provider: ProviderKind) -> bool {
matches!(
provider,
ProviderKind::Openrouter
| ProviderKind::Telecomjs
| ProviderKind::Edenai
| ProviderKind::Zenmux
| ProviderKind::Concentrate
| ProviderKind::Codewhale
| ProviderKind::Ollama
) || provider == ProviderKind::Custom
}
/// Whether a catalog scope holds an account-scoped roster that must never be
/// shared across credentials (#6289).
///
+27 -3
View File
@@ -142,16 +142,16 @@ impl ToolSpec for GithubTool {
"Post an evidence-backed GitHub issue/PR comment with gh. Requires approval. Use blocker comments for partial work; do not claim closure without evidence."
}
Some("close_issue") => {
"Close a GitHub issue only when structured acceptance evidence is present and approved. For pull requests use github_close_pr; do not call PRs issues in user-facing output. Never close merely because the agent is stopping."
"Close a GitHub issue only when structured acceptance evidence is present and approved. Rejected when the worktree is dirty unless allow_dirty=true. For pull requests use github_close_pr; do not call PRs issues in user-facing output. Never close merely because the agent is stopping."
}
Some("close_pr") => {
"Close a GitHub pull request only when structured acceptance evidence is present and approved. Use this for PRs instead of github_close_issue so the UI, audit trail, and comments keep PR wording clear."
"Close a GitHub pull request only when structured acceptance evidence is present and approved. Rejected when the worktree is dirty unless allow_dirty=true. Use this for PRs instead of github_close_issue so the UI, audit trail, and comments keep PR wording clear."
}
_ if self.read_only => {
"Read GitHub issue/PR context using gh (issue_context, pr_context), or read a local current-session Codewhale issue draft with report_read. Local report publication is unavailable."
}
_ => {
"GitHub context (issue_context, pr_context) and guarded comment/close_issue/close_pr actions. Also report_draft and report_read: save/revise/read a LOCAL structured Codewhale issue draft in this session, without network or publication. When you observe evidence of a likely Codewhale/runtime/tool defect, you may draft it yourself, separate observations from inferences, offer /feedback review, and continue the original task. Ordinary user-code failures alone are not Codewhale defects; avoid repeated reports. Include only bounded narrative evidence, never prompts, logs, private code, credentials or paths. report_draft revises an existing draft when revises is supplied; exact repeats converge. Draft publication and duplicate search are unavailable: do not use other tools to post the draft without separate explicit user authorization. No push/merge."
"GitHub context (issue_context, pr_context) and guarded comment/close_issue/close_pr actions. Closes are rejected when the worktree is dirty unless allow_dirty=true. Also report_draft and report_read: save/revise/read a LOCAL structured Codewhale issue draft in this session, without network or publication. When you observe evidence of a likely Codewhale/runtime/tool defect, you may draft it yourself, separate observations from inferences, offer /feedback review, and continue the original task. Ordinary user-code failures alone are not Codewhale defects; avoid repeated reports. Include only bounded narrative evidence, never prompts, logs, private code, credentials or paths. report_draft revises an existing draft when revises is supplied; exact repeats converge. Draft publication and duplicate search are unavailable: do not use other tools to post the draft without separate explicit user authorization. No push/merge."
}
}
}
@@ -339,6 +339,30 @@ mod tests {
);
}
// The dirty-worktree refusal is easy to hit on a real close; the
// model-facing text must disclose it and the flag that overrides it.
#[test]
fn close_disclosure_names_the_dirty_worktree_refusal() {
for description in [
GithubTool::alias("github_close_issue", "close_issue").description(),
GithubTool::alias("github_close_pr", "close_pr").description(),
GithubTool::new("github").description(),
] {
assert!(
description.contains("dirty") && description.contains("allow_dirty"),
"close descriptions must disclose the dirty-worktree refusal: {description}"
);
}
let schema = GithubTool::new("github").input_schema();
let allow_dirty = schema["properties"]["allow_dirty"]["description"]
.as_str()
.expect("allow_dirty must carry a description");
assert!(
allow_dirty.contains("rejected when the worktree is dirty"),
"allow_dirty must disclose the refusal it overrides: {allow_dirty}"
);
}
/// D03-03: only the schema's `issue`/`pr` select a thread kind. Any other
/// spelling is refused before `gh` runs instead of commenting on an issue.
#[tokio::test]
+1 -1
View File
@@ -64,7 +64,7 @@ pub(super) fn canonical_schema(allowed_actions: &[&str], read_only: bool) -> Val
);
properties.insert(
"allow_dirty".to_string(),
json!({ "type": "boolean", "default": false, "description": "(action=close_issue/close_pr)" }),
json!({ "type": "boolean", "default": false, "description": "(action=close_issue/close_pr) Close is rejected when the worktree is dirty unless this is true." }),
);
properties.insert(
"dry_run".to_string(),
+62 -19
View File
@@ -31,11 +31,12 @@
use std::ffi::OsString;
use std::path::{Path, PathBuf};
use std::process::{Command, Stdio};
use std::sync::OnceLock;
use std::process::Stdio;
use std::time::Duration;
use async_trait::async_trait;
use serde_json::{Value, json};
use tokio::process::Command as TokioCommand;
use super::spec::{
ApprovalRequirement, ToolCapability, ToolContext, ToolError, ToolResult, ToolSpec,
@@ -60,6 +61,18 @@ pub(crate) const SUPPORTED_TARGET_FORMATS: &[&str] = &[
"asciidoc", // AsciiDoc
];
/// Wall-clock bound for one pandoc conversion: a pathological document
/// (giant epub, pathological LaTeX) would otherwise hold the call for as
/// long as pandoc felt like taking. Mirrors the 600s interpreter budget
/// used by js_execution.
const PANDOC_TIMEOUT: Duration = Duration::from_secs(600);
/// Bound for the one-shot `pandoc --version` probe. A probe that cannot
/// finish in 10s means the binary itself is wedged; like an unparseable
/// banner, the gate then lets the conversion through and pandoc reports
/// any flag it does not know itself.
const PANDOC_VERSION_PROBE_TIMEOUT: Duration = Duration::from_secs(10);
/// Tool implementing `pandoc_convert`. Converts a source file into
/// a target format and either writes the output to disk or returns
/// the converted text inline.
@@ -162,20 +175,27 @@ impl ToolSpec for PandocConvertTool {
Windows: `winget install JohnMacFarlane.Pandoc`) and restart codewhale.",
)
})?;
require_sandbox_support(&pandoc)?;
require_sandbox_support(&pandoc).await?;
let mut cmd = Command::new(&pandoc);
let mut cmd = TokioCommand::new(&pandoc);
cmd.args(pandoc_args(
&source_path,
&target_format,
resolved_output_path.as_deref(),
));
// Kill the converter if the timeout below drops the output()
// future: pandoc on a pathological document would otherwise keep
// running orphaned after the call already failed.
cmd.kill_on_drop(true);
cmd.stdin(Stdio::null())
.stdout(Stdio::piped())
.stderr(Stdio::piped());
let output = cmd
.output()
let output = tokio::time::timeout(PANDOC_TIMEOUT, cmd.output())
.await
.map_err(|_| ToolError::Timeout {
seconds: PANDOC_TIMEOUT.as_secs(),
})?
.map_err(|e| ToolError::execution_failed(format!("failed to launch pandoc: {e}")))?;
if !output.status.success() {
@@ -235,19 +255,27 @@ fn parse_pandoc_version(banner: &str) -> Option<(u32, u32)> {
/// Refuse a pandoc older than 2.15 with an actionable message instead of
/// pandoc's own "Unknown option --sandbox". The version probe runs once per
/// process; an unparseable banner is let through, and pandoc itself then
/// rejects the flag if it does not know it.
fn require_sandbox_support(pandoc: &str) -> Result<(), ToolError> {
static VERSION: OnceLock<Option<(u32, u32)>> = OnceLock::new();
let version = *VERSION.get_or_init(|| {
let out = Command::new(pandoc)
.arg("--version")
.stdin(Stdio::null())
.stderr(Stdio::null())
.output()
.ok()?;
parse_pandoc_version(&String::from_utf8_lossy(&out.stdout))
});
/// process; an unparseable banner — or a probe that outlives its bound — is
/// let through, and pandoc itself then rejects the flag if it does not know
/// it. The child is async and kill-on-drop so a wedged `--version` can
/// neither block the executor nor outlive the probe.
async fn require_sandbox_support(pandoc: &str) -> Result<(), ToolError> {
static VERSION: tokio::sync::OnceCell<Option<(u32, u32)>> = tokio::sync::OnceCell::const_new();
let version = *VERSION
.get_or_init(|| async {
let mut cmd = TokioCommand::new(pandoc);
cmd.arg("--version")
.stdin(Stdio::null())
.stdout(Stdio::piped())
.stderr(Stdio::null())
.kill_on_drop(true);
let out = tokio::time::timeout(PANDOC_VERSION_PROBE_TIMEOUT, cmd.output())
.await
.ok()?
.ok()?;
parse_pandoc_version(&String::from_utf8_lossy(&out.stdout))
})
.await;
match version {
Some(found) if found < MIN_SANDBOX_VERSION => Err(ToolError::execution_failed(format!(
"pandoc_convert: pandoc {}.{} or newer is required (found {}.{}). \
@@ -420,6 +448,21 @@ mod tests {
assert!((3, 0) >= MIN_SANDBOX_VERSION);
}
// Both invocations are bounded in time; a conversion that outlives its
// budget is answered with ToolError::Timeout and its child killed, but
// exercising that path would need a wedged `pandoc` injected past the
// process-global resolve_pandoc() cache, so the bounds are pinned here
// instead.
#[test]
fn pandoc_invocations_carry_time_bounds() {
assert_eq!(PANDOC_TIMEOUT, Duration::from_secs(600));
assert_eq!(PANDOC_VERSION_PROBE_TIMEOUT, Duration::from_secs(10));
assert!(
PANDOC_VERSION_PROBE_TIMEOUT < PANDOC_TIMEOUT,
"a wedged version probe must fail fast, not ride the conversion budget"
);
}
#[tokio::test]
async fn pandoc_convert_rejects_unsupported_target_format() {
let tmp = tempdir().expect("tempdir");
+40 -5
View File
@@ -185,23 +185,23 @@ impl ToolSpec for TasksTool {
"Cancel a queued or running durable task through TaskManager. Requires approval because it changes work state."
}
Some("gate_run") => {
"Run an approved verification gate command and return structured evidence. When inside a durable task, the gate result and log artifact are attached to that task."
"Run an approved verification gate command and return structured evidence. When inside a durable task, the gate result and log artifact are attached to that task. Dangerous commands are BLOCKED unless auto-approve is enabled; default timeout 120s."
}
Some("pr_attempt_record") => {
"Capture current git diff as a durable PR work attempt with patch artifact, changed files, and verification notes."
"Capture current git diff as a durable PR work attempt with patch artifact, changed files, and verification notes. Requires approval because it records work state."
}
Some("pr_attempt_list") => "List PR attempts recorded on a durable task.",
Some("pr_attempt_read") => {
"Read one recorded PR attempt and its patch artifact reference."
}
Some("pr_attempt_preflight") => {
"Run `git apply --check` for a recorded attempt patch. This is a no-mutation preflight; actual apply remains explicit and approval-gated elsewhere."
"Run `git apply --check` for a recorded attempt patch. This is a no-mutation preflight and itself requires approval; the actual apply stays a separate explicit step."
}
_ if self.read_only => {
"Inspect durable tasks and their PR attempts. Actions: \"list\", \"read\", \"pr_attempt_list\", \"pr_attempt_read\"."
}
_ => {
"Manage durable background tasks through TaskManager. Durable tasks are restart-aware executable work, distinct from sub-agents. Actions: \"create\" (enqueue; approval), \"list\", \"read\", \"cancel\" (approval), \"gate_run\" (run an approved verification gate command and return structured evidence; approval), \"pr_attempt_record\", \"pr_attempt_list\", \"pr_attempt_read\", \"pr_attempt_preflight\". Use task_shell_start for long-running shell work."
"Manage durable background tasks through TaskManager. Durable tasks are restart-aware executable work, distinct from sub-agents. Actions: \"create\" (enqueue; approval), \"list\", \"read\", \"cancel\" (approval), \"gate_run\" (run an approved verification gate command and return structured evidence; approval), \"pr_attempt_record\" (approval), \"pr_attempt_list\", \"pr_attempt_read\", \"pr_attempt_preflight\" (approval). Use task_shell_start for long-running shell work."
}
}
}
@@ -935,7 +935,7 @@ impl ToolSpec for TaskShellStartTool {
"properties": {
"command": { "type": "string" },
"cwd": { "type": "string", "description": "Optional working directory within the workspace." },
"timeout_ms": { "type": "integer", "minimum": 1000, "maximum": 600000 },
"timeout_ms": { "type": "integer", "minimum": 1000, "maximum": 600000, "description": "Accepted for interface compatibility but not enforced: the command always starts in the background and is not bounded by this timeout. A running shell task can be stopped from the model surface via exec_shell with action=cancel and the returned task_id; otherwise it ends when the command finishes." },
"stdin": { "type": "string" },
"tty": { "type": "boolean" }
},
@@ -1609,6 +1609,41 @@ mod tests {
assert!(wait_schema["properties"]["gate"].is_object());
}
// `timeout_ms` rides along for interface compatibility only: the shell
// always starts in the background, where no deadline enforces it. The
// schema must say so instead of implying a bounded run.
#[test]
fn background_shell_timeout_disclosure_admits_it_is_not_enforced() {
let schema = TaskShellStartTool.input_schema();
let timeout_ms = schema["properties"]["timeout_ms"]["description"]
.as_str()
.expect("timeout_ms must carry a description");
assert!(
timeout_ms.contains("not enforced"),
"timeout_ms must disclose that the background run is unbounded: {timeout_ms}"
);
assert!(
timeout_ms.contains("cancel"),
"timeout_ms must point at the exec_shell cancel path that can stop the run: {timeout_ms}"
);
}
// The gate blocks dangerous commands unless auto-approve is on, and the
// description is where the model learns that before composing a command.
#[test]
fn gate_run_description_discloses_dangerous_command_block_and_budget() {
let gate_run = TasksTool::alias("task_gate_run", "gate_run");
let description = gate_run.description();
assert!(
description.contains("BLOCKED") && description.contains("auto-approve"),
"gate_run description must disclose the dangerous-command block: {description}"
);
assert!(
description.contains("120s"),
"gate_run description must disclose the default gate timeout: {description}"
);
}
#[test]
fn runtime_surface_hardening_task_gate_recording_requires_approval() {
let tool = TaskShellWaitTool;
+36 -5
View File
@@ -867,7 +867,7 @@ pub struct TerminalRunTool;
impl ToolSpec for TerminalRunTool {
terminal_tool_common!(
"terminal/run",
"Run a command in a persistent PTY shell session. cd, exports, shell functions, and activated environments persist across calls in this process. Identity and a non-secret last-known summary persist across restarts; prior shells are surfaced as stale/lost and are never reattached."
"Run a command in a persistent PTY shell session. cd, exports, shell functions, and activated environments persist across calls in this process. Identity and a non-secret last-known summary persist across restarts; prior shells are surfaced as stale/lost and are never reattached. On timeout the wait is abandoned but the command keeps running in the session (use terminal/wait or cancel). (Unix only)"
);
fn input_schema(&self) -> serde_json::Value {
json!({"type":"object","properties":{"command":{"type":"string"},"session":{"type":"string","default":"term-1"},"timeout_secs":{"type":"integer","default":120}},"required":["command"]})
@@ -914,7 +914,7 @@ pub struct TerminalSendTool;
impl ToolSpec for TerminalSendTool {
terminal_tool_common!(
"terminal/send",
"Send raw input to a live persistent terminal session. Use a literal ETX control byte to interrupt an interactive process. A prior-process shell is reported as stale/lost rather than reattached."
"Send raw input to a live persistent terminal session. Use a literal ETX control byte to interrupt an interactive process. A prior-process shell is reported as stale/lost rather than reattached. (Unix only)"
);
fn input_schema(&self) -> serde_json::Value {
json!({"type":"object","properties":{"session":{"type":"string"},"text":{"type":"string"},"wait_ms":{"type":"integer","default":250}},"required":["session","text"]})
@@ -956,7 +956,7 @@ pub struct TerminalWaitTool;
impl ToolSpec for TerminalWaitTool {
terminal_tool_common!(
"terminal/wait",
"Wait for the current foreground command in a live persistent terminal session and return buffered output. A prior-process shell is reported as stale/lost rather than reattached."
"Wait for the current foreground command in a live persistent terminal session and return buffered output. A prior-process shell is reported as stale/lost rather than reattached. Output buffer holds at most 512KiB; older bytes are dropped silently. (Unix only)"
);
fn input_schema(&self) -> serde_json::Value {
json!({"type":"object","properties":{"session":{"type":"string"},"timeout_secs":{"type":"integer","default":120}},"required":["session"]})
@@ -995,7 +995,7 @@ pub struct TerminalCancelTool;
impl ToolSpec for TerminalCancelTool {
terminal_tool_common!(
"terminal/cancel",
"Interrupt the running foreground command with ETX. The live terminal session survives and can be reused; its non-secret summary persists."
"Interrupt the running foreground command with ETX. The live terminal session survives and can be reused; its non-secret summary persists. (Unix only)"
);
fn input_schema(&self) -> serde_json::Value {
json!({"type":"object","properties":{"session":{"type":"string"}},"required":["session"]})
@@ -1039,7 +1039,7 @@ pub struct TerminalResetTool;
impl ToolSpec for TerminalResetTool {
terminal_tool_common!(
"terminal/reset",
"Kill and recreate a persistent terminal session with a fresh environment. This loses live cd, exports, functions, activated environments, and running work while retaining the prior historical summary."
"Kill and recreate a persistent terminal session with a fresh environment. This loses live cd, exports, functions, activated environments, and running work while retaining the prior historical summary. (Unix only)"
);
fn input_schema(&self) -> serde_json::Value {
json!({"type":"object","properties":{"session":{"type":"string"}},"required":["session"]})
@@ -1129,6 +1129,37 @@ mod tests {
assert_eq!(head.len() + omitted + tail.len(), input.len());
}
// The whole family is compiled and registered on Unix only, a run's
// timeout abandons the wait without stopping the command, and the output
// ring silently drops the oldest bytes past 512KiB — the descriptions
// must say all three so the model can plan around them.
#[test]
fn terminal_descriptions_disclose_platform_and_wait_semantics() {
let descriptions = [
(TerminalRunTool.name(), TerminalRunTool.description()),
(TerminalSendTool.name(), TerminalSendTool.description()),
(TerminalWaitTool.name(), TerminalWaitTool.description()),
(TerminalCancelTool.name(), TerminalCancelTool.description()),
(TerminalResetTool.name(), TerminalResetTool.description()),
];
for (name, description) in descriptions {
assert!(
description.contains("(Unix only)"),
"{name} must disclose its platform restriction: {description}"
);
}
assert!(
TerminalRunTool
.description()
.contains("the command keeps running in the session"),
"terminal/run must disclose that a timeout abandons the wait, not the command"
);
assert!(
TerminalWaitTool.description().contains("512KiB"),
"terminal/wait must disclose the retained-output bound"
);
}
#[test]
#[cfg(unix)]
fn cd_persists_between_runs() {
+4 -11
View File
@@ -2599,17 +2599,10 @@ fn provider_catalog_receipt_for_route(
.ok()?;
(admitted.provider == provider).then_some(())?;
let identity = admitted.key.as_str();
// A custom route owns its catalog only on Baseten's endpoint, whose
// account-scoped roster no snapshot can serve (#6289).
let owns_provider_catalog = matches!(
provider,
ProviderKind::Openrouter
| ProviderKind::Telecomjs
| ProviderKind::Edenai
| ProviderKind::Zenmux
) || (provider == ProviderKind::Custom
&& codewhale_config::catalog::endpoint_is_baseten(&config.base_url_for_route(&admitted)));
if !owns_provider_catalog {
// One predicate with the active-provider refresh: any route whose roster
// is probed (named live gateways and custom hosts) reports its freshness
// here, so a widened probe cannot leave this receipt behind.
if !crate::provider_catalog_live::provider_owns_live_catalog(provider) {
return None;
}