Integrate observed turn-stop and request-retry diagnostics

Preserve Discussion6010 reporter hxfhd credit. One terminal snapshot uses existing tool inspection, Engine trace and Runtime journal; model requests and retry reasons are counted without adding normal-turn notices or inferring continuation from prose. Explicit tool-stop with no decoded call is a localized protocol failure. Donor formatting and 15 locale parity checks pass; Rust execution awaits combined build. Retry defaults already existed in v0.9.12, so no token-consumption regression claim.
This commit is contained in:
CodeWhale Bot committed 2026-09-08 06:18:52 -07:00
commit 4467d41ba0
23 files changed
+425 -24

No files matched your search

+1
View File
@@ -284,6 +284,7 @@
"ScreenModeMouseCaptureOff": "Captura del ratolí desactivada; el terminal controla la selecció.",
"ScreenModeUnchanged": "Pantalla sense canvis: {reason}.",
"ImageInputRejectedResent": "{model} no accepta imatges — s'han reenviat com a text; usa image_ocr per llegir-les",
"ProviderToolCallMissing": "El proveïdor ha acabat amb `{reason}`, però no ha proporcionat cap crida a una eina. Torna a provar aquest torn per continuar.",
"ConfigLabelShowThinking": "Raonament del model al xat",
"ConfigLabelThinkingHighlight": "Ressaltat de fons del raonament",
"ConfigLabelShowToolDetails": "Nivell de detall de les eines",
+1
View File
@@ -284,6 +284,7 @@
"ScreenModeMouseCaptureOff": "Mausaufnahme deaktiviert; das Terminal steuert die Auswahl.",
"ScreenModeUnchanged": "Bildschirm unverändert: {reason}.",
"ImageInputRejectedResent": "{model} akzeptiert keine Bilder — sie wurden als Text erneut gesendet; nutze image_ocr zum Lesen",
"ProviderToolCallMissing": "Der Anbieter hat mit `{reason}` beendet, aber keinen Werkzeugaufruf geliefert. Wiederhole diesen Durchgang, um fortzufahren.",
"ConfigLabelShowThinking": "Modell-Reasoning im Chat",
"ConfigLabelThinkingHighlight": "Reasoning-Hintergrund hervorheben",
"ConfigLabelShowToolDetails": "Tool-Detailgrad",
+1
View File
@@ -285,6 +285,7 @@
"ScreenModeMouseCaptureOff": "Mouse off — the terminal owns selection.",
"ScreenModeUnchanged": "Screen unchanged: {reason}.",
"ImageInputRejectedResent": "{model} does not accept images — resent as text; use image_ocr to read them",
"ProviderToolCallMissing": "Provider ended with `{reason}` but supplied no tool call. Retry the turn to continue.",
"ConfigLabelShowThinking": "Model reasoning in chat",
"ConfigLabelThinkingHighlight": "Reasoning background highlight",
"ConfigLabelShowToolDetails": "Tool detail level",
+1
View File
@@ -284,6 +284,7 @@
"ScreenModeMouseCaptureOff": "Captura del mouse desactivada; la terminal controla la selección.",
"ScreenModeUnchanged": "Pantalla sin cambios: {reason}.",
"ImageInputRejectedResent": "{model} no acepta imágenes — se reenviaron como texto; usa image_ocr para leerlas",
"ProviderToolCallMissing": "El proveedor terminó con `{reason}`, pero no incluyó ninguna llamada a herramientas. Vuelve a intentar este turno para continuar.",
"ConfigLabelShowThinking": "Razonamiento del modelo en el chat",
"ConfigLabelThinkingHighlight": "Resaltado de fondo del razonamiento",
"ConfigLabelShowToolDetails": "Nivel de detalle de herramientas",
+1
View File
@@ -284,6 +284,7 @@
"ScreenModeMouseCaptureOff": "Capture de la souris désactivée ; le terminal contrôle la sélection.",
"ScreenModeUnchanged": "Écran inchangé : {reason}.",
"ImageInputRejectedResent": "{model} n’accepte pas les images — elles ont été renvoyées sous forme de texte ; utilisez image_ocr pour les lire",
"ProviderToolCallMissing": "Le fournisseur a terminé avec `{reason}`, mais sans fournir d’appel d’outil. Réessayez ce tour pour continuer.",
"ConfigLabelShowThinking": "Raisonnement du modèle dans le chat",
"ConfigLabelThinkingHighlight": "Surlignage du fond du raisonnement",
"ConfigLabelShowToolDetails": "Niveau de détail des outils",
+1
View File
@@ -284,6 +284,7 @@
"ScreenModeMouseCaptureOff": "माउस कैप्चर बंद; चयन टर्मिनल के नियंत्रण में है।",
"ScreenModeUnchanged": "स्क्रीन अपरिवर्तित: {reason}।",
"ImageInputRejectedResent": "{model} इमेज स्वीकार नहीं करता — उन्हें टेक्स्ट के रूप में फिर भेजा गया; पढ़ने के लिए image_ocr का उपयोग करें",
"ProviderToolCallMissing": "प्रदाता ने `{reason}` के साथ उत्तर समाप्त किया, लेकिन कोई टूल कॉल नहीं दिया। जारी रखने के लिए इस टर्न को फिर से आज़माएँ।",
"ConfigLabelShowThinking": "चैट में मॉडल रीज़निंग",
"ConfigLabelThinkingHighlight": "रीज़निंग पृष्ठभूमि हाइलाइट",
"ConfigLabelShowToolDetails": "टूल विवरण स्तर",
+1
View File
@@ -284,6 +284,7 @@
"ScreenModeMouseCaptureOff": "Penangkapan mouse nonaktif; terminal mengatur seleksi.",
"ScreenModeUnchanged": "Layar tidak berubah: {reason}.",
"ImageInputRejectedResent": "{model} tidak menerima gambar — gambar dikirim ulang sebagai teks; gunakan image_ocr untuk membacanya",
"ProviderToolCallMissing": "Penyedia mengakhiri respons dengan `{reason}`, tetapi tidak menyertakan panggilan alat. Coba ulang giliran ini untuk melanjutkan.",
"ConfigLabelShowThinking": "Penalaran model di chat",
"ConfigLabelThinkingHighlight": "Sorotan latar penalaran",
"ConfigLabelShowToolDetails": "Tingkat detail alat",
+1
View File
@@ -284,6 +284,7 @@
"ScreenModeMouseCaptureOff": "マウスキャプチャ:オフ。選択はターミナルが管理します。",
"ScreenModeUnchanged": "画面は変更されませんでした:{reason}。",
"ImageInputRejectedResent": "{model} は画像を受け付けないため、画像をテキストとして再送しました。画像を読むには image_ocr を使用してください",
"ProviderToolCallMissing": "プロバイダーは `{reason}` で応答を終了しましたが、ツール呼び出しがありません。続けるにはこのターンを再試行してください。",
"ConfigLabelShowThinking": "会話にモデル推論を表示",
"ConfigLabelThinkingHighlight": "推論の背景ハイライト",
"ConfigLabelShowToolDetails": "ツール詳細レベル",
+1
View File
@@ -284,6 +284,7 @@
"ScreenModeMouseCaptureOff": "마우스 캡처 꺼짐; 선택은 터미널이 담당합니다.",
"ScreenModeUnchanged": "화면 변경 없음: {reason}.",
"ImageInputRejectedResent": "{model}은(는) 이미지를 지원하지 않아 텍스트로 다시 보냈습니다. image_ocr를 사용해 읽으세요",
"ProviderToolCallMissing": "공급자가 `{reason}`으로 응답을 종료했지만 도구 호출을 제공하지 않았습니다. 계속하려면 이 턴을 다시 시도하세요.",
"ConfigLabelShowThinking": "채팅에 모델 추론 표시",
"ConfigLabelThinkingHighlight": "추론 배경 강조",
"ConfigLabelShowToolDetails": "도구 세부 수준",
+1
View File
@@ -284,6 +284,7 @@
"ScreenModeMouseCaptureOff": "Captura do mouse desativada; o terminal controla a seleção.",
"ScreenModeUnchanged": "Tela inalterada: {reason}.",
"ImageInputRejectedResent": "{model} não aceita imagens — reenviado como texto; use image_ocr para lê-las",
"ProviderToolCallMissing": "O provedor encerrou com `{reason}`, mas não forneceu nenhuma chamada de ferramenta. Tente este turno novamente para continuar.",
"ConfigLabelShowThinking": "Raciocínio do modelo no chat",
"ConfigLabelThinkingHighlight": "Destaque do fundo do raciocínio",
"ConfigLabelShowToolDetails": "Nível de detalhe das ferramentas",
+1
View File
@@ -284,6 +284,7 @@
"ScreenModeMouseCaptureOff": "Захват мыши выключен; выбором управляет терминал.",
"ScreenModeUnchanged": "Экран без изменений: {reason}.",
"ImageInputRejectedResent": "{model} не принимает изображения — они отправлены повторно как текст; используйте image_ocr для чтения",
"ProviderToolCallMissing": "Провайдер завершил ответ с `{reason}`, но не передал вызов инструмента. Повторите этот ход, чтобы продолжить.",
"ConfigLabelShowThinking": "Рассуждения модели в чате",
"ConfigLabelThinkingHighlight": "Подсветка фона рассуждений",
"ConfigLabelShowToolDetails": "Детализация инструментов",
+1
View File
@@ -284,6 +284,7 @@
"ScreenModeMouseCaptureOff": "Захоплення миші вимкнено; вибором керує термінал.",
"ScreenModeUnchanged": "Екран без змін: {reason}.",
"ImageInputRejectedResent": "{model} не приймає зображення — їх повторно надіслано як текст; скористайтеся image_ocr, щоб їх прочитати",
"ProviderToolCallMissing": "Провайдер завершив відповідь із `{reason}`, але не передав виклик інструмента. Повторіть цей хід, щоб продовжити.",
"ConfigLabelShowThinking": "Міркування моделі в чаті",
"ConfigLabelThinkingHighlight": "Підсвічування тла міркувань",
"ConfigLabelShowToolDetails": "Рівень деталізації інструментів",
+1
View File
@@ -284,6 +284,7 @@
"ScreenModeMouseCaptureOff": "Đã tắt bắt chuột; thiết bị đầu cuối quản lý việc chọn.",
"ScreenModeUnchanged": "Màn hình không thay đổi: {reason}.",
"ImageInputRejectedResent": "{model} không chấp nhận hình ảnh — đã gửi lại dưới dạng văn bản; hãy dùng image_ocr để đọc chúng",
"ProviderToolCallMissing": "Nhà cung cấp kết thúc với `{reason}` nhưng không gửi lệnh gọi công cụ nào. Hãy thử lại lượt này để tiếp tục.",
"ConfigLabelShowThinking": "Hiển thị suy luận trong trò chuyện",
"ConfigLabelThinkingHighlight": "Tô nền phần suy luận",
"ConfigLabelShowToolDetails": "Mức chi tiết công cụ",
+1
View File
@@ -284,6 +284,7 @@
"ScreenModeMouseCaptureOff": "鼠标捕获已关闭;选择由终端管理。",
"ScreenModeUnchanged": "屏幕未改变:{reason}。",
"ImageInputRejectedResent": "{model} 不接受图像,已将图像替换为文本重新发送;请使用 image_ocr 读取图像",
"ProviderToolCallMissing": "提供商以 `{reason}` 结束响应,但未提供工具调用。请重试本轮以继续。",
"ConfigLabelShowThinking": "对话中显示模型推理",
"ConfigLabelThinkingHighlight": "推理背景高亮",
"ConfigLabelShowToolDetails": "工具详情级别",
+1
View File
@@ -576,6 +576,7 @@
"ScreenModeMouseCaptureOff": "滑鼠擷取已關閉;選取由終端機管理。",
"ScreenModeUnchanged": "畫面未變更:{reason}。",
"ImageInputRejectedResent": "{model} 不接受圖片,已將圖片替換為文字重新傳送;請使用 image_ocr 讀取圖片",
"ProviderToolCallMissing": "提供商以 `{reason}` 結束回應,但未提供工具呼叫。請重試本輪以繼續。",
"ConfigLabelLocale": "語言",
"ConfigLabelLowMotion": "減少動態效果",
"ConfigLabelManagedAllowShell": "Shell 存取(受管理)",
+10
View File
@@ -5433,6 +5433,12 @@ impl Engine {
if status == TurnOutcomeStatus::Interrupted {
self.emit_interrupted_survivor_status().await;
}
if let Some(snapshot) = turn.terminal_request_snapshot(status) {
let _ = self
.tx_event
.send(Event::ToolRequestSnapshot { snapshot })
.await;
}
drop(turn_control);
let turn_complete_delivered = self
.tx_event
@@ -5995,6 +6001,10 @@ impl Engine {
};
let id = format!("compact_{}", &uuid::Uuid::new_v4().to_string()[..8]);
turn.stop_diagnostics.emergency_compaction_attempts = turn
.stop_diagnostics
.emergency_compaction_attempts
.saturating_add(1);
let start_message = format!("Emergency context compaction started ({reason})");
self.emit_compaction_started(id.clone(), true, start_message)
.await;
+5 -5
View File
@@ -47,17 +47,17 @@ pub(super) const STREAM_MAX_DURATION_SECS: u64 = 1800; // 30 minutes (was 300s;
/// tolerate a longer streak before giving up on the turn.
pub(super) const MAX_STREAM_ERRORS_BEFORE_FAIL: u32 = 5;
/// Cap on transparent stream-level retries — these only happen when the wire
/// dies before any content was streamed, so DeepSeek hasn't billed us and
/// the user hasn't seen anything. Two attempts is enough to ride out a
/// dies before any content was streamed. The user has seen nothing, but
/// provider usage or billing may already exist. Two attempts can ride out a
/// flaky edge node without amplifying real outages (#103).
pub(super) const MAX_TRANSPARENT_STREAM_RETRIES: u32 = 2;
/// Decide whether a stream error is eligible for a transparent retry.
///
/// True only when ALL three conditions hold:
/// 1. No content has been received on the current attempt — otherwise DeepSeek
/// has already billed us for output tokens and the user has seen partial
/// deltas; resending would double-bill and desync the UI.
/// 1. No content has been received on the current attempt. Reissuing after
/// visible partial deltas needs a separate recovery policy. This content
/// check is not evidence that the provider consumed or billed zero tokens.
/// 2. We still have transparent-retry budget remaining.
/// 3. The turn has not been cancelled.
///
+220 -3
View File
@@ -6505,6 +6505,147 @@ async fn request_selector_distinguishes_absent_tools_from_present_empty_tools()
assert_eq!(present_empty.payload_json_bytes, Some(2));
}
#[tokio::test]
async fn terminal_diagnostics_distinguish_narration_from_missing_protocol_tool_calls() {
use crate::llm_client::mock::{MockLlmClient, canned};
use crate::tool_inspection::TurnStopReason;
for finish in ["end_turn", "tool_calls", "tool_use"] {
let workspace = tempdir().expect("tempdir");
let response = vec![
canned::message_start("stop_diagnostic_fixture"),
canned::text_block_start(0),
canned::text_delta(0, "I will edit the file. 我现在修改文件。"),
canned::block_stop(0),
canned::message_delta(
finish,
Some(Usage {
input_tokens: 100,
output_tokens: 12,
reasoning_tokens: Some(4),
prompt_cache_hit_tokens: Some(80),
prompt_cache_miss_tokens: Some(20),
..Usage::default()
}),
),
canned::message_stop(),
];
let mock = std::sync::Arc::new(MockLlmClient::new(vec![response]));
let client: crate::core::model_client::SharedModelClient = mock.clone();
let (mut engine, handle) = Engine::new_with_model_client(
deterministic_engine_config(workspace.path()),
&Config::default(),
client,
);
let registry = crate::tools::ToolRegistry::new(crate::tools::ToolContext::new(
workspace.path().to_path_buf(),
));
let surface = test_tool_surface(&engine, registry, None, AppMode::Agent);
let mut turn = crate::core::turn::TurnContext::new(4);
let (status, error) = engine.run_turn(&mut turn, surface, None, None).await;
let expected = if finish == "end_turn" {
assert!(
error.is_none(),
"ordinary narration is not a protocol error"
);
TurnStopReason::ProviderNoToolCall
} else {
assert_eq!(status, TurnOutcomeStatus::Failed);
assert!(
error
.as_deref()
.is_some_and(|error| error.contains("supplied no tool call"))
);
TurnStopReason::ProviderToolCallMissing
};
let snapshot = turn
.terminal_request_snapshot(status)
.expect("terminal request snapshot");
let terminal = snapshot.terminal.as_ref().expect("terminal facts");
assert_eq!(terminal.reason, Some(expected));
assert_eq!(terminal.model_requests_started, 1);
assert_eq!(terminal.last_reported_input_tokens, Some(100));
assert_eq!(terminal.last_response_tool_calls, Some(0));
assert_eq!(terminal.last_response_tool_calls_suppressed, Some(0));
assert_eq!(
terminal.last_provider_finish_reason.as_ref().unwrap().value,
finish
);
assert_eq!(
mock.captured_requests().len(),
1,
"narration must not synthesize continuation"
);
assert_eq!(turn.usage.input_tokens, 100);
assert_eq!(
turn.usage.output_tokens, 12,
"reasoning is an output subset, not additional output"
);
assert!(snapshot.render_text().contains("Terminal diagnostics"));
let json = serde_json::to_value(&snapshot).expect("serialize snapshot");
assert_eq!(json["terminal"]["model_requests_started"], 1);
let mut events = handle.rx_event.write().await;
assert!(
!std::iter::from_fn(|| events.try_recv().ok())
.any(|event| matches!(event, Event::ToolCallStarted { .. }))
);
}
}
#[tokio::test]
async fn terminal_diagnostics_merge_cumulative_usage_within_one_request() {
use crate::llm_client::mock::{MockLlmClient, canned};
let workspace = tempdir().expect("tempdir");
let usage = Usage {
input_tokens: 100,
output_tokens: 12,
reasoning_tokens: Some(4),
prompt_cache_hit_tokens: Some(80),
prompt_cache_miss_tokens: Some(20),
..Usage::default()
};
let mut start = canned::message_start("cumulative_usage_fixture");
if let StreamEvent::MessageStart { message } = &mut start {
message.usage = usage.clone();
}
let mock = std::sync::Arc::new(MockLlmClient::new(vec![vec![
start,
canned::text_block_start(0),
canned::text_delta(0, "Done."),
canned::block_stop(0),
canned::message_delta("end_turn", Some(usage.clone())),
canned::message_delta("end_turn", Some(usage.clone())),
canned::message_stop(),
]]));
let client: crate::core::model_client::SharedModelClient = mock.clone();
let (mut engine, handle) = Engine::new_with_model_client(
deterministic_engine_config(workspace.path()),
&Config::default(),
client,
);
let registry = crate::tools::ToolRegistry::new(crate::tools::ToolContext::new(
workspace.path().to_path_buf(),
));
let surface = test_tool_surface(&engine, registry, None, AppMode::Agent);
let mut turn = crate::core::turn::TurnContext::new(4);
let (status, error) = engine.run_turn(&mut turn, surface, None, None).await;
assert_eq!(status, TurnOutcomeStatus::Completed, "{error:?}");
assert_eq!(mock.captured_requests().len(), 1);
assert_eq!(
turn.usage, usage,
"repeated cumulative receipts are counted once"
);
let mut events = handle.rx_event.write().await;
let usages = std::iter::from_fn(|| events.try_recv().ok())
.filter_map(|event| match event {
Event::TurnUsage { usage, .. } => Some(usage),
_ => None,
})
.collect::<Vec<_>>();
assert_eq!(usages, vec![usage]);
}
#[tokio::test]
async fn request_snapshots_advance_to_the_latest_tool_step() {
use crate::llm_client::mock::{MockLlmClient, canned};
@@ -19878,6 +20019,7 @@ struct FlakyNetworkDropModelClient {
calls: std::sync::atomic::AtomicUsize,
failures: usize,
terminal_before_drop: bool,
content_before_drop: bool,
}
#[async_trait::async_trait]
@@ -19932,6 +20074,14 @@ impl crate::core::model_client::ModelClient for FlakyNetworkDropModelClient {
];
return Ok(Box::pin(futures_util::stream::iter(events)));
}
if !self.content_before_drop {
return Ok(Box::pin(futures_util::stream::iter(vec![
Ok(canned::message_start("empty_then_drop")),
Err(anyhow::anyhow!(
"Stream read error: error decoding response body"
)),
])));
}
// Partial content first — this flips `any_content_received` so
// the #103 transparent retry cannot fire — then the transport
// dies the way the 0.9.4 Terminal-Bench crashes did.
@@ -19969,6 +20119,7 @@ async fn run_headless_turn_with_flaky_network(
calls: std::sync::atomic::AtomicUsize::new(0),
failures,
terminal_before_drop: false,
content_before_drop: true,
});
let client: crate::core::model_client::SharedModelClient = model.clone();
let config = Config::default();
@@ -20034,6 +20185,17 @@ async fn run_headless_turn_with_flaky_network(
async fn headless_turn_retries_mid_stream_network_drop_and_recovers() {
let (model, events) = run_headless_turn_with_flaky_network(1).await;
let terminal = events
.iter()
.find_map(|event| match event {
Event::ToolRequestSnapshot { snapshot } => snapshot.terminal.as_ref(),
_ => None,
})
.expect("terminal diagnostics through existing event authority");
assert_eq!(terminal.model_requests_started, 2);
assert_eq!(terminal.stream_resumes, 1);
assert_eq!(terminal.transparent_stream_retries, 0);
assert_eq!(
model.calls.load(std::sync::atomic::Ordering::SeqCst),
2,
@@ -20091,12 +20253,54 @@ async fn headless_turn_retries_mid_stream_network_drop_and_recovers() {
);
}
#[tokio::test]
async fn terminal_diagnostics_count_transparent_stream_requests_without_extra_snapshots() {
let workspace = tempdir().expect("tempdir");
let model = std::sync::Arc::new(FlakyNetworkDropModelClient {
calls: std::sync::atomic::AtomicUsize::new(0),
failures: 1,
terminal_before_drop: false,
content_before_drop: false,
});
let client: crate::core::model_client::SharedModelClient = model.clone();
let (mut engine, handle) = Engine::new_with_model_client(
deterministic_engine_config(workspace.path()),
&Config::default(),
client,
);
let registry = crate::tools::ToolRegistry::new(crate::tools::ToolContext::new(
workspace.path().to_path_buf(),
));
let surface = test_tool_surface(&engine, registry, None, AppMode::Agent);
let mut turn = crate::core::turn::TurnContext::new(4);
let (status, error) = engine.run_turn(&mut turn, surface, None, None).await;
assert_eq!(status, TurnOutcomeStatus::Completed, "{error:?}");
assert_eq!(model.calls.load(std::sync::atomic::Ordering::SeqCst), 2);
let snapshot = turn
.terminal_request_snapshot(status)
.expect("terminal snapshot");
let terminal = snapshot.terminal.expect("terminal facts");
assert_eq!(terminal.model_requests_started, 2);
assert_eq!(terminal.transparent_stream_retries, 1);
assert_eq!(terminal.stream_resumes, 0);
assert_eq!(terminal.model_step_index, 0);
let mut events = handle.rx_event.write().await;
let snapshots = std::iter::from_fn(|| events.try_recv().ok())
.filter(|event| matches!(event, Event::ToolRequestSnapshot { .. }))
.count();
assert_eq!(
snapshots, 1,
"request construction is distinct from stream retries"
);
}
#[tokio::test]
async fn terminal_output_limit_followed_by_stream_error_is_charged_and_not_retried() {
let model = std::sync::Arc::new(FlakyNetworkDropModelClient {
calls: std::sync::atomic::AtomicUsize::new(0),
failures: 1,
terminal_before_drop: true,
content_before_drop: true,
});
let client: crate::core::model_client::SharedModelClient = model.clone();
let config = Config::default();
@@ -20454,6 +20658,7 @@ async fn run_interactive_turn_with_flaky_network(
calls: std::sync::atomic::AtomicUsize::new(0),
failures,
terminal_before_drop: false,
content_before_drop: true,
});
let client: crate::core::model_client::SharedModelClient = model.clone();
let config = Config::default();
@@ -20995,12 +21200,24 @@ async fn run_reasoning_only_turn_with_reprompts(
(model, events)
}
/// A reasoning-only clean-stop response is re-requested (cheap: the prefix is
/// cached) and the turn recovers with the real answer instead of dead-ending.
/// A reasoning-only clean-stop response is re-requested and the turn recovers
/// with the real answer. Local fixtures make no cache-hit or billing claim.
#[tokio::test]
async fn reasoning_only_clean_stop_is_retried_and_recovers() {
let (model, events) = run_reasoning_only_turn(1, "stop").await;
let terminal = events
.iter()
.find_map(|event| match event {
Event::ToolRequestSnapshot { snapshot } => snapshot.terminal.as_ref(),
_ => None,
})
.expect("terminal diagnostics through existing event authority");
assert_eq!(terminal.model_requests_started, 2);
assert_eq!(terminal.reasoning_only_reprompts, 1);
assert_eq!(terminal.transparent_stream_retries, 0);
assert_eq!(terminal.status, Some(TurnOutcomeStatus::Completed));
assert_eq!(
model.calls.load(std::sync::atomic::Ordering::SeqCst),
2,
@@ -21113,7 +21330,7 @@ async fn the_reasoning_only_nudge_rides_one_request_and_never_joins_the_session(
.contains(nudge)
};
assert!(!carries_nudge(&requests[0]), "no nudge before any failure");
assert!(!carries_nudge(&requests[1]), "no nudge on the free retry");
assert!(!carries_nudge(&requests[1]), "no nudge on the first retry");
assert!(
carries_nudge(&requests[2]),
"nudge present once retrying again"
+72 -15
View File
@@ -15,6 +15,7 @@ use crate::runtime_handoff::{
shell_completion_runtime_message, subagent_completion_runtime_message,
subagent_failure_runtime_message, waiting_for_subagents_runtime_message,
};
use crate::tool_inspection::TurnStopReason;
use crate::tools::canonical_action::canonical_action_alias;
use crate::tools::tool_call_budget::ToolCallBudget;
use codewhale_core::request::{PrimaryTurnRequest, prepare_primary_turn_request};
@@ -714,8 +715,6 @@ impl Engine {
// Cleared when the loop continues only for optional runtime work
// (a goal continuation) after the model already delivered an answer.
let mut step_budget_exhaustion_is_terminal = true;
// A1: one soft-landing notice at ~80% of a finite step budget.
let mut soft_landing_sent = false;
// A2: one final report turn after the budget is exhausted, so a child
// that owes work never finishes silently.
let mut final_report_sent = false;
@@ -758,8 +757,8 @@ impl Engine {
// reasoning — a protocol-complete but answerless response that reaches
// the failure tail with `stream_errors == 0`, so the transport resume
// path above never sees it. A clean stop there is almost always
// transient; re-request a bounded number of times (the prefix is
// cached, so each retry is cheap) before surfacing a hard failure.
// transient; re-request a bounded number of times before surfacing
// a hard failure. Each retry may incur provider usage and cost.
let mut reasoning_only_reprompts: u32 = 0;
// Nudge for the *next* request only. A reasoning-only reply persists
// nothing (a bare Thinking block is not sendable), so the first retry
@@ -876,11 +875,11 @@ impl Engine {
// spent tell the model once to stop exploring and write its final
// report. Savings proved out by the grok-style parity work (ops
// A1): a step-faithful harness ends mid-report far too often.
if !soft_landing_sent
if !turn.stop_diagnostics.soft_landing_sent
&& turn.max_steps > 0
&& turn.steps_used() >= ((turn.max_steps as f32 * 0.8).floor() as u32).max(1)
{
soft_landing_sent = true;
turn.stop_diagnostics.soft_landing_sent = true;
let notice = format!(
"Step budget soft landing: you have used about {}% of your {} step budget ({}). Stop exploring; write your final, complete report now, in final form, with evidence.",
80,
@@ -898,6 +897,7 @@ impl Engine {
}
if turn.at_max_steps() && turn_end_child_coordination_responses_remaining == 0 {
turn.stop_diagnostics.reason = Some(TurnStopReason::StepBudgetExhausted);
if self
.request_turn_owned_child_coordination(
foreground_children.as_ref(),
@@ -1036,6 +1036,10 @@ impl Engine {
&& compaction_go
{
let compaction_id = format!("compact_{}", &uuid::Uuid::new_v4().to_string()[..8]);
turn.stop_diagnostics.automatic_compaction_attempts = turn
.stop_diagnostics
.automatic_compaction_attempts
.saturating_add(1);
let compaction_cancel = self
.claim_compaction(&compaction_id)
.expect("a fresh automatic compaction id cannot be pre-canceled");
@@ -1457,6 +1461,10 @@ impl Engine {
request.tools.as_deref(),
inspection_surface.as_ref(),
);
turn.last_request_snapshot = Some(tool_request_snapshot.clone());
turn.stop_diagnostics.route_context_window_tokens = self
.active_route_limits
.and_then(|limits| limits.context_tokens);
// Stream the response. Keep the request around (cloned into the
// first call) so we can resend it on a transparent retry below
@@ -1509,7 +1517,13 @@ impl Engine {
let _ = self.tx_event.send(Event::status("Request cancelled")).await;
return (TurnOutcomeStatus::Interrupted, None);
}
result = client.create_message_stream(stream_request.clone()) => result,
result = async {
turn.stop_diagnostics.model_requests_started = turn
.stop_diagnostics
.model_requests_started
.saturating_add(1);
client.create_message_stream(stream_request.clone()).await
} => result,
};
let stream = match stream_result {
Ok(s) => {
@@ -1588,9 +1602,12 @@ impl Engine {
&stream_request,
request_dispatched_at,
stream_retry_budget.spent(),
&mut turn.stop_diagnostics,
)
.await;
turn_error = turn_error.or(stream_error);
turn.stop_diagnostics
.observe_provider_response(stop_reason.as_deref(), tool_uses.len());
// These belong to post-stream response assembly, not stream
// consumption: blocks are built from the completed stream state,
// and truncation is derived from its terminal stop reason below.
@@ -1710,6 +1727,8 @@ impl Engine {
if let Some(resume) = pending_resume
&& let Some(attempt) = stream_retry_budget.authorize()
{
turn.stop_diagnostics.stream_resumes =
turn.stop_diagnostics.stream_resumes.saturating_add(1);
match resume {
StreamResume::AfterSleep => {
crate::logging::warn(format!(
@@ -1905,6 +1924,30 @@ impl Engine {
}
}
// A protocol-level tool stop promises a call, unlike ordinary
// text that merely describes an intended action. Keep that
// distinction factual; never synthesize a tool or another request.
if tool_uses.is_empty()
&& turn_error.is_none()
&& matches!(stop_reason.as_deref(), Some("tool_calls" | "tool_use"))
{
turn.stop_diagnostics.reason = Some(TurnStopReason::ProviderToolCallMissing);
turn.stop_diagnostics.last_response_tool_calls_suppressed = Some(0);
self.add_interrupted_assistant_text(&current_text_visible)
.await;
let reason = stop_reason.as_deref().expect("matched tool stop");
return (
TurnOutcomeStatus::Failed,
Some(
crate::localization::tr(
crate::localization::resolve_locale(&self.config.locale_tag),
crate::localization::MessageId::ProviderToolCallMissing,
)
.replace("{reason}", reason),
),
);
}
for tool in &mut tool_uses {
let Some(schema) = tool_catalog
.iter()
@@ -2480,10 +2523,11 @@ impl Engine {
// no prefix churn. An output-length stop is excluded above
// because retrying would only reproduce it.
reasoning_only_reprompts += 1;
turn.stop_diagnostics.reasoning_only_reprompts = reasoning_only_reprompts;
let attempt = reasoning_only_reprompts;
let max_reprompts = self.config.reasoning_only_max_reprompts;
// Attempt 1 is the free one: the prefix is cached and a
// clean stop here is usually transient. From attempt 2 on,
// Attempt 1 preserves the prefix; a cache hit or lower
// cost is not guaranteed. From attempt 2 on,
// an identical request has already failed once, so carry
// the nudge rather than reproduce the same answerless reply.
let nudged = attempt > 1;
@@ -2548,6 +2592,13 @@ impl Engine {
.await;
}
if turn_error.is_none() {
if !turn.budget_exhausted_final_report {
turn.stop_diagnostics.reason = Some(TurnStopReason::ProviderNoToolCall);
}
// This branch received no calls and dispatches no tools.
turn.stop_diagnostics.last_response_tool_calls_suppressed = Some(0);
}
break;
}
@@ -4390,6 +4441,7 @@ impl Engine {
stream_request: &crate::models::MessageRequest,
mut request_dispatched_at: Instant,
drop_resumes_spent: u32,
diagnostics: &mut crate::tool_inspection::TurnStopDiagnostics,
) -> StreamOutcome {
// The stream value is itself `Pin<Box<dyn Stream + Send>>`, which
// is `Unpin`, so we can rebind it on a transparent retry without
@@ -4434,8 +4486,8 @@ impl Engine {
let mut last_text_index: Option<usize> = None;
let mut stream_errors = 0u32;
// #103 transparent retry bookkeeping. `any_content_received` flips
// on the first non-MessageStart event so we know whether DeepSeek
// billed us / the user has seen any output for this turn yet.
// on the first actionable content event so we know whether the user
// has seen output. Absence of content does not establish zero usage.
// This is distinct from the outer drop-resume budget (which
// restarts the whole turn-step when a stream died with no
// content-block delta delivered to the consumer).
@@ -4582,9 +4634,8 @@ impl Engine {
}
// #103: when the stream errors before any content was
// streamed AND we still have retry budget, transparently
// resend the request. DeepSeek has not billed for any
// output and the user has seen nothing — re-trying is
// the right user-visible behavior.
// resend the request. The user has seen nothing, but the
// provider may already have consumed or billed tokens.
if should_transparently_retry_stream(
any_content_received,
transparent_stream_retries,
@@ -4601,7 +4652,13 @@ impl Engine {
let retry_stream_result = tokio::select! {
biased;
() = self.cancel_token.cancelled() => break,
result = client.create_message_stream(stream_request.clone()) => result,
result = async {
diagnostics.transparent_stream_retries =
diagnostics.transparent_stream_retries.saturating_add(1);
diagnostics.model_requests_started =
diagnostics.model_requests_started.saturating_add(1);
client.create_message_stream(stream_request.clone()).await
} => result,
};
match retry_stream_result {
Ok(fresh) => {
+2 -1
View File
@@ -17,7 +17,8 @@ use crate::tools::subagent::{AgentWorkerStatus, CoordinationDetailProjection, Su
use crate::tools::user_input::UserInputRequest;
/// Final status for a turn.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize)]
#[serde(rename_all = "snake_case")]
pub enum TurnOutcomeStatus {
Completed,
Interrupted,
+34
View File
@@ -64,6 +64,9 @@ pub struct TurnContext {
/// it so an exhausted goal pauses instead of re-arming.
pub budget_exhausted_final_report: bool,
pub(crate) stop_diagnostics: crate::tool_inspection::TurnStopDiagnostics,
pub(crate) last_request_snapshot: Option<crate::tool_inspection::ToolInspectionSnapshot>,
/// Number of tool calls made in this turn.
/// Whether the turn has been cancelled
@@ -114,6 +117,12 @@ impl TurnContext {
max_steps,
budget_source,
budget_exhausted_final_report: false,
stop_diagnostics: crate::tool_inspection::TurnStopDiagnostics {
effective_max_steps: max_steps,
step_budget_source: budget_source.key_label(),
..Default::default()
},
last_request_snapshot: None,
cancelled: false,
usage: Usage {
input_tokens: 0,
@@ -157,6 +166,31 @@ impl TurnContext {
self.started_at.elapsed()
}
/// Complete the existing request projection with observed turn-exit facts.
/// A turn that never prepared a request has no request snapshot to publish.
pub(crate) fn terminal_request_snapshot(
&mut self,
status: super::events::TurnOutcomeStatus,
) -> Option<crate::tool_inspection::ToolInspectionSnapshot> {
use crate::tool_inspection::TurnStopReason;
self.stop_diagnostics.status = Some(status);
self.stop_diagnostics.model_step_index = self.step;
self.stop_diagnostics.final_report_requested = self.budget_exhausted_final_report;
self.stop_diagnostics.last_reported_input_tokens = self.latest_parent_input_tokens;
match status {
super::events::TurnOutcomeStatus::Interrupted => {
self.stop_diagnostics.reason = Some(TurnStopReason::Interrupted);
}
super::events::TurnOutcomeStatus::Failed if self.stop_diagnostics.reason.is_none() => {
self.stop_diagnostics.reason = Some(TurnStopReason::Failed);
}
_ => {}
}
let mut snapshot = self.last_request_snapshot.take()?;
snapshot.terminal = Some(self.stop_diagnostics.clone());
Some(snapshot)
}
/// Add usage from an API response
pub fn add_usage(&mut self, usage: &Usage) {
add_usage_to(&mut self.usage, usage);
+2
View File
@@ -168,6 +168,7 @@ pub enum MessageId {
ScreenModeMouseCaptureOff,
ScreenModeUnchanged,
ImageInputRejectedResent,
ProviderToolCallMissing,
// First-run anonymous usage disclosure.
TelemetryNoticeDefaultOn,
TelemetryNoticeHeadline,
@@ -2469,6 +2470,7 @@ pub const ALL_MESSAGE_IDS: &[MessageId] = &[
MessageId::ScreenModeMouseCaptureOff,
MessageId::ScreenModeUnchanged,
MessageId::ImageInputRejectedResent,
MessageId::ProviderToolCallMissing,
MessageId::TelemetryNoticeDefaultOn,
MessageId::TelemetryNoticeHeadline,
MessageId::TelemetryNoticeBody,
+65
View File
@@ -45,6 +45,61 @@ pub struct BoundedString {
pub truncated: bool,
}
/// Observed engine exit boundary. This never classifies the assistant's prose.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum TurnStopReason {
ProviderNoToolCall,
ProviderToolCallMissing,
StepBudgetExhausted,
Interrupted,
Failed,
}
/// Terminal facts attached to the existing request inspector, without adding
/// conversation input or a notice to an ordinary successful response.
#[derive(Debug, Clone, Default, PartialEq, Serialize)]
pub struct TurnStopDiagnostics {
pub status: Option<crate::core::events::TurnOutcomeStatus>,
/// None means the precise runtime exit boundary was not observed.
pub reason: Option<TurnStopReason>,
pub effective_max_steps: u32,
pub step_budget_source: &'static str,
/// Existing zero-based scheduler step; transport retries do not advance it.
pub model_step_index: u32,
/// Parent streaming ModelClient calls, including stream retries. Excludes
/// HTTP retries inside the client, compaction and child calls; not invoices.
pub model_requests_started: u32,
pub transparent_stream_retries: u32,
pub stream_resumes: u32,
pub reasoning_only_reprompts: u32,
pub soft_landing_sent: bool,
pub final_report_requested: bool,
pub last_provider_finish_reason: Option<BoundedString>,
/// Structured calls decoded from the stream, before legacy text-call parsing.
pub last_response_tool_calls: Option<usize>,
/// None means suppression was not counted at this exit boundary.
pub last_response_tool_calls_suppressed: Option<usize>,
/// Last parent response's reported input tokens, not cumulative billing.
pub last_reported_input_tokens: Option<u32>,
pub route_context_window_tokens: Option<u64>,
pub automatic_compaction_attempts: u32,
pub emergency_compaction_attempts: u32,
}
impl TurnStopDiagnostics {
pub(crate) fn observe_provider_response(
&mut self,
finish_reason: Option<&str>,
tool_calls: usize,
) {
self.last_provider_finish_reason =
finish_reason.map(|reason| bounded_chars(reason, MAX_NAME_CHARS));
self.last_response_tool_calls = Some(tool_calls);
self.last_response_tool_calls_suppressed = None;
}
}
#[derive(Debug, Clone, PartialEq, Serialize)]
#[serde(tag = "status", rename_all = "snake_case")]
pub enum Evidence<T> {
@@ -260,6 +315,8 @@ pub struct ToolInspectionSnapshot {
pub delivery_status: &'static str,
pub turn_id: BoundedString,
pub step: u32,
#[serde(skip_serializing_if = "Option::is_none")]
pub terminal: Option<TurnStopDiagnostics>,
pub tools_field_present: bool,
pub tool_count: usize,
pub rendered_tool_count: usize,
@@ -372,6 +429,7 @@ impl ToolInspectionSnapshot {
delivery_status: "unknown (capture does not prove provider delivery)",
turn_id: bounded_chars(turn_id, MAX_AUXILIARY_CHARS),
step,
terminal: None,
tools_field_present: tools.is_some(),
tool_count,
rendered_tool_count: projected.len(),
@@ -408,6 +466,13 @@ impl ToolInspectionSnapshot {
yes_no(self.turn_id.truncated)
));
out.push_str(&format!("Step: {}\n", self.step));
if let Some(terminal) = &self.terminal {
out.push_str("Terminal diagnostics (observed facts; null means unknown):\n");
if let Ok(json) = serde_json::to_string_pretty(terminal) {
out.push_str(&json);
out.push('\n');
}
}
out.push_str(&format!(
"Tools field: {}\nTool count: {}\n",
if self.tools_field_present {