Merge commit '5ef2e3a50c4d2e4861899c977bcf65f706c1789e' into opus/0101-conformance-harness

Signed-off-by: CodeWhale Bot <[email protected]>
This commit is contained in:
CodeWhale Bot committed 2026-09-30 03:15:23 -07:00
commit afe46fca0a
158 files changed
+6783 -1109

No files matched your search

+18
View File
@@ -498,6 +498,24 @@ assert.equal(
);
assert.match(release, /overwrite_files:\s*false/);
assert.match(release, /fail_on_unmatched_files:\s*true/);
// Assets land in a draft; the whole verified set becomes public at once, and
// derived channels (the tagged container manifest) follow the canonical release.
assert.match(release, /files: artifacts\/\*\n\s+draft: true\n/);
assert.match(
namedStep(release, "Verify the draft's exact asset set, then publish it at once"),
/verify-release-inventory\.js \\\n\s+--draft --asset-dir artifacts --publish/,
);
const releaseJob = release.match(/\n release:\n([\s\S]*?)\n npm:\n/);
assert.ok(releaseJob, "public release must retain its release job");
assert.match(releaseJob[1], /^ needs: \[artifacts, docker-build, resolve\]$/m);
const dockerJob = release.match(/\n docker:\n([\s\S]*?)\n release:\n/);
assert.ok(dockerJob, "public release must retain its container manifest job");
assert.match(dockerJob[1], /^ needs: \[docker-build, release, resolve\]$/m);
assert.match(dockerJob[1], /needs\.release\.result == 'success'/);
assert.match(
namedStep(republish, "Require a complete published release"),
/verify-release-inventory\.js --manifest/,
);
assert.match(release, /^ docker-build:\n/m);
assert.match(release, /^ docker:\n/m);
+1
View File
@@ -269,6 +269,7 @@ jobs:
node .github/scripts/release-workflows.test.js
node --test scripts/release/assemble-release-assets.test.js
node --test scripts/release/ensure-release-assets-absent.test.js
node --test scripts/release/verify-release-inventory.test.js
- name: Run runtime web client tests
# crates/tui/tests/runtime_web_client.test.mjs exercises the embedded
# web client's event/snapshot state machine; it ran nowhere before.
+7 -11
View File
@@ -75,21 +75,17 @@ jobs:
echo "version=${INPUT_VERSION}"
} >> "${GITHUB_OUTPUT}"
echo "Resolved ${tag} -> ${sha}"
- name: Require an existing published release
- name: Require a complete published release
# The inverse of release.yml's immutability guard. That one refuses to
# run when assets exist; this one refuses to run when they do not,
# because there is nothing to derive a channel from.
# run when assets exist; this one refuses to run unless the published
# release carries exactly the assets its own checksum manifest lists,
# each fully uploaded, because a channel derived from a partial set
# would advertise bytes that are not there. The tag may predate the
# current inventory, so its manifest, not today's list, is the check.
env:
GH_TOKEN: ${{ github.token }}
TAG: ${{ steps.release.outputs.tag }}
run: |
set -euo pipefail
count="$(gh release view "${TAG}" --repo "${GITHUB_REPOSITORY}" --json assets --jq '.assets | length')"
if [[ "${count}" -eq 0 ]]; then
echo "::error::${TAG} has no published assets. Run release.yml, not this recovery workflow." >&2
exit 1
fi
echo "${TAG} has ${count} published assets."
run: node scripts/release/verify-release-inventory.js --manifest "${GITHUB_REPOSITORY}" "${TAG}"
- name: Verify the remote tag still points at this commit
env:
EXPECTED_SHA: ${{ steps.release.outputs.sha }}
+20 -4
View File
@@ -266,10 +266,15 @@ jobs:
retention-days: 1
overwrite: true
# Derived channel: the tagged/latest manifest is published only after the
# canonical GitHub Release exists. docker-build pushes by digest (untagged)
# and smokes each image before the release, so a broken image still stops
# the release, but no public tag can point at a version whose GitHub
# Release was never published.
docker:
timeout-minutes: 30
needs: [docker-build, resolve]
if: ${{ !cancelled() && needs.docker-build.result == 'success' }}
needs: [docker-build, release, resolve]
if: ${{ !cancelled() && needs.docker-build.result == 'success' && needs.release.result == 'success' }}
runs-on: ubuntu-latest
permissions:
contents: read
@@ -373,8 +378,8 @@ jobs:
release:
timeout-minutes: 30
needs: [artifacts, docker, resolve]
if: ${{ !cancelled() && needs.artifacts.result == 'success' && needs.docker.result == 'success' }}
needs: [artifacts, docker-build, resolve]
if: ${{ !cancelled() && needs.artifacts.result == 'success' && needs.docker-build.result == 'success' }}
runs-on: ubuntu-latest
permissions:
contents: write
@@ -412,14 +417,25 @@ jobs:
GH_TOKEN: ${{ github.token }}
TAG: ${{ needs.resolve.outputs.tag }}
run: node repo/scripts/release/ensure-release-assets-absent.js "${GITHUB_REPOSITORY}" "${TAG}"
# Upload into a DRAFT: 34 sequential uploads that die halfway must leave
# nothing public (and nothing a rerun's immutability guard refuses).
- uses: softprops/action-gh-release@efb35369e0ad2afab669f228072c1b0d510eae64 # v3
with:
tag_name: ${{ needs.resolve.outputs.tag }}
files: artifacts/*
draft: true
prerelease: false
body_path: release-body.md
overwrite_files: false
fail_on_unmatched_files: true
- name: Verify the draft's exact asset set, then publish it at once
env:
GH_TOKEN: ${{ github.token }}
TAG: ${{ needs.resolve.outputs.tag }}
run: |
node repo/scripts/release/verify-release-inventory.js \
--draft --asset-dir artifacts --publish \
"${GITHUB_REPOSITORY}" "${TAG}"
npm:
timeout-minutes: 20
+1
View File
@@ -77,6 +77,7 @@ quieter, and Fleet runs can be checked before they spend anything.
### Contributors
- **[@zhuowp](https://github.com/zhuowp)** — supplied the process-scoped PowerShell execution-policy repair adapted for Codewhale, preserving machine and user Group Policy precedence ([#6745](https://github.com/Hmbown/Codewhale/issues/6745)).
- **[@gaord](https://github.com/gaord)** — let a client fork a thread at a named turn ([#6580](https://github.com/Hmbown/Codewhale/pull/6580)), let undo roll back files for the turn it is undoing ([#6483](https://github.com/Hmbown/Codewhale/pull/6483)), stopped resume and fork from duplicating threads and sessions ([#6406](https://github.com/Hmbown/Codewhale/pull/6406)), exposed user-defined provider routes to native clients ([#6404](https://github.com/Hmbown/Codewhale/pull/6404)), and kept a fork going when a turn lost its tool call ([#6664](https://github.com/Hmbown/Codewhale/pull/6664)).
- **[@Lstarsky0](https://github.com/Lstarsky0)** — moved the docs/work, legal, digest and FAQ pages onto the dictionary spine ([#6405](https://github.com/Hmbown/Codewhale/pull/6405), [#6417](https://github.com/Hmbown/Codewhale/pull/6417), [#6499](https://github.com/Hmbown/Codewhale/pull/6499), [#6574](https://github.com/Hmbown/Codewhale/pull/6574)), tightened the Chinese-branching ceiling to 18 ([#6403](https://github.com/Hmbown/Codewhale/pull/6403)), and made Fleet publish without a two-link window ([#6431](https://github.com/Hmbown/Codewhale/pull/6431)). Also moved the constitution page onto the dictionary spine and kept its install link in the selected locale ([#6733](https://github.com/Hmbown/Codewhale/pull/6733)).
- **[@aboimpinto](https://github.com/aboimpinto)** — restored a green Linux full-workspace test gate without loosening any test, twice ([#6581](https://github.com/Hmbown/Codewhale/pull/6581), [#6666](https://github.com/Hmbown/Codewhale/pull/6666)).
Generated
-2
View File
@@ -786,7 +786,6 @@ dependencies = [
"codewhale-config",
"codewhale-core",
"codewhale-hooks",
"codewhale-mcp",
"codewhale-paths",
"codewhale-protocol",
"codewhale-release",
@@ -912,7 +911,6 @@ dependencies = [
"chrono",
"codewhale-config",
"codewhale-hooks",
"codewhale-mcp",
"codewhale-protocol",
"codewhale-state",
"regex",
-1
View File
@@ -19,7 +19,6 @@ codewhale-agent = { path = "../agent", version = "0.10.1" }
codewhale-config = { path = "../config", version = "0.10.1" }
codewhale-core = { path = "../core", version = "0.10.1" }
codewhale-hooks = { path = "../hooks", version = "0.10.1" }
codewhale-mcp = { path = "../mcp", version = "0.10.1" }
codewhale-paths = { path = "../paths", version = "0.10.1" }
codewhale-protocol = { path = "../protocol", version = "0.10.1" }
codewhale-release = { path = "../release", version = "0.10.1" }
+38 -20
View File
@@ -17,7 +17,6 @@ use codewhale_agent::ModelRegistry;
use codewhale_config::ConfigStore;
use codewhale_core::Runtime;
use codewhale_hooks::{HookDispatcher, JsonlHookSink, StdoutHookSink, UnixSocketHookSink};
use codewhale_mcp::McpManager;
use codewhale_protocol::{
AppRequest, AppResponse, EventFrame, PromptRequest, PromptResponse, ResponseChannel,
ThreadGoalClearParams, ThreadGoalGetParams, ThreadGoalSetParams, ThreadRequest, ThreadResponse,
@@ -373,7 +372,7 @@ async fn shutdown_signal() {
/// `RuntimeBridge::stream_turn_events` forwards only `item.delta` and the
/// turn's completion, and there is no decision route, so approval-gated work
/// belongs on the Runtime API (`/v1/threads/*`, `POST /v1/approvals/{id}`).
const ADVERTISED_ROUTES: &[&str] = &["/thread", "/app", "/prompt", "/jobs", "/mcp/startup"];
const ADVERTISED_ROUTES: &[&str] = &["/thread", "/app", "/prompt", "/jobs"];
fn app_router(state: AppState, cors_origins: &[String]) -> Router {
let protected_routes = Router::new()
@@ -391,7 +390,6 @@ fn app_router(state: AppState, cors_origins: &[String]) -> Router {
)),
)
.route("/jobs", get(jobs_handler))
.route("/mcp/startup", post(mcp_startup_handler))
.route(
"/v1/chat/completions",
post(chat_completions::chat_completions_handler),
@@ -740,15 +738,6 @@ async fn jobs_handler(State(state): State<AppState>) -> Json<AppResponse> {
Json(runtime.app_status())
}
async fn mcp_startup_handler(State(state): State<AppState>) -> Json<Value> {
let runtime = state.runtime.read().await;
let summary = runtime.mcp_startup().await;
Json(json!({
"ok": true,
"summary": summary
}))
}
async fn app_handler(
State(state): State<AppState>,
Json(req): Json<AppRequest>,
@@ -817,12 +806,7 @@ fn build_state_with_transport(
hooks.add_sink(Arc::new(UnixSocketHookSink::new(socket_path.clone())));
}
let runtime = Runtime::new(
config.clone(),
state_store,
Arc::new(McpManager::default()),
hooks,
);
let runtime = Runtime::new(config.clone(), state_store, hooks);
Ok(AppState {
config_path,
@@ -2658,7 +2642,7 @@ async fn process_app_request(
data: json!({
"routes": ADVERTISED_ROUTES,
"config": ["get", "set", "unset", "list", "reload"],
"events": ["response_start", "response_delta", "response_end", "tool_call_start", "tool_call_result", "mcp_startup_update", "mcp_startup_complete"],
"events": ["response_start", "response_delta", "response_end", "tool_call_start", "tool_call_result"],
"transport": "stdio+http",
"config_path": state.config_path.as_ref().map(|p| p.display().to_string()),
}),
@@ -3095,6 +3079,40 @@ mod tests {
assert!(!advertised.iter().any(|route| route == "/tool"));
}
#[tokio::test]
async fn mcp_startup_route_cannot_start_a_parallel_pool() {
let (app, tmp) = app_with_config(Some("test-token"));
let response = app
.oneshot(
Request::builder()
.method(Method::POST)
.uri("/mcp/startup")
.header(header::AUTHORIZATION, "Bearer test-token")
.body(Body::empty())
.expect("request"),
)
.await
.expect("response");
assert_eq!(response.status(), StatusCode::NOT_FOUND);
let state = build_state(Some(tmp.path().join("config.toml")), None).expect("state");
let caps = process_app_request(&state, AppRequest::Capabilities, AppTransport::Http).await;
assert!(
!caps.data["routes"]
.as_array()
.expect("routes")
.iter()
.any(|route| route == "/mcp/startup")
);
assert!(
!caps.data["events"]
.as_array()
.expect("events")
.iter()
.any(|event| event == "mcp_startup_update" || event == "mcp_startup_complete")
);
}
#[tokio::test]
async fn cors_does_not_allow_arbitrary_origins() {
let (app, _tmp) = app_with_config(Some("test-token"));
@@ -3953,7 +3971,7 @@ mod tests {
let state = build_state(Some(config_path.clone()), None).expect("state");
*state.runtime_bridge.lock().await = Some(sentinel_bridge());
// A long reader (e.g. `/mcp/startup`) holds the runtime, so the set
// A concurrent reader holds the runtime, so the set
// saves to disk and then waits to propagate.
let runtime_reader = state.runtime.read().await;
let request = process_app_request(
-1
View File
@@ -17,7 +17,6 @@ serde.workspace = true
thiserror.workspace = true
codewhale-config = { path = "../config", version = "0.10.1" }
codewhale-hooks = { path = "../hooks", version = "0.10.1" }
codewhale-mcp = { path = "../mcp", version = "0.10.1" }
codewhale-protocol = { path = "../protocol", version = "0.10.1" }
codewhale-state = { path = "../state", version = "0.10.1" }
regex = "1.11"
+3 -66
View File
@@ -16,14 +16,10 @@ pub use context_reference::{
use std::collections::{HashMap, HashSet};
use std::path::{Path, PathBuf};
use std::sync::Arc;
use anyhow::{Result, anyhow};
use codewhale_config::{ConfigToml, ProviderKind};
use codewhale_hooks::{HookDispatcher, HookEvent};
use codewhale_mcp::{
McpManager, McpStartupCompleteEvent, McpStartupStatus as McpManagerStartupStatus,
};
use codewhale_hooks::HookDispatcher;
use codewhale_protocol::{
AppResponse, EventFrame, ResponseChannel, Status, Thread, ThreadForkParams, ThreadGoal,
ThreadGoalClearParams, ThreadGoalGetParams, ThreadGoalProgressParams, ThreadGoalSetParams,
@@ -922,7 +918,7 @@ impl ThreadManager {
}
}
/// Top-level headless runtime combining config, threads, MCP, and hooks.
/// Headless bookkeeping combining config, threads, jobs, and hooks.
///
/// It does not execute tools and holds no approval policy: the Engine behind
/// the app-server's runtime bridge is the only tool and approval authority.
@@ -931,8 +927,6 @@ pub struct Runtime {
pub config: ConfigToml,
/// Manages conversation thread lifecycle.
pub thread_manager: ThreadManager,
/// Manager for MCP server connections.
pub mcp_manager: Arc<McpManager>,
/// Dispatcher for lifecycle hooks.
pub hooks: HookDispatcher,
/// Manager for background job lifecycle.
@@ -941,12 +935,7 @@ pub struct Runtime {
impl Runtime {
/// Constructs a new `Runtime`, loading existing jobs from the state store.
pub fn new(
config: ConfigToml,
state: StateStore,
mcp_manager: Arc<McpManager>,
hooks: HookDispatcher,
) -> Self {
pub fn new(config: ConfigToml, state: StateStore, hooks: HookDispatcher) -> Self {
let mut jobs = JobManager::default();
if let Err(e) = jobs.load_from_store(&state) {
tracing::warn!("Failed to load job store, starting with empty job list: {e}");
@@ -954,7 +943,6 @@ impl Runtime {
Self {
config,
thread_manager: ThreadManager::new(state),
mcp_manager,
hooks,
jobs,
}
@@ -1308,55 +1296,6 @@ impl Runtime {
}
}
/// Starts all configured MCP servers and emits startup events via hooks.
pub async fn mcp_startup(&self) -> McpStartupCompleteEvent {
let mut updates = Vec::new();
let summary = self.mcp_manager.start_all(|update| {
updates.push(update);
});
for update in updates {
let status = match update.status {
McpManagerStartupStatus::Starting => codewhale_protocol::McpStartupStatus::Starting,
McpManagerStartupStatus::Ready => codewhale_protocol::McpStartupStatus::Ready,
McpManagerStartupStatus::Failed { error } => {
codewhale_protocol::McpStartupStatus::Failed { error }
}
McpManagerStartupStatus::Cancelled => {
codewhale_protocol::McpStartupStatus::Cancelled
}
};
self.hooks
.emit(HookEvent::GenericEventFrame {
frame: Box::new(EventFrame::McpStartupUpdate {
update: codewhale_protocol::McpStartupUpdateEvent {
server_name: update.server_name,
status,
},
}),
})
.await;
}
self.hooks
.emit(HookEvent::GenericEventFrame {
frame: Box::new(EventFrame::McpStartupComplete {
summary: codewhale_protocol::McpStartupCompleteEvent {
ready: summary.ready.clone(),
failed: summary
.failed
.iter()
.map(|f| codewhale_protocol::McpStartupFailure {
server_name: f.server_name.clone(),
error: f.error.clone(),
})
.collect(),
cancelled: summary.cancelled.clone(),
},
}),
})
.await;
summary
}
/// Returns the current application status including all jobs and their history.
pub fn app_status(&self) -> AppResponse {
let jobs = self.jobs.list();
@@ -2591,7 +2530,6 @@ mod tests {
let mut runtime = Runtime::new(
ConfigToml::default(),
temp_core_state("archive-unknown"),
Arc::new(McpManager::default()),
HookDispatcher::default(),
);
for request in [
@@ -2641,7 +2579,6 @@ mod tests {
let mut runtime = Runtime::new(
ConfigToml::default(),
temp_core_state("message-refused"),
Arc::new(McpManager::default()),
HookDispatcher::default(),
);
let spawned = runtime
+4
View File
@@ -1535,6 +1535,10 @@
"SessionsSortSize": "mida",
"SessionsSearchPrompt": "Cerca: escriu per filtrar, Enter per aplicar",
"SessionsDeleteFailed": "L'eliminació ha fallat: {error}",
"SessionsDeleteOpenHere": "No es pot eliminar la sessió oberta aquí. Canvia primer a una altra sessió.",
"SessionsDeleteOpenElsewhere": "No es pot eliminar: aquesta sessió està oberta en una altra finestra de Codewhale.",
"SessionSaveFailed": "No s'ha pogut desar la sessió {id} ({error}). És possible que la feina recent no estigui desada.",
"SessionSaveFailedAtExit": "codewhale: l'últim desament de la sessió {id} ha fallat ({error}); és possible que la feina recent no s'hagi desat.",
"SessionsDeleted": "Sessió {id} eliminada",
"SessionsNoSelection": "Cap sessió seleccionada",
"SessionsTitleLength": "El títol ha de tenir entre 1 i 100 caràcters",
+4
View File
@@ -1535,6 +1535,10 @@
"SessionsSortSize": "Größe",
"SessionsSearchPrompt": "Suche: tippen zum Filtern, Enter zum Anwenden",
"SessionsDeleteFailed": "Löschen fehlgeschlagen: {error}",
"SessionsDeleteOpenHere": "Die hier geöffnete Sitzung kann nicht gelöscht werden. Wechsle zuerst zu einer anderen Sitzung.",
"SessionsDeleteOpenElsewhere": "Löschen nicht möglich: Diese Sitzung ist in einem anderen Codewhale-Fenster geöffnet.",
"SessionSaveFailed": "Sitzung {id} konnte nicht gespeichert werden ({error}). Neuere Arbeit ist möglicherweise nicht gespeichert.",
"SessionSaveFailedAtExit": "codewhale: Das letzte Speichern der Sitzung {id} ist fehlgeschlagen ({error}); neuere Arbeit wurde möglicherweise nicht gespeichert.",
"SessionsDeleted": "Sitzung {id} gelöscht",
"SessionsNoSelection": "Keine Sitzung ausgewählt",
"SessionsTitleLength": "Titel muss 1–100 Zeichen haben",
+4
View File
@@ -1558,6 +1558,10 @@
"SessionsSortSize": "size",
"SessionsSearchPrompt": "Search: type, Enter to apply",
"SessionsDeleteFailed": "Could not delete: {error}",
"SessionsDeleteOpenHere": "Can't delete the session open here. Switch to another session first.",
"SessionsDeleteOpenElsewhere": "Can't delete: this session is open in another Codewhale window.",
"SessionSaveFailed": "Session {id} could not be saved ({error}). Recent work may not be saved.",
"SessionSaveFailedAtExit": "codewhale: the latest save of session {id} failed ({error}); recent work may not have been saved.",
"SessionsDeleted": "Deleted {id}",
"SessionsNoSelection": "No session selected",
"SessionsTitleLength": "Title must be 1–100 characters",
+4
View File
@@ -1558,6 +1558,10 @@
"SessionsSortSize": "tamaño",
"SessionsSearchPrompt": "Buscar: escribe para filtrar, Enter para aplicar",
"SessionsDeleteFailed": "Error al eliminar: {error}",
"SessionsDeleteOpenHere": "No se puede eliminar la sesión abierta aquí. Cambia primero a otra sesión.",
"SessionsDeleteOpenElsewhere": "No se puede eliminar: esta sesión está abierta en otra ventana de Codewhale.",
"SessionSaveFailed": "No se pudo guardar la sesión {id} ({error}). Es posible que el trabajo reciente no esté guardado.",
"SessionSaveFailedAtExit": "codewhale: el último guardado de la sesión {id} falló ({error}); es posible que el trabajo reciente no se haya guardado.",
"SessionsDeleted": "Sesión {id} eliminada",
"SessionsNoSelection": "Ninguna sesión seleccionada",
"SessionsTitleLength": "El título debe tener entre 1 y 100 caracteres",
+4
View File
@@ -1535,6 +1535,10 @@
"SessionsSortSize": "taille",
"SessionsSearchPrompt": "Recherche : tapez pour filtrer, Enter pour appliquer",
"SessionsDeleteFailed": "Échec de la suppression : {error}",
"SessionsDeleteOpenHere": "Impossible de supprimer la session ouverte ici. Passez d'abord à une autre session.",
"SessionsDeleteOpenElsewhere": "Suppression impossible : cette session est ouverte dans une autre fenêtre Codewhale.",
"SessionSaveFailed": "Impossible d'enregistrer la session {id} ({error}). Le travail récent n'est peut-être pas enregistré.",
"SessionSaveFailedAtExit": "codewhale : le dernier enregistrement de la session {id} a échoué ({error}) ; le travail récent n'a peut-être pas été enregistré.",
"SessionsDeleted": "Session {id} supprimée",
"SessionsNoSelection": "Aucune session sélectionnée",
"SessionsTitleLength": "Le titre doit faire 1 à 100 caractères",
+4
View File
@@ -1535,6 +1535,10 @@
"SessionsSortSize": "आकार",
"SessionsSearchPrompt": "खोज: फ़िल्टर के लिए टाइप करें, लागू के लिए Enter",
"SessionsDeleteFailed": "हटाना विफल: {error}",
"SessionsDeleteOpenHere": "यहाँ खुला सत्र हटाया नहीं जा सकता। पहले किसी दूसरे सत्र पर जाएँ।",
"SessionsDeleteOpenElsewhere": "हटाया नहीं जा सकता: यह सत्र किसी दूसरी Codewhale विंडो में खुला है।",
"SessionSaveFailed": "सत्र {id} सहेजा नहीं जा सका ({error})। हाल का काम शायद सहेजा नहीं गया है।",
"SessionSaveFailedAtExit": "codewhale: सत्र {id} को पिछली बार सहेजना विफल रहा ({error}); हाल का काम शायद सहेजा नहीं गया।",
"SessionsDeleted": "सत्र {id} हटाया गया",
"SessionsNoSelection": "कोई सत्र चुना नहीं गया",
"SessionsTitleLength": "शीर्षक 1–100 अक्षरों का होना चाहिए",
+4
View File
@@ -1535,6 +1535,10 @@
"SessionsSortSize": "ukuran",
"SessionsSearchPrompt": "Cari: ketik untuk memfilter, Enter untuk menerapkan",
"SessionsDeleteFailed": "Hapus gagal: {error}",
"SessionsDeleteOpenHere": "Sesi yang terbuka di sini tidak dapat dihapus. Beralih ke sesi lain terlebih dahulu.",
"SessionsDeleteOpenElsewhere": "Tidak dapat dihapus: sesi ini terbuka di jendela Codewhale lain.",
"SessionSaveFailed": "Sesi {id} tidak dapat disimpan ({error}). Pekerjaan terbaru mungkin belum tersimpan.",
"SessionSaveFailedAtExit": "codewhale: penyimpanan terakhir sesi {id} gagal ({error}); pekerjaan terbaru mungkin belum tersimpan.",
"SessionsDeleted": "Sesi {id} dihapus",
"SessionsNoSelection": "Tidak ada sesi terpilih",
"SessionsTitleLength": "Judul harus 1–100 karakter",
+4
View File
@@ -1558,6 +1558,10 @@
"SessionsSortSize": "サイズ",
"SessionsSearchPrompt": "検索: 入力して絞り込み、Enter で適用",
"SessionsDeleteFailed": "削除に失敗しました: {error}",
"SessionsDeleteOpenHere": "ここで開いているセッションは削除できません。先に別のセッションに切り替えてください。",
"SessionsDeleteOpenElsewhere": "削除できません: このセッションは別の Codewhale ウィンドウで開かれています。",
"SessionSaveFailed": "セッション {id} を保存できませんでした ({error})。最近の作業が保存されていない可能性があります。",
"SessionSaveFailedAtExit": "codewhale: セッション {id} の最後の保存に失敗しました ({error})。最近の作業が保存されていない可能性があります。",
"SessionsDeleted": "セッション {id} を削除しました",
"SessionsNoSelection": "セッションが選択されていません",
"SessionsTitleLength": "タイトルは 1〜100 文字にしてください",
+4
View File
@@ -1558,6 +1558,10 @@
"SessionsSortSize": "크기",
"SessionsSearchPrompt": "검색: 입력하여 필터, Enter로 적용",
"SessionsDeleteFailed": "삭제 실패: {error}",
"SessionsDeleteOpenHere": "여기에서 열려 있는 세션은 삭제할 수 없습니다. 먼저 다른 세션으로 전환하세요.",
"SessionsDeleteOpenElsewhere": "삭제할 수 없음: 이 세션은 다른 Codewhale 창에서 열려 있습니다.",
"SessionSaveFailed": "세션 {id}을(를) 저장하지 못했습니다 ({error}). 최근 작업이 저장되지 않았을 수 있습니다.",
"SessionSaveFailedAtExit": "codewhale: 세션 {id}의 마지막 저장에 실패했습니다 ({error}). 최근 작업이 저장되지 않았을 수 있습니다.",
"SessionsDeleted": "세션 {id} 삭제됨",
"SessionsNoSelection": "선택된 세션 없음",
"SessionsTitleLength": "제목은 1–100자여야 합니다",
+4
View File
@@ -1558,6 +1558,10 @@
"SessionsSortSize": "tamanho",
"SessionsSearchPrompt": "Buscar: digite para filtrar, Enter para aplicar",
"SessionsDeleteFailed": "Falha ao excluir: {error}",
"SessionsDeleteOpenHere": "Não é possível excluir a sessão aberta aqui. Mude para outra sessão primeiro.",
"SessionsDeleteOpenElsewhere": "Não é possível excluir: esta sessão está aberta em outra janela do Codewhale.",
"SessionSaveFailed": "Não foi possível salvar a sessão {id} ({error}). O trabalho recente pode não estar salvo.",
"SessionSaveFailedAtExit": "codewhale: o último salvamento da sessão {id} falhou ({error}); o trabalho recente pode não ter sido salvo.",
"SessionsDeleted": "Sessão {id} excluída",
"SessionsNoSelection": "Nenhuma sessão selecionada",
"SessionsTitleLength": "O título deve ter de 1 a 100 caracteres",
+4
View File
@@ -1535,6 +1535,10 @@
"SessionsSortSize": "размер",
"SessionsSearchPrompt": "Поиск: введите для фильтра, Enter — применить",
"SessionsDeleteFailed": "Ошибка удаления: {error}",
"SessionsDeleteOpenHere": "Нельзя удалить сессию, открытую здесь. Сначала переключитесь на другую сессию.",
"SessionsDeleteOpenElsewhere": "Удаление невозможно: эта сессия открыта в другом окне Codewhale.",
"SessionSaveFailed": "Не удалось сохранить сессию {id} ({error}). Последние изменения могут быть не сохранены.",
"SessionSaveFailedAtExit": "codewhale: последнее сохранение сессии {id} не удалось ({error}); последние изменения могли не сохраниться.",
"SessionsDeleted": "Сессия {id} удалена",
"SessionsNoSelection": "Сессия не выбрана",
"SessionsTitleLength": "Заголовок должен быть от 1 до 100 символов",
+4
View File
@@ -1535,6 +1535,10 @@
"SessionsSortSize": "розмір",
"SessionsSearchPrompt": "Пошук: вводьте для фільтрації, Enter — застосувати",
"SessionsDeleteFailed": "Видалення не вдалося: {error}",
"SessionsDeleteOpenHere": "Не можна видалити сесію, відкриту тут. Спочатку перейдіть до іншої сесії.",
"SessionsDeleteOpenElsewhere": "Видалення неможливе: ця сесія відкрита в іншому вікні Codewhale.",
"SessionSaveFailed": "Не вдалося зберегти сесію {id} ({error}). Останні зміни можуть бути не збережені.",
"SessionSaveFailedAtExit": "codewhale: останнє збереження сесії {id} не вдалося ({error}); останні зміни могли не зберегтися.",
"SessionsDeleted": "Сесію {id} видалено",
"SessionsNoSelection": "Сесію не вибрано",
"SessionsTitleLength": "Назва має містити 1–100 символів",
+4
View File
@@ -1558,6 +1558,10 @@
"SessionsSortSize": "kích thước",
"SessionsSearchPrompt": "Tìm: gõ để lọc, Enter để áp dụng",
"SessionsDeleteFailed": "Xóa thất bại: {error}",
"SessionsDeleteOpenHere": "Không thể xóa phiên đang mở ở đây. Hãy chuyển sang phiên khác trước.",
"SessionsDeleteOpenElsewhere": "Không thể xóa: phiên này đang mở trong một cửa sổ Codewhale khác.",
"SessionSaveFailed": "Không thể lưu phiên {id} ({error}). Công việc gần đây có thể chưa được lưu.",
"SessionSaveFailedAtExit": "codewhale: lần lưu gần nhất của phiên {id} thất bại ({error}); công việc gần đây có thể chưa được lưu.",
"SessionsDeleted": "Đã xóa phiên {id}",
"SessionsNoSelection": "Chưa chọn phiên",
"SessionsTitleLength": "Tiêu đề phải từ 1–100 ký tự",
+5 -3
View File
@@ -1558,6 +1558,10 @@
"SessionsSortSize": "大小",
"SessionsSearchPrompt": "搜索:输入筛选,Enter 应用",
"SessionsDeleteFailed": "删除失败:{error}",
"SessionsDeleteOpenHere": "无法删除此处打开的会话。请先切换到其他会话。",
"SessionsDeleteOpenElsewhere": "无法删除:此会话已在另一个 Codewhale 窗口中打开。",
"SessionSaveFailed": "无法保存会话 {id}({error})。最近的工作可能尚未保存。",
"SessionSaveFailedAtExit": "codewhale: 会话 {id} 的最近一次保存失败({error});最近的工作可能尚未保存。",
"SessionsDeleted": "已删除会话 {id}",
"SessionsNoSelection": "未选择会话",
"SessionsTitleLength": "标题长度须为 1–100 个字符",
@@ -2434,7 +2438,5 @@
"ModelPickerPinReorderFailed": "无法调整固定顺序:{error}",
"ModelPickerReadinessRefreshed": "已刷新模型就绪状态 · 已重建目录行",
"ModelPickerOpenToRefresh": "打开 /model 以刷新就绪状态和目录",
"ModelPickerPinnedChip": "已固定",
"TranscriptThought": "思考完毕",
"TranscriptThoughtFor": "思考用时 {duration}"
"ModelPickerPinnedChip": "已固定"
}
+4
View File
@@ -1556,6 +1556,10 @@
"SessionsSortSize": "大小",
"SessionsSearchPrompt": "搜尋:輸入篩選,Enter 套用",
"SessionsDeleteFailed": "刪除失敗:{error}",
"SessionsDeleteOpenHere": "無法刪除此處開啟的工作階段。請先切換到其他工作階段。",
"SessionsDeleteOpenElsewhere": "無法刪除:此工作階段已在另一個 Codewhale 視窗中開啟。",
"SessionSaveFailed": "無法儲存工作階段 {id}({error})。最近的工作可能尚未儲存。",
"SessionSaveFailedAtExit": "codewhale: 工作階段 {id} 的最近一次儲存失敗({error});最近的工作可能尚未儲存。",
"SessionsDeleted": "已刪除工作階段 {id}",
"SessionsNoSelection": "未選擇工作階段",
"SessionsTitleLength": "標題長度須為 1–100 個字元",
+8
View File
@@ -1940,6 +1940,10 @@ pub enum MessageId {
SessionsSortSize,
SessionsSearchPrompt,
SessionsDeleteFailed,
SessionsDeleteOpenHere,
SessionsDeleteOpenElsewhere,
SessionSaveFailed,
SessionSaveFailedAtExit,
SessionsDeleted,
SessionsNoSelection,
SessionsTitleLength,
@@ -4415,6 +4419,10 @@ pub const ALL_MESSAGE_IDS: &[MessageId] = &[
MessageId::SessionsSortSize,
MessageId::SessionsSearchPrompt,
MessageId::SessionsDeleteFailed,
MessageId::SessionsDeleteOpenHere,
MessageId::SessionsDeleteOpenElsewhere,
MessageId::SessionSaveFailed,
MessageId::SessionSaveFailedAtExit,
MessageId::SessionsDeleted,
MessageId::SessionsNoSelection,
MessageId::SessionsTitleLength,
+65 -12
View File
@@ -1,10 +1,22 @@
//! Compiled-regex cache for patterns a user or model supplies (tool search,
//! purge). Those patterns are untrusted input, so compilation is bounded:
//! the pattern length is capped, the compiled program and lazy DFA have
//! explicit size limits, compilation runs outside the cache lock, and a
//! rejected pattern is remembered so repeating it costs a lookup.
use std::num::NonZeroUsize;
use std::sync::{Mutex, OnceLock};
use lru::LruCache;
use regex::Regex;
use regex::{Regex, RegexBuilder};
const DEFAULT_USER_REGEX_CACHE_CAPACITY: usize = 64;
/// Longest pattern accepted, in bytes. Search and purge patterns are short.
pub const MAX_USER_REGEX_PATTERN_BYTES: usize = 4 * 1024;
/// Compiled program limit (the `regex` default is 10 MiB).
const USER_REGEX_SIZE_LIMIT: usize = 1024 * 1024;
/// Lazy DFA cache limit per matcher.
const USER_REGEX_DFA_SIZE_LIMIT: usize = 1024 * 1024;
static USER_REGEX_CACHE: OnceLock<UserRegexCache> = OnceLock::new();
@@ -16,8 +28,23 @@ fn user_regex_cache() -> &'static UserRegexCache {
USER_REGEX_CACHE.get_or_init(UserRegexCache::new)
}
fn compile_bounded(pattern: &str) -> Result<Regex, regex::Error> {
if pattern.len() > MAX_USER_REGEX_PATTERN_BYTES {
return Err(regex::Error::Syntax(format!(
"pattern is {} bytes; the limit is {MAX_USER_REGEX_PATTERN_BYTES}",
pattern.len()
)));
}
RegexBuilder::new(pattern)
.size_limit(USER_REGEX_SIZE_LIMIT)
.dfa_size_limit(USER_REGEX_DFA_SIZE_LIMIT)
.build()
}
struct UserRegexCache {
inner: Mutex<LruCache<String, Regex>>,
/// Compiled patterns and rejected ones alike, so a repeated bad pattern
/// is not recompiled.
inner: Mutex<LruCache<String, Result<Regex, regex::Error>>>,
}
impl UserRegexCache {
@@ -34,16 +61,22 @@ impl UserRegexCache {
}
fn compile(&self, pattern: &str) -> Result<Regex, regex::Error> {
let Ok(mut cache) = self.inner.lock() else {
return Regex::new(pattern);
};
if let Some(regex) = cache.get(pattern) {
return Ok(regex.clone());
// An over-long pattern is refused before it is hashed or stored.
if pattern.len() > MAX_USER_REGEX_PATTERN_BYTES {
return compile_bounded(pattern);
}
let regex = Regex::new(pattern)?;
cache.put(pattern.to_string(), regex.clone());
Ok(regex)
if let Ok(mut cache) = self.inner.lock()
&& let Some(cached) = cache.get(pattern)
{
return cached.clone();
}
// Compile without the lock: one slow pattern must not stall every
// other caller. Two racing misses both compile; the result is equal.
let compiled = compile_bounded(pattern);
if let Ok(mut cache) = self.inner.lock() {
cache.put(pattern.to_string(), compiled.clone());
}
compiled
}
#[cfg(test)]
@@ -88,11 +121,31 @@ mod tests {
}
#[test]
fn invalid_pattern_is_not_cached() {
fn invalid_pattern_is_remembered_as_invalid() {
let cache = UserRegexCache::with_capacity(NonZeroUsize::new(2).unwrap());
assert!(cache.compile("[").is_err());
assert!(cache.contains("["));
assert!(cache.compile("[").is_err());
assert_eq!(cache.len(), 1);
}
#[test]
fn oversized_patterns_are_refused_without_being_cached() {
let cache = UserRegexCache::with_capacity(NonZeroUsize::new(2).unwrap());
let long = "a".repeat(MAX_USER_REGEX_PATTERN_BYTES + 1);
assert!(cache.compile(&long).is_err());
assert_eq!(cache.len(), 0);
// Within the length cap, a pattern whose compiled form exceeds the
// program limit is refused rather than built.
let explosive = r"(?:\w{100}){100}";
assert!(explosive.len() <= MAX_USER_REGEX_PATTERN_BYTES);
assert!(matches!(
cache.compile(explosive),
Err(regex::Error::CompiledTooBig(_))
));
}
}
+56 -25
View File
@@ -23,10 +23,20 @@
//! supposed to be invisible. If a future feature ever needs per-engine
//! retry surfaces, swap this for an `Arc<RwLock<...>>` carried on the
//! `EngineHandle`; the public API stays the same.
//!
//! The `Retry-After` pause is process-wide too, but keyed by provider scope:
//! every request to the rate-limited route waits it out, and requests to any
//! other route (another provider, a local runtime) do not.
use std::collections::HashMap;
use std::sync::{Mutex, OnceLock};
use std::time::{Duration, Instant};
/// Rate-limit pause deadlines, one per provider scope (see
/// [`note_rate_limit`]). A 429 from one provider must not stall requests to
/// another provider, a local runtime, or a sub-agent on a different route.
type RateLimitPauses = HashMap<String, Instant>;
/// One in-flight retry attempt. `deadline` is the wall-clock time the
/// next request will fire — the UI subtracts `Instant::now()` from it
/// to render a live countdown.
@@ -97,10 +107,10 @@ fn with_state<R>(f: impl FnOnce(&mut RetryState) -> R) -> R {
}
#[cfg(not(any(test, all(feature = "test-thread-scoped-state", debug_assertions))))]
fn with_rate_limit<R>(f: impl FnOnce(&mut Option<Instant>) -> R) -> R {
static STATE: OnceLock<Mutex<Option<Instant>>> = OnceLock::new();
fn with_rate_limit<R>(f: impl FnOnce(&mut RateLimitPauses) -> R) -> R {
static STATE: OnceLock<Mutex<RateLimitPauses>> = OnceLock::new();
let mut state = STATE
.get_or_init(|| Mutex::new(None))
.get_or_init(|| Mutex::new(HashMap::new()))
.lock()
.unwrap_or_else(|error| error.into_inner());
f(&mut state)
@@ -140,13 +150,11 @@ fn with_state<R>(f: impl FnOnce(&mut RetryState) -> R) -> R {
}
#[cfg(any(test, all(feature = "test-thread-scoped-state", debug_assertions)))]
fn with_rate_limit<R>(f: impl FnOnce(&mut Option<Instant>) -> R) -> R {
#[allow(clippy::type_complexity)]
static STATE: OnceLock<
Mutex<std::collections::HashMap<std::thread::ThreadId, Option<Instant>>>,
> = OnceLock::new();
fn with_rate_limit<R>(f: impl FnOnce(&mut RateLimitPauses) -> R) -> R {
static STATE: OnceLock<Mutex<HashMap<std::thread::ThreadId, RateLimitPauses>>> =
OnceLock::new();
let mut by_thread = STATE
.get_or_init(|| Mutex::new(std::collections::HashMap::new()))
.get_or_init(|| Mutex::new(HashMap::new()))
.lock()
.unwrap_or_else(|error| error.into_inner());
f(by_thread.entry(std::thread::current().id()).or_default())
@@ -160,26 +168,32 @@ pub fn snapshot() -> RetryState {
with_state(|state| state.clone())
}
/// Extend the provider-wide rate-limit pause window. This is separate from
/// the footer banner so one successful concurrent request cannot clear another
/// request's active `Retry-After` window.
pub fn note_rate_limit(delay: Duration) {
let deadline = Instant::now() + delay;
with_rate_limit(|current| {
if current.is_none_or(|existing| existing < deadline) {
*current = Some(deadline);
/// Extend the rate-limit pause window for one provider `scope` (the caller's
/// route identity and host). This is separate from the footer banner so one
/// successful concurrent request cannot clear another request's active
/// `Retry-After` window, and it is keyed so a 429 from one provider never
/// pauses requests to another. Expired scopes are dropped here, so the map
/// holds at most the providers currently rate limited.
pub fn note_rate_limit(scope: &str, delay: Duration) {
let now = Instant::now();
let deadline = now + delay;
with_rate_limit(|pauses| {
pauses.retain(|_, existing| *existing > now);
let current = pauses.entry(scope.to_string()).or_insert(deadline);
if *current < deadline {
*current = deadline;
}
});
}
/// Remaining provider-wide rate-limit pause, if any.
/// Remaining rate-limit pause for `scope`, if any.
#[must_use]
pub fn rate_limit_remaining() -> Option<Duration> {
pub fn rate_limit_remaining(scope: &str) -> Option<Duration> {
let now = Instant::now();
with_rate_limit(|current| match *current {
with_rate_limit(|pauses| match pauses.get(scope).copied() {
Some(deadline) if deadline > now => Some(deadline.duration_since(now)),
Some(_) => {
*current = None;
pauses.remove(scope);
None
}
None => None,
@@ -220,9 +234,10 @@ pub fn clear() {
with_state(|state| *state = RetryState::Idle);
}
/// Drop every scope's rate-limit pause.
#[cfg(any(test, feature = "test-support"))]
pub fn clear_rate_limit() {
with_rate_limit(|current| *current = None);
with_rate_limit(HashMap::clear);
}
/// Test helper: serialize tests that touch the global state so cargo's
@@ -300,13 +315,29 @@ mod tests {
#[test]
fn rate_limit_deadline_survives_banner_clear() {
let _g = setup();
note_rate_limit(Duration::from_secs(5));
note_rate_limit("provider-a", Duration::from_secs(5));
start(1, Duration::from_secs(5), "rate limited");
succeeded();
assert!(
rate_limit_remaining().is_some(),
"provider-wide rate limit pause must not be cleared by an unrelated success"
rate_limit_remaining("provider-a").is_some(),
"provider rate limit pause must not be cleared by an unrelated success"
);
clear_rate_limit();
}
#[test]
fn one_providers_rate_limit_does_not_pause_another() {
let _g = setup();
note_rate_limit("[email protected]", Duration::from_secs(30));
assert!(rate_limit_remaining("[email protected]").is_some());
assert_eq!(rate_limit_remaining("[email protected]:11434"), None);
// A shorter Retry-After never shortens an existing window.
note_rate_limit("[email protected]", Duration::from_secs(1));
assert!(
rate_limit_remaining("[email protected]").unwrap() > Duration::from_secs(20)
);
clear_rate_limit();
assert_eq!(rate_limit_remaining("[email protected]"), None);
}
}
+14 -2
View File
@@ -901,6 +901,11 @@ async function callTool(params) {
try { teardown = await destroyDockerComputer(entry); }
catch (err) { teardown = { destroyed: false, cleanup_error: err.message ?? String(err) }; }
}
if (teardown?.reason === "other_session") {
// Another live (or crashed) MCP session owns this desktop. Leave its
// container and registry entry alone; the owner removes both at exit.
return { content: [{ type: "text", text: JSON.stringify(fail(entry, "computer_owned_elsewhere", `computer "${entry.id}" was spawned by another session and is still its disposable desktop — it is removed when that session ends. If that session is gone, remove the container with docker rm -f ${entry.container}.`)) }], isError: true };
}
const res = registry.remove(args.computer);
if (activeComputerId === args.computer) activeComputerId = "local";
res.active = activeComputerId;
@@ -1107,7 +1112,12 @@ async function callTool(params) {
const timeoutMs = backendMethod.startsWith("recording") || backendMethod === "get_app_state" ? 60_000 : 30_000;
const invoke = async (tool, a) => {
const r = await remoteCall({ tool, args: a }, { timeoutMs });
if (!r.ok) throw new ServerError(r.error?.code ?? "remote_error", r.error?.message ?? "remote agent failed");
if (!r.ok) {
const error = new ServerError(r.error?.code ?? "remote_error", r.error?.message ?? "remote agent failed");
// The helper's backend reported that input may already have landed.
if (r.error?.request_dispatched === true) error.requestDispatched = true;
throw error;
}
return r.data;
};
data = name === "type" ? await invokeType(invoke, wireArgs) : await invoke(backendMethod, wireArgs);
@@ -1246,7 +1256,9 @@ async function callTool(params) {
// A failed open_application cleared the backend's input binding before it
// attempted anything — the tracked bound app must not claim otherwise.
if (name === "open_application") boundApps.delete(computer.id);
let outcomeUnknown = !!err.requestDispatched;
// A local backend that timed out or was cancelled after posting input
// (inputMayHaveBeenSent) is as unknown as a transport that lost the reply.
let outcomeUnknown = !!(err.requestDispatched || err.inputMayHaveBeenSent);
if (dispatched && !outcomeUnknown) {
// A transport/backend can fail after delivering input. Reconcile its
// captured route on failure too, without replacing the original error
@@ -30,7 +30,7 @@ Never retry a refusal unchanged — re-observe, re-target, or change route.
| `menu_item_not_found` | exact title not present (menus expose items only while open) | check the exact title; an ellipsis is part of it |
| `menu_item_disabled` | item present but the app refuses it right now (often a missing key window) | use the window's own control element instead |
| `app_not_found` | selector missed — `open_application` names/bundle ids that resolve nowhere and dead pids report it too | `list_apps` (or `all:true`) for exact names/pids |
| `ambiguous_application` | `kill_app` name matched several running apps | pass `pid` to choose one |
| `ambiguous_application` | an app name or bundle id matched several running apps (observation, input or `kill_app`) | pass `pid` to choose one |
| `protected_application` | the target is the Computer Use helper or its host | name the intended app instead; these cannot be terminated through the plugin |
| `browser_not_running` | browser action before `browser {action:"start"}` (or the browser went away) | start it; a closed CDP connection clears the session state |
| `browser_not_installed` | no Chromium-family browser found | install one, or set `CODEWHALE_CU_BROWSER_APP` to the app path |
@@ -56,6 +56,7 @@ Never retry a refusal unchanged — re-observe, re-target, or change route.
| `spawn_failed` | provisioning failed or the desktop did not become ready | read the message; the failed container is removed automatically — fix the cause and spawn again |
| `invalid_container` | a docker registry entry lacks a valid container name | register it through `computer spawn`, never by hand |
| `cleanup_failed` | `docker rm` failed while tearing down a spawned computer | the registry entry is still removed; check `docker ps` for the labeled container and remove it manually |
| `computer_owned_elsewhere` | `computer remove` named a desktop another session spawned | leave it; its session removes it at exit (the message names the container if that session is gone) |
| `script_error` | osascript exited non-zero; stderr is in the message | read the error, check the app's scripting dictionary (`sdef`), fix the script |
| `script_timeout` | the script — or a consent dialog — was still open at the deadline | narrow the script; a consent prompt is the person's choice, report it |
| `script_cancelled` | the script's own dialog was cancelled (-128) | the user declined in-app; stop or ask |
@@ -75,5 +76,7 @@ computer; a typing pause does not authorize foreground control.
not task success. Verify the effect with a fresh observation.
- `front_lease` / `front_restored` describe focus accounting for window-record
deliveries in explicitly authorized foreground mode. `front_restored:false` is a person-visible event: say it out loud.
- `input_may_have_been_sent` on an error means the press left before the
failure: observe the target before doing anything else.
- `outcome_unknown: true` (with `request_dispatched: true`) on an error means
the action was already handed to the helper, remote agent or browser when it
timed out, was cancelled or lost its reply: observe the target before doing
anything else, and never retry the action blindly.
@@ -226,7 +226,7 @@ export async function handle(req, { computerId = "local", sessionId = "direct",
return { ok: true, platform: process.platform, tool, data };
}));
} catch (err) {
return { ok: false, platform: process.platform, tool, error: { code: err?.code ?? "tool_error", message: String(err?.message ?? err) } };
return { ok: false, platform: process.platform, tool, error: { code: err?.code ?? "tool_error", message: String(err?.message ?? err), ...(err?.inputMayHaveBeenSent || err?.requestDispatched ? { request_dispatched: true } : {}) } };
} finally {
signal?.removeEventListener("abort", abort);
session.requests.delete(controller);
@@ -74,27 +74,37 @@ export function writeRegistration(reg) {
fs.writeFileSync(registrationPath(), JSON.stringify({ ...reg, registeredAt: new Date().toISOString() }, null, 2) + "\n");
}
/** Send one request to the app and await its single-line reply. */
/**
* Send one request to the app and await its single-line reply.
*
* Once the request line is written the helper may already have acted on it,
* so every failure after that point (timeout, cancel, a dropped connection, a
* malformed reply) carries `requestDispatched: true`. The MCP server turns
* that into `outcome_unknown` instead of a plain cancel that invites a blind
* retry of an input action that may have landed.
*/
function requestConnection(request, { timeoutMs = 30_000, signal = currentSignal(), keepOpen = false } = {}) {
throwIfAborted(signal);
return new Promise((resolve, reject) => {
const sock = net.connect(socketPath());
let buf = "";
let settled = false;
let written = false;
const failure = (message, code) => Object.assign(new ExecError(message), { code }, written ? { requestDispatched: true } : {});
const done = (fn, v) => { if (settled) return; settled = true; clearTimeout(timer); signal?.removeEventListener("abort", abort); if (!keepOpen || fn === reject) sock.destroy(); fn(v); };
const abort = () => done(reject, Object.assign(new ExecError("computer request cancelled"), { code: "cancelled" }));
const timer = setTimeout(() => done(reject, Object.assign(new ExecError(`${APP_NAME}: request timed out after ${timeoutMs}ms`), { code: "app_timeout" })), timeoutMs);
const abort = () => done(reject, failure("computer request cancelled", "cancelled"));
const timer = setTimeout(() => done(reject, failure(`${APP_NAME}: request timed out after ${timeoutMs}ms`, "app_timeout")), timeoutMs);
signal?.addEventListener("abort", abort, { once: true });
sock.on("error", (err) => done(reject, Object.assign(new ExecError(`${APP_NAME} is not reachable at ${socketPath()}: ${err.code ?? err.message}`), { code: "app_unavailable" })));
sock.on("connect", () => sock.write(JSON.stringify(request) + "\n"));
sock.on("error", (err) => done(reject, failure(`${APP_NAME} is not reachable at ${socketPath()}: ${err.code ?? err.message}`, "app_unavailable")));
sock.on("connect", () => { written = true; sock.write(JSON.stringify(request) + "\n"); });
sock.on("data", (d) => {
buf += d.toString("utf8");
const nl = buf.indexOf("\n");
if (nl === -1) return;
try { const reply = JSON.parse(buf.slice(0, nl)); done(resolve, keepOpen ? { reply, socket: sock } : reply); }
catch { done(reject, Object.assign(new ExecError(`${APP_NAME}: malformed reply`), { code: "app_bad_reply" })); }
catch { done(reject, failure(`${APP_NAME}: malformed reply`, "app_bad_reply")); }
});
sock.on("close", () => done(reject, Object.assign(new ExecError(`${APP_NAME}: connection closed before a reply`), { code: "app_unavailable" })));
sock.on("close", () => done(reject, failure(`${APP_NAME}: connection closed before a reply`, "app_unavailable")));
});
}
@@ -128,6 +138,11 @@ export function openAppSession(sessionId) {
// Library clients need not keep Node alive solely for an idle lease.
socket.unref();
return lease;
}, (error) => {
// Opening a lease sends no input, so its failure is never
// outcome-unknown for the action that was waiting on it.
if (error && typeof error === "object") delete error.requestDispatched;
throw error;
});
// A refused or unreachable open is retried on the next request, not cached.
pending.catch(() => { if (sessionLeases.get(sessionId) === pending) sessionLeases.delete(sessionId); });
@@ -856,13 +856,23 @@ static NSRunningApplication *resolve(NSDictionary *ref) {
} else return nil;
}
NSString *bundle=ref[@"bundle_id"], *name=ref[@"name"];
// A name or bundle shared by several running processes (two instances of
// one browser, a helper and its app) must not resolve to whichever happens
// to be listed first: that sends observation and input to the wrong window.
// Refuse like kill_app does; the observed pid disambiguates.
NSMutableArray<NSRunningApplication *> *hits=[NSMutableArray array];
for(NSRunningApplication *a in NSWorkspace.sharedWorkspace.runningApplications) {
if(ref[@"pid"] && a.processIdentifier!=[ref[@"pid"] intValue]) continue;
if(bundle && !matchesName(a.bundleIdentifier,bundle)) continue;
if(name && !matchesName(a.localizedName,name)) continue;
return a;
[hits addObject:a];
}
return nil;
if(hits.count>1) {
NSMutableArray *desc=[NSMutableArray array];
for(NSRunningApplication *a in hits) [desc addObject:[NSString stringWithFormat:@"%@ (pid %d)",a.localizedName?:@"?",a.processIdentifier]];
@throw [NSException exceptionWithName:@"app" reason:[NSString stringWithFormat:@"several running applications match (%@); pass pid to choose one",[desc componentsJoinedByString:@", "]] userInfo:nil];
}
return hits.firstObject;
}
static CGEventRef textEvent(NSString *text, BOOL down) {
UniChar *chars=calloc(text.length,sizeof(UniChar)); [text getCharacters:chars range:NSMakeRange(0,text.length)];
@@ -105,12 +105,22 @@ export function create({ exec }) {
return { action_sent: true, strategy: "event", backend: "uitest" };
}
/**
* Re-find an observed element in a fresh dump. uitest indexes are
* positional and shift whenever a sibling appears, so the index alone can
* name a different control; the element is addressed by its observed tree
* path and must still carry the observed role and label, or nothing is sent.
*/
async function centerOf(target) {
rejectAppSelectors(target);
const stale = (why) => Object.assign(new ExecError(`element_stale — ${why}; re-run get_app_state (uitest indexes change with the UI)`), { code: "element_stale" });
if (!Array.isArray(target?.path)) throw stale("the element target carries no observed tree path");
const tree = await dumpLayout();
const els = flatten(tree);
const el = els[target.index];
if (!el || !el.bounds) throw new ExecError("element_stale — re-run get_app_state; uitest indexes change with the UI");
const key = JSON.stringify(target.path);
const el = flatten(tree).find((candidate) => JSON.stringify(candidate.path) === key);
if (!el || !el.bounds) throw stale("the observed element is gone");
if (target.role !== undefined && el.role !== target.role) throw stale(`role changed (${target.role} → ${el.role})`);
if (target.label !== undefined && el.label !== target.label) throw stale(`label changed (${target.label} → ${el.label})`);
return el.bounds;
}
@@ -191,6 +191,12 @@ function defaultConnect(url, { timeoutMs = 8_000 } = {}) {
});
}
/** Per-command CDP reply deadline; teardown uses its own shorter bound. */
const commandTimeoutMs = () => Number(process.env.CODEWHALE_CU_BROWSER_COMMAND_TIMEOUT_MS) || 30_000;
// Teardown ignores the (possibly already cancelled) request signal and gives
// an unanswered peer at most three seconds per command.
const teardown = () => ({ timeoutMs: Math.min(3_000, commandTimeoutMs()), signal: null });
/** id-matched JSON-RPC over the WebSocket, plus CDP event fan-out. */
function makeChannel(ws) {
let nextId = 0;
@@ -199,7 +205,7 @@ function makeChannel(ws) {
let closed = false;
const failAll = (reason) => {
closed = true;
for (const [, entry] of pending) entry.reject(Object.assign(new ExecError(reason), { code: "browser_not_running" }));
for (const entry of [...pending.values()]) entry.reject(Object.assign(new ExecError(reason), { code: "browser_not_running" }));
pending.clear();
};
ws.addEventListener("message", (event) => {
@@ -218,11 +224,29 @@ function makeChannel(ws) {
ws.addEventListener("error", () => {});
return {
get alive() { return !closed; },
send(method, params = {}, sessionId) {
/**
* One CDP command. Bounded: a peer that keeps the socket open but never
* answers must not hang the request (or a stop/teardown awaiting it)
* forever. After the command is written its effect is unknown, so a
* timeout or cancel carries requestDispatched; a late reply is dropped.
*/
send(method, params = {}, sessionId, { timeoutMs = commandTimeoutMs(), signal = currentSignal() } = {}) {
if (closed) return Promise.reject(Object.assign(new ExecError("the browser is no longer reachable (CDP connection closed)"), { code: "browser_not_running" }));
if (signal?.aborted) return Promise.reject(Object.assign(new ExecError("computer request cancelled"), { code: "cancelled" }));
return new Promise((resolve, reject) => {
const id = ++nextId;
pending.set(id, { resolve, reject });
let timer = null;
const finish = () => { clearTimeout(timer); signal?.removeEventListener("abort", onAbort); pending.delete(id); };
const onAbort = () => { finish(); reject(Object.assign(new ExecError("computer request cancelled"), { code: "cancelled", requestDispatched: true })); };
pending.set(id, {
resolve: (value) => { finish(); resolve(value); },
reject: (error) => { finish(); reject(error); },
});
timer = setTimeout(() => {
finish();
reject(Object.assign(new ExecError(`CDP ${method} got no reply within ${timeoutMs}ms`), { code: "timeout", requestDispatched: true }));
}, timeoutMs);
signal?.addEventListener("abort", onAbort, { once: true });
ws.send(JSON.stringify({ id, method, params, ...(sessionId ? { sessionId } : {}) }));
});
},
@@ -280,8 +304,8 @@ export function createBrowser({
const { targetInfo: info } = await state.channel.send("Target.getTargetInfo", { targetId });
return { url: info?.url ?? "", title: info?.title ?? "" };
}
async function listTabs() {
const { targetInfos } = await state.channel.send("Target.getTargets", {});
async function listTabs(options) {
const { targetInfos } = await state.channel.send("Target.getTargets", {}, undefined, options);
return (targetInfos ?? []).filter((t) => t.type === "page");
}
async function ensureDom() {
@@ -570,17 +594,17 @@ export function createBrowser({
if (!state.channel) return { running: false, note: "no browser session for this computer session" };
if (state.attached) {
// The person's browser: never close a tab or the browser, only detach.
if (state.sessionId) await state.channel.send("Target.detachFromTarget", { sessionId: state.sessionId }).catch(() => {});
if (state.sessionId) await state.channel.send("Target.detachFromTarget", { sessionId: state.sessionId }, undefined, teardown()).catch(() => {});
state.channel.close();
state.channel = null; state.targetId = null; state.sessionId = null; state.pageEnabled = false; state.domEnabled = false; state.attached = false;
return { running: false, detached: true, browser_closed: false, note: "detached from the shared browser; its window and tabs stay as they are" };
}
try { await state.channel.send("Target.closeTarget", { targetId: state.targetId }); } catch { /* the tab may already be gone */ }
try { await state.channel.send("Target.closeTarget", { targetId: state.targetId }, undefined, teardown()); } catch { /* the tab may already be gone */ }
let remaining = null;
try { remaining = (await listTabs()).length; } catch { remaining = null; }
try { remaining = (await listTabs(teardown())).length; } catch { remaining = null; }
let browserClosed = false;
if (remaining === 0) {
try { await state.channel.send("Browser.close"); browserClosed = true; } catch {}
try { await state.channel.send("Browser.close", {}, undefined, teardown()); browserClosed = true; } catch {}
await sleep(200);
}
state.channel.close();
+11 -7
View File
@@ -116,19 +116,23 @@ export async function spawnDockerComputer({ id, image = DEFAULT_IMAGE } = {}) {
}
/**
* Destroy a spawned container — but only one this plugin created. A docker
* computer whose container lacks our spawn label is left running and reported
* not_spawned; removing its registry entry is still the caller's choice.
* Destroy a spawned container — but only one this MCP process created. A
* docker computer whose container lacks our spawn label is left running and
* reported not_spawned; one spawned by another session (the registry is
* shared between MCP processes) is left running and reported other_session,
* so one task cannot tear down a desktop another task is driving.
*/
export async function destroyDockerComputer(computer) {
export async function destroyDockerComputer(computer, command = docker) {
const container = computer?.container;
if (!container || !CONTAINER_RE.test(container)) {
throw new SpawnError("invalid_container", "docker computer has no valid container name");
}
const insp = await docker(["container", "inspect", "--format", `{{index .Config.Labels "${SPAWN_LABEL}"}}`, container], { timeoutMs: 10_000, signal: null });
const insp = await command(["container", "inspect", "--format", `{{index .Config.Labels "${SPAWN_LABEL}"}}|{{index .Config.Labels "${SESSION_LABEL}"}}`, container], { timeoutMs: 10_000, signal: null });
if (insp.code !== 0) return { destroyed: false, reason: "container_gone" };
if (insp.stdout.trim() !== "1") return { destroyed: false, reason: "not_spawned" };
const r = await docker(["rm", "-f", container], { timeoutMs: 20_000, signal: null });
const [spawned, session] = insp.stdout.trim().split("|");
if (spawned !== "1") return { destroyed: false, reason: "not_spawned" };
if (session !== SESSION_ID) return { destroyed: false, reason: "other_session" };
const r = await command(["rm", "-f", container], { timeoutMs: 20_000, signal: null });
if (r.code !== 0) throw new SpawnError("cleanup_failed", `docker rm -f ${container} failed: ${trim(r.stderr)}`, r);
return { destroyed: true };
}
@@ -218,8 +218,10 @@ export function sshExec(computer, binding) {
const r = await run("ssh", [...base, "node", remoteAgent, b64({ args: request.args ?? {}, tool: request.tool, nonce: crypto.randomBytes(6).toString("hex") })], {
timeoutMs: opts.timeoutMs ?? 25_000,
});
if (r.aborted) throw Object.assign(new ExecError("computer request cancelled", r), { code: "cancelled" });
if (r.timedOut) throw new ExecError(`ssh ${userHost}: timed out`, r);
// A spawned one-shot agent may have acted before it was cancelled or
// timed out; say so rather than inviting a blind retry.
if (r.aborted) throw Object.assign(new ExecError("computer request cancelled", r), { code: "cancelled" }, r.spawned ? { requestDispatched: true } : {});
if (r.timedOut) throw Object.assign(new ExecError(`ssh ${userHost}: timed out`, r), { code: "remote_timeout", requestDispatched: true });
if (r.code !== 0) throw new ExecError(`ssh ${userHost} exited ${r.code}: ${r.stderr.trim().slice(0, 400)}`, r);
// The agent prints exactly one JSON line; anything before it is MOTD noise.
const line = r.stdout.trim().split("\n").filter((l) => l.startsWith("{")).pop();
@@ -254,8 +256,8 @@ export function dockerExec(computer, binding) {
const r = await run("docker", ["exec", container, "/bin/sh", remoteAgent, b64({ args: request.args ?? {}, tool: request.tool, nonce: crypto.randomBytes(6).toString("hex") })], {
timeoutMs: opts.timeoutMs ?? 25_000,
});
if (r.aborted) throw Object.assign(new ExecError("computer request cancelled", r), { code: "cancelled" });
if (r.timedOut) throw new ExecError(`docker exec ${container}: timed out`, r);
if (r.aborted) throw Object.assign(new ExecError("computer request cancelled", r), { code: "cancelled" }, r.spawned ? { requestDispatched: true } : {});
if (r.timedOut) throw Object.assign(new ExecError(`docker exec ${container}: timed out`, r), { code: "remote_timeout", requestDispatched: true });
if (r.code !== 0) throw new ExecError(`docker exec ${container} exited ${r.code}: ${r.stderr.trim().slice(0, 400)}`, r);
const line = r.stdout.trim().split("\n").filter((l) => l.startsWith("{")).pop();
const reply = line ? JSON.parse(line) : null;
@@ -71,6 +71,17 @@ test('real handler/backend/native resolver never redirects an explicit app refer
assert.equal(reply.ok,false,`${tool} must reject ${JSON.stringify(app_ref)}`);
assert.match(reply.error.message,/application not found/);
}
// A shared name or bundle must never pick whichever process is listed first.
for(const app_ref of [{name:'Twin'},{bundle_id:'test.twin'},{name:'twin',bundle_id:'test.twin'}]) {
const reply=await call(tool,{app_ref});
assert.equal(reply.ok,false,`${tool} must refuse ambiguous ${JSON.stringify(app_ref)}`);
assert.equal(reply.error.code,'ambiguous_application',JSON.stringify(reply));
assert.match(reply.error.message,/Twin \(pid 200\), Twin \(pid 201\)/);
}
}
for(const pid of [200,201]) {
const reply=await call('list_windows',{app_ref:{pid,name:'Twin'}});
assert.equal(reply.ok,true,JSON.stringify(reply));assert.equal(reply.data.pid,pid);
}
} finally { delete process.env.CU_TARGETING_NATIVE; }
});
@@ -47,7 +47,7 @@ function harmonyFixtureExec(t) {
return fakeJpeg(168, 120);
},
};
return { exec, calls };
return { exec, calls, layout };
}
test("harmony: parseBounds handles uitest bounds strings", () => {
@@ -102,6 +102,24 @@ test("harmony: click routes through uitest uiInput with validated args", async (
assert.deepEqual(ui.args, ["uitest", "uiInput", "click", "124", "45"]);
});
test("harmony: element actions re-find the observed element and refuse a reordered tree", async () => {
const { exec, calls, layout } = harmonyFixtureExec();
const mod = await import("../src/backends/harmonyos.mjs");
const b = mod.create({ exec });
const ok = (await b.get_app_state({})).elements.find((e) => e.label === "OK");
const target = { index: ok.index, path: ok.path, role: ok.role, label: ok.label };
await b.perform_action({ target, action: "click" });
assert.deepEqual(calls.filter((c) => c.args?.[1] === "uiInput").at(-1).args, ["uitest", "uiInput", "click", "200", "230"]);
// A new sibling ahead of OK shifts every uitest index: the stored index now
// names "Delete". Nothing may be clicked.
layout.children.unshift({ attributes: { type: "Button", text: "Delete", bounds: "[0,300][100,360]" }, children: [] });
const before = calls.filter((c) => c.args?.[1] === "uiInput").length;
await assert.rejects(b.perform_action({ target, action: "click" }), (error) => error.code === "element_stale" && /label changed/.test(error.message));
await assert.rejects(b.set_value({ target, value: "x" }), (error) => error.code === "element_stale");
await assert.rejects(b.perform_action({ target: { index: ok.index }, action: "click" }), (error) => error.code === "element_stale", "a bare index has no identity to verify");
assert.equal(calls.filter((c) => c.args?.[1] === "uiInput").length, before, "no input after the tree changed");
});
test("harmony: clipboard and select_text fail closed with named reasons", async () => {
const { exec } = harmonyFixtureExec();
const mod = await import("../src/backends/harmonyos.mjs");
@@ -271,3 +271,19 @@ test('findBrowserApp uses actual Windows vendor folders and executable names', (
assert.equal(findBrowserApp('win32', { [root]: 'C:\\fixture' }, candidate => candidate === expected), expected);
}
});
test("an unanswered CDP command times out as outcome-unknown and stop still finishes", async (t) => {
const prior = process.env.CODEWHALE_CU_BROWSER_COMMAND_TIMEOUT_MS;
process.env.CODEWHALE_CU_BROWSER_COMMAND_TIMEOUT_MS = "60";
t.after(() => { if (prior === undefined) delete process.env.CODEWHALE_CU_BROWSER_COMMAND_TIMEOUT_MS; else process.env.CODEWHALE_CU_BROWSER_COMMAND_TIMEOUT_MS = prior; });
// An open peer that never answers these commands (a wedged renderer).
const script = { ...defaultScript(pageState()), "Input.insertText": undefined, "Target.closeTarget": undefined, "Browser.close": undefined };
const { browser, sockets } = harness(t, { script });
await browser.start({});
await assert.rejects(browser.type({ text: "hello" }), (error) => error.code === "timeout" && error.requestDispatched === true && /Input\.insertText/.test(error.message));
const stopped = await Promise.race([browser.stop(), new Promise((resolve) => setTimeout(() => resolve("hung"), 2_000))]);
assert.notEqual(stopped, "hung", "stop must not wait forever on an unanswered peer");
assert.equal(stopped.closed, true);
assert.equal(sockets[0].ws.sentOf("Target.closeTarget").length, 1);
assert.equal((await browser.status()).running, false);
});
@@ -7,7 +7,7 @@ import path from "node:path";
import net from "node:net";
import { run, runOk, runInputLease, ExecError, have, trim, withSignal } from "../src/exec.mjs";
import { safeRemotePath, b64, localExec, hdcExec, executorFor } from "../src/transport.mjs";
import { ensureApp, writeRegistration } from "../src/app-socket.mjs";
import { appRequest, appSessionRequest, ensureApp, writeRegistration } from "../src/app-socket.mjs";
test("a missing registered bundle gives a repair path without falling back to host input",async t=>{
const directory=fs.mkdtempSync(path.join(os.tmpdir(),"cu-missing-app-"));
@@ -208,3 +208,30 @@ test("hdc pullFile rejects traversal and shell metacharacters before execution",
await assert.rejects(ex.pullFile(remote, "/unused-fixture-output"), /refusing unsafe remote path/);
}
});
test("a helper request that was written reports an unknown outcome when it times out or is cancelled", {skip:process.platform==="win32"}, async t => {
const dir=fs.mkdtempSync(path.join(os.tmpdir(),"cu-silent-helper-"));
const saved=process.env.CODEWHALE_CU_APP_SOCKET;
process.env.CODEWHALE_CU_APP_SOCKET=path.join(dir,"app.sock");
const received=[];
// Accepts and reads the request, then never answers (a wedged helper).
const server=net.createServer(socket=>socket.on("data",data=>received.push(String(data))));
await new Promise(resolve=>server.listen(process.env.CODEWHALE_CU_APP_SOCKET,resolve));
t.after(async()=>{
server.close();
if(saved===undefined) delete process.env.CODEWHALE_CU_APP_SOCKET; else process.env.CODEWHALE_CU_APP_SOCKET=saved;
fs.rmSync(dir,{recursive:true,force:true});
});
await assert.rejects(appRequest({tool:"left_click",args:{}},{timeoutMs:80,signal:null}),error=>error.code==="app_timeout"&&error.requestDispatched===true);
const controller=new AbortController();
const pending=appRequest({tool:"left_click",args:{}},{timeoutMs:5_000,signal:controller.signal});
while(received.length<2) await new Promise(resolve=>setTimeout(resolve,5));
controller.abort();
await assert.rejects(pending,error=>error.code==="cancelled"&&error.requestDispatched===true);
assert.equal(received.length,2,"each request reached the helper before its failure");
// Opening the session lease sends no input: its timeout is not outcome-unknown.
await assert.rejects(appSessionRequest({tool:"left_click",args:{},sessionId:"silent-lease"},{timeoutMs:80,signal:null}),error=>error.code==="app_timeout"&&error.requestDispatched===undefined);
// Nothing written: an unreachable helper is a plain failure, safe to retry.
process.env.CODEWHALE_CU_APP_SOCKET=path.join(dir,"absent.sock");
await assert.rejects(appRequest({tool:"left_click",args:{}},{timeoutMs:1_000,signal:null}),error=>error.code==="app_unavailable"&&error.requestDispatched===undefined);
});
@@ -20,7 +20,12 @@
CUFixtureApplication *target=[CUFixtureApplication new], *front=[CUFixtureApplication new];
target.processIdentifier=123; target.localizedName=@"Fixture"; target.bundleIdentifier=@"test.fixture";
front.processIdentifier=999; front.localizedName=@"Other"; front.bundleIdentifier=@"test.other";
return @[front,target];
// Two running instances sharing one name and bundle (a second browser
// profile, an `open -n` copy): only a pid may choose between them.
CUFixtureApplication *twinA=[CUFixtureApplication new], *twinB=[CUFixtureApplication new];
twinA.processIdentifier=200; twinA.localizedName=@"Twin"; twinA.bundleIdentifier=@"test.twin";
twinB.processIdentifier=201; twinB.localizedName=@"Twin"; twinB.bundleIdentifier=@"test.twin";
return @[front,target,twinA,twinB];
}
- (NSRunningApplication *)frontmostApplication { return (NSRunningApplication *)self.runningApplications[0]; }
@end
@@ -50,7 +55,9 @@ int main(int argc, const char *argv[]) { @autoreleasepool {
return 0;
}
if(![@[@"list_windows",@"get_app_state",@"resolve_element",@"window_info"] containsObject:request[@"tool"]]) return 2;
NSRunningApplication *app=resolve(request[@"args"][@"app_ref"]);
NSRunningApplication *app=nil;
@try { app=resolve(request[@"args"][@"app_ref"]); }
@catch(NSException *error) { fprintf(stderr,"%s\n",error.reason.UTF8String); return 1; }
if(!app) { fputs("application not found\n",stderr); return 1; }
NSDictionary *receipt=@{@"found":@YES,@"pid":@(app.processIdentifier),@"name":app.localizedName,@"bundle_id":app.bundleIdentifier,@"windows":@[],@"elements":@[]};
NSData *json=[NSJSONSerialization dataWithJSONObject:receipt options:0 error:nil];
@@ -231,3 +231,17 @@ test("Docker desktop entrypoint survives repeated orderly restarts", { ...NEED_D
assert.equal(observed, true, `desktop unavailable after restart ${cycle}: ${last}`);
}
});
test("destroyDockerComputer never removes a desktop another session spawned", async () => {
const issued = [];
const labels = (value) => async (args) => {
issued.push(args);
return args[0] === "container" ? { code: 0, stdout: `${value}\n`, stderr: "" } : { code: 0, stdout: "", stderr: "" };
};
const computer = { container: "cu-spawn-other-ab12cd" };
assert.deepEqual(await spawnMod.destroyDockerComputer(computer, labels("1|another-session")), { destroyed: false, reason: "other_session" });
assert.deepEqual(await spawnMod.destroyDockerComputer(computer, labels("|")), { destroyed: false, reason: "not_spawned" });
assert.equal(issued.filter((args) => args[0] === "rm").length, 0, "no docker rm for a container this session does not own");
assert.deepEqual(await spawnMod.destroyDockerComputer(computer, labels(`1|${SESSION_ID}`)), { destroyed: true });
assert.deepEqual(issued.at(-1), ["rm", "-f", "cu-spawn-other-ab12cd"]);
});
+20 -4
View File
@@ -77,6 +77,11 @@ const MAX_ACP_SESSIONS: usize = 64;
/// broken client cannot strand the stdio server forever after cancellation.
const ACP_PERMISSION_CANCEL_GRACE: Duration = Duration::from_secs(2);
/// How long a cancelled tool gets to observe its token and wind down (kill a
/// child, flush a write) before the turn stops waiting and drops it. A tool
/// that ignores its token must not hold the stdio server forever.
const ACP_TOOL_CANCEL_GRACE: Duration = Duration::from_secs(5);
/// Agent-originated JSON-RPC request ids have their own namespace. Strings
/// avoid the client-specific numeric response-id compatibility shim used for
/// replies to client-originated requests.
@@ -1226,11 +1231,22 @@ where
write_jsonrpc_result(writer, msg_id, json!(null)).await?;
}
cancel_token.cancel();
// Give the tool a chance to observe the token and
// wind down (e.g. kill a running child process)
// before we drop it.
// Give the tool a bounded chance to observe the
// token and wind down (e.g. kill a running child
// process) before we drop it.
cancelled = true;
break (&mut exec_fut).await;
break match tokio::time::timeout(
ACP_TOOL_CANCEL_GRACE,
&mut exec_fut,
)
.await
{
Ok(result) => result,
Err(_) => Err(ToolError::cancelled(format!(
"the tool did not stop within {}s of cancellation and was abandoned",
ACP_TOOL_CANCEL_GRACE.as_secs()
))),
};
}
if let Some(msg_id) = msg_id {
let msg_id = response_id_policy.response_id(msg_id);
+32 -12
View File
@@ -3765,17 +3765,21 @@ impl CodewhaleClient {
return self.send_with_isolated_retry(build, disclosure).await;
}
let retry_cfg: LlmRetryConfig = self.retry.clone().into();
let pause_scope = self.rate_limit_scope();
let callback_scope = pause_scope.clone();
let request_result = with_retry(
&retry_cfg,
|| {
let request = build();
let pause_scope = pause_scope.as_str();
async move {
// Sleep in bounded slices rather than the full remaining
// window: the pause is process-global, so a concurrent
// `clear_rate_limit()` (or a shortened deadline) must
// release requests that are already waiting instead of
// stranding them for the whole original window.
while let Some(delay) = crate::retry_status::rate_limit_remaining() {
// window: the pause is shared by every request to this
// route, so a concurrent `clear_rate_limit()` (or a
// shortened deadline) must release requests that are
// already waiting instead of stranding them for the whole
// original window.
while let Some(delay) = crate::retry_status::rate_limit_remaining(pause_scope) {
tokio::time::sleep(delay.min(RATE_LIMIT_PAUSE_RECHECK_INTERVAL)).await;
}
self.wait_for_rate_limit().await;
@@ -3793,7 +3797,7 @@ impl CodewhaleClient {
Err(self.http_error_with_route_context(status.as_u16(), &body, retry_after))
}
},
Some(Box::new(|err, attempt, delay| {
Some(Box::new(move |err, attempt, delay| {
let (reason_label, human_reason) = retry_reason_label_and_human(err);
logging::warn(format!(
"HTTP retry reason={} attempt={} delay={:.2}s",
@@ -3802,7 +3806,7 @@ impl CodewhaleClient {
delay.as_secs_f64(),
));
if matches!(err, LlmError::RateLimited { .. }) {
crate::retry_status::note_rate_limit(delay);
crate::retry_status::note_rate_limit(&callback_scope, delay);
}
crate::retry_status::start(attempt + 1, delay, human_reason);
})),
@@ -3818,6 +3822,7 @@ impl CodewhaleClient {
Err(err) => {
if let LlmError::RateLimited { retry_after, .. } = &err.last_error {
crate::retry_status::note_rate_limit(
&pause_scope,
retry_after
.unwrap_or_else(|| retry_cfg.delay_for_attempt(retry_cfg.max_retries)),
);
@@ -3838,6 +3843,21 @@ impl CodewhaleClient {
}
}
/// Key for this route's shared `Retry-After` pause: the configured route
/// identity plus the host it reaches. A 429 from one provider pauses only
/// requests that would hit the same limit, never another provider, a
/// local runtime, or a sub-agent on a different route.
pub(crate) fn rate_limit_scope(&self) -> String {
let route = if self.provider_identity.is_empty() {
self.api_provider.as_str()
} else {
self.provider_identity.as_str()
};
let host = crate::llm_client::base_url_authority(&self.base_url)
.unwrap_or_else(|| redact_url_for_display(&self.base_url));
format!("{route}@{host}")
}
/// The same bounded transport retry policy without process-global retry
/// banners, provider-wide pause cells, or shared connection-health writes.
/// Used only by the Auto classifier during read-only request inspection.
@@ -6596,7 +6616,6 @@ mod tests {
crate::retry_status::clear();
crate::retry_status::clear_rate_limit();
crate::retry_status::start(7, Duration::from_secs(60), "foreground sentinel");
crate::retry_status::note_rate_limit(Duration::from_secs(60));
let server = MockServer::start().await;
Mock::given(method("POST"))
.respond_with(ResponseTemplate::new(200).set_body_json(json!({
@@ -6615,6 +6634,7 @@ mod tests {
.await;
let client = deepseek_request_boundary_client("https://api.deepseek.com/v1", server.uri());
crate::retry_status::note_rate_limit(&client.rate_limit_scope(), Duration::from_secs(60));
let request = MessageRequest {
model: "deepseek-v4-pro".to_string(),
messages: vec![Message {
@@ -6681,7 +6701,7 @@ mod tests {
state => panic!("isolated success mutated retry state: {state:?}"),
}
assert!(
crate::retry_status::rate_limit_remaining().is_some(),
crate::retry_status::rate_limit_remaining(&client.rate_limit_scope()).is_some(),
"isolated success must not clear the foreground provider pause"
);
crate::retry_status::clear();
@@ -6697,7 +6717,6 @@ mod tests {
crate::retry_status::clear();
crate::retry_status::clear_rate_limit();
crate::retry_status::start(9, Duration::from_secs(60), "foreground sentinel 429");
crate::retry_status::note_rate_limit(Duration::from_secs(60));
let server = MockServer::start().await;
Mock::given(method("POST"))
@@ -6713,6 +6732,7 @@ mod tests {
deepseek_request_boundary_client("https://api.deepseek.com/v1", server.uri());
client.retry.enabled = false;
client.retry.max_retries = 0;
crate::retry_status::note_rate_limit(&client.rate_limit_scope(), Duration::from_secs(60));
let request = MessageRequest {
model: "deepseek-v4-pro".to_string(),
messages: vec![Message {
@@ -6751,8 +6771,8 @@ mod tests {
}
state => panic!("isolated 429 mutated retry state: {state:?}"),
}
let remaining =
crate::retry_status::rate_limit_remaining().expect("foreground provider pause remains");
let remaining = crate::retry_status::rate_limit_remaining(&client.rate_limit_scope())
.expect("foreground provider pause remains");
assert!(
remaining < Duration::from_secs(70),
"classifier Retry-After must not extend the global pause: {remaining:?}"
+191 -83
View File
@@ -34,10 +34,10 @@ use crate::logging;
use crate::tools::schema_sanitize;
use codewhale_models::{ContentBlock, MessageRequest, MessageResponse, StreamEvent, Usage};
use super::CodewhaleClient;
use super::prepared::WireDialect;
use super::role_placement::{RolePlacement, role_placement};
use super::wire::{extract_sse_data_value, next_sse_line};
use super::{CodewhaleClient, ERROR_BODY_MAX_BYTES, bounded_error_text};
use super::wire::{extract_sse_data_value, next_sse_line, push_sse_event_data};
/// Maximum `cache_control` breakpoints Anthropic accepts per request.
const MAX_CACHE_BREAKPOINTS: usize = 4;
@@ -213,78 +213,64 @@ impl CodewhaleClient {
body
}
/// Non-streaming send through the shared typed retry path
/// (`send_with_retry`): 429/5xx/transport failures retry with backoff and
/// honor `Retry-After`, and a final failure stays a downcastable
/// `LlmError` so the engine can classify auth, rate-limit, context and
/// invalid-request failures like every other wire.
async fn send_anthropic_request(&self, url: &str, body: &Value) -> Result<reqwest::Response> {
let url = self.messages_transport_url(url);
self.wait_for_rate_limit().await;
let response = self
.http_client
.post(&url)
.header("Accept", "text/event-stream")
.json(body)
.send()
.await
.context("Anthropic Messages API request failed")?;
self.check_anthropic_response(response).await
}
/// Shared status/error-envelope handling for streaming and
/// non-streaming Messages responses.
async fn check_anthropic_response(
&self,
response: reqwest::Response,
) -> Result<reqwest::Response> {
let status = response.status();
if !status.is_success() {
let raw = bounded_error_text(response, ERROR_BODY_MAX_BYTES).await;
let (error_type, message) = parse_anthropic_error_envelope(&raw);
self.mark_request_failure(&format!("anthropic status={status}"))
.await;
anyhow::bail!("Anthropic API error (HTTP {status} {error_type}): {message}");
}
self.mark_request_success().await;
Ok(response)
let request_body =
serde_json::to_vec(body).context("Failed to serialize Anthropic Messages request")?;
self.send_with_retry(|| {
self.http_client
.post(&url)
.header(reqwest::header::CONTENT_TYPE, "application/json")
.header("Accept", "text/event-stream")
.body(request_body.clone())
})
.await
.context("Anthropic Messages API request failed")
}
/// Open the streaming Messages request through the shared stream-entry
/// transport policy: bounded header wait, dual-client selection, and at
/// most one HTTP/1.1 fallback retry on a classified H2 header stall.
/// Wire-specific request construction (headers, endpoint, body) stays
/// here at the adapter edge.
/// Inside each open attempt the provider retry loop (`send_with_retry`)
/// handles rate limits and transient upstream failures before any stream
/// body exists, as the Chat and Responses adapters do. Wire-specific
/// request construction (headers, endpoint, body) stays here at the
/// adapter edge.
async fn open_anthropic_stream_response(
&self,
url: &str,
body: &Value,
) -> Result<reqwest::Response> {
let url = self.messages_transport_url(url);
let request_body =
serde_json::to_vec(body).context("Failed to serialize Anthropic Messages request")?;
let open_req = self.stream_open_request();
let opened = super::stream_entry::open_sse_response(&open_req, |policy| {
super::stream_entry::open_sse_response(&open_req, |policy| {
let url = url.clone();
let request_body = request_body.clone();
async move {
self.wait_for_rate_limit().await;
let client = super::stream_entry::client_for_policy(
&self.http_client,
self.http1_fallback_client(),
policy,
);
client
.post(&url)
.header("Accept", "text/event-stream")
.json(body)
.send()
.await
.context("Anthropic Messages API request failed")
self.send_with_retry(|| {
client
.post(&url)
.header(reqwest::header::CONTENT_TYPE, "application/json")
.header("Accept", "text/event-stream")
.body(request_body.clone())
})
.await
.context("Anthropic Messages API request failed")
}
})
.await;
let response = match opened {
Ok(response) => response,
Err(err) => {
self.mark_request_failure(&format!("anthropic stream open: {err}"))
.await;
return Err(err);
}
};
self.check_anthropic_response(response).await
.await
}
/// Handle a streaming Messages API request.
@@ -312,6 +298,9 @@ impl CodewhaleClient {
// multi-byte UTF-8 char (CJK/emoji) split across HTTP/2 DATA is
// never corrupted to U+FFFD. Genuine invalid bytes fail closed.
let mut buffer: Vec<u8> = Vec::new();
// `data:` fields of the event being assembled, dispatched at the
// blank line that ends it (or at stream end).
let mut event_data = String::new();
let stream_start = std::time::Instant::now();
let mut last_chunk_at = std::time::Instant::now();
let mut bytes_received: usize = 0;
@@ -350,22 +339,36 @@ impl CodewhaleClient {
}
loop {
let line = match next_sse_line(&mut buffer, ended) {
Ok(Some(line)) => line,
let data = match next_sse_line(&mut buffer, ended) {
// A blank line ends the event.
Ok(Some(line)) if line.is_empty() => std::mem::take(&mut event_data),
Ok(Some(line)) => {
// `event:` lines are redundant (the data payload
// carries `type`) and comment/heartbeat lines are
// ignorable.
if let Some(value) = extract_sse_data_value(&line)
&& let Err(err) = push_sse_event_data(&mut event_data, value)
{
yield Err(anyhow::anyhow!("{err}"));
return;
}
continue;
}
// The final event may arrive without its blank line.
Ok(None) if ended && !event_data.is_empty() => {
std::mem::take(&mut event_data)
}
Ok(None) => break,
Err(err) => {
yield Err(anyhow::anyhow!("{err}"));
return;
}
};
// `event:` lines are redundant (the data payload carries
// `type`) and comment/heartbeat lines are ignorable.
let Some(data) = extract_sse_data_value(&line) else {
if data.is_empty() {
continue;
};
}
match convert_anthropic_sse_data(data) {
match convert_anthropic_sse_data(&data) {
Some(Ok(StreamEvent::Error { error })) => {
let (error_type, message) = anthropic_error_fields(&error);
yield Err(anyhow::anyhow!(
@@ -391,6 +394,12 @@ impl CodewhaleClient {
break;
}
}
// Only `message_stop` (returned above) or a provider error proves
// the response is whole. A bare HTTP EOF is truncation, and ending
// the stream quietly would hand the turn a partial answer as done.
yield Err(anyhow::anyhow!(
"Anthropic Messages stream closed before message_stop"
));
};
Ok(Box::pin(stream))
@@ -994,14 +1003,6 @@ fn parse_anthropic_usage(usage: &Value) -> Usage {
/// Extract `error.type` / `error.message` from an Anthropic error envelope
/// (`{"type":"error","error":{"type":...,"message":...}}`), falling back to
/// the raw body so nothing is swallowed.
fn parse_anthropic_error_envelope(raw: &str) -> (String, String) {
let Ok(value) = serde_json::from_str::<Value>(raw) else {
return ("unknown".to_string(), raw.to_string());
};
let error = value.get("error").unwrap_or(&value);
anthropic_error_fields(error)
}
fn anthropic_error_fields(error: &Value) -> (String, String) {
let error_type = error
.get("type")
@@ -2027,19 +2028,6 @@ mod tests {
assert_eq!(usage.prompt_cache_write_tokens, Some(2045));
}
#[test]
fn error_envelope_parses_type_and_message() {
let (error_type, message) = parse_anthropic_error_envelope(
r#"{"type":"error","error":{"type":"rate_limit_error","message":"Too many requests"},"request_id":"req_1"}"#,
);
assert_eq!(error_type, "rate_limit_error");
assert_eq!(message, "Too many requests");
let (error_type, message) = parse_anthropic_error_envelope("upstream blew up");
assert_eq!(error_type, "unknown");
assert_eq!(message, "upstream blew up");
}
#[test]
fn data_url_image_becomes_a_base64_source_not_a_url_source() {
// Anthropic rejects a `data:` URL under `{"type":"url"}`. This is the
@@ -2248,6 +2236,73 @@ mod tests {
assert!(saw_stop, "message_stop should arrive through the seam");
}
async fn collect_anthropic_stream(body: &'static str) -> Vec<Result<StreamEvent>> {
use futures_util::StreamExt;
use wiremock::matchers::{method, path};
use wiremock::{Mock, MockServer, ResponseTemplate};
let server = MockServer::start().await;
Mock::given(method("POST"))
.and(path("/v1/messages"))
.respond_with(
ResponseTemplate::new(200)
.insert_header("Content-Type", "text/event-stream")
.set_body_string(body),
)
.mount(&server)
.await;
let client = deepseek_test_client(&server.uri());
let stream = client
.handle_anthropic_stream(
&client
.prepare_outbound_request(request_with("deepseek-v4", None, None, None), true)
.expect("anthropic request prepares"),
)
.await
.expect("stream opens");
tokio::time::timeout(std::time::Duration::from_secs(5), stream.collect())
.await
.expect("stream ends")
}
#[tokio::test]
async fn anthropic_stream_joins_multiline_data_fields_into_one_event() {
use codewhale_models::Delta;
// One JSON payload split across two `data:` fields of a single event.
let events = collect_anthropic_stream(concat!(
"event: content_block_delta\n",
"data: {\"type\":\"content_block_delta\",\"index\":0,\n",
"data: \"delta\":{\"type\":\"text_delta\",\"text\":\"joined\"}}\n\n",
"data: {\"type\":\"message_stop\"}\n\n",
))
.await;
assert!(
events.iter().any(|event| matches!(
event,
Ok(StreamEvent::ContentBlockDelta {
delta: Delta::TextDelta { text },
..
}) if text == "joined"
)),
"the split event was lost: {events:?}"
);
assert!(matches!(events.last(), Some(Ok(StreamEvent::MessageStop))));
}
#[tokio::test]
async fn anthropic_stream_eof_without_message_stop_is_an_error() {
let events = collect_anthropic_stream(
"data: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\"partial\"}}\n\n",
)
.await;
let last = events.last().expect("the delta and a terminal item");
assert!(
last.as_ref()
.is_err_and(|error| error.to_string().contains("closed before message_stop")),
"a truncated stream must not end quietly: {events:?}"
);
}
/// Fault injection (#6184): a provider that answers the headers and then
/// sends nothing fails the stream at the first-byte bound with a
/// distinct error and a `crashes/` stall record, instead of holding the
@@ -2344,10 +2399,63 @@ mod tests {
Ok(_) => panic!("auth errors must fail fast"),
Err(err) => err,
};
let text = err.to_string();
// Typed through the shared classifier, provider message kept.
assert!(
text.contains("HTTP 401") && text.contains("authentication_error"),
"error envelope should be preserved: {text}"
matches!(
err.chain()
.find_map(|cause| cause.downcast_ref::<crate::llm_client::LlmError>()),
Some(crate::llm_client::LlmError::AuthenticationError(_))
),
"a 401 must stay a typed authentication error: {err:#}"
);
assert!(format!("{err:#}").contains("bad key"), "{err:#}");
}
#[tokio::test]
async fn anthropic_rate_limit_is_retried_honoring_retry_after() {
use wiremock::matchers::{method, path};
use wiremock::{Mock, MockServer, ResponseTemplate};
let server = MockServer::start().await;
Mock::given(method("POST"))
.and(path("/v1/messages"))
.respond_with(
ResponseTemplate::new(429)
.insert_header("retry-after", "0")
.set_body_string(
"{\"type\":\"error\",\"error\":{\"type\":\"rate_limit_error\",\"message\":\"slow down\"}}",
),
)
.up_to_n_times(1)
.expect(1)
.mount(&server)
.await;
Mock::given(method("POST"))
.and(path("/v1/messages"))
.respond_with(
ResponseTemplate::new(200)
.insert_header("Content-Type", "text/event-stream")
.set_body_string("data: {\"type\":\"message_stop\"}\n\n"),
)
.expect(1)
.mount(&server)
.await;
let mut client = deepseek_test_client(&server.uri());
client.retry.enabled = true;
client.retry.max_retries = 2;
client.retry.initial_delay = 0.0;
client.retry.max_delay = 0.0;
let stream = client
.handle_anthropic_stream(
&client
.prepare_outbound_request(request_with("deepseek-v4", None, None, None), true)
.expect("anthropic request prepares"),
)
.await;
assert!(
stream.is_ok(),
"a 429 before the stream body must be retried, not surfaced"
);
}
+14 -15
View File
@@ -33,7 +33,7 @@ use codewhale_models::{
use super::prepared::WireDialect;
use super::role_placement::{RolePlacement, role_placement};
use super::wire::{extract_sse_data_value, flush_sse_line, take_sse_line};
use super::wire::{extract_sse_data_value, flush_sse_line, push_sse_event_data, take_sse_line};
use super::{
CodewhaleClient, ERROR_BODY_MAX_BYTES, SSE_BACKPRESSURE_HIGH_WATERMARK,
SSE_BACKPRESSURE_SLEEP_MS, SSE_MAX_LINES_PER_CHUNK, acquire_stream_buffer,
@@ -1603,15 +1603,13 @@ impl CodewhaleClient {
continue;
}
if let Some(data) = extract_sse_data_value(&line) {
// The SSE spec joins multiple `data:` fields within one
// event with '\n'; concatenating with no separator would
// yield `{…}{…}` and fail JSON parsing, silently dropping
// the frame.
if !line_buf.is_empty() {
line_buf.push('\n');
}
line_buf.push_str(data);
if let Some(data) = extract_sse_data_value(&line)
&& let Err(err) = push_sse_event_data(&mut line_buf, data)
{
decode_failed = true;
stream_failed = true;
yield Err(anyhow::anyhow!("{err}"));
break 'stream;
}
// Ignore other SSE fields (event:, id:, retry:)
@@ -1639,11 +1637,12 @@ impl CodewhaleClient {
if !saw_done && !decode_failed {
match flush_sse_line(&mut byte_buf) {
Ok(Some(line)) => {
if let Some(data) = extract_sse_data_value(&line) {
if !line_buf.is_empty() {
line_buf.push('\n');
}
line_buf.push_str(data);
if let Some(data) = extract_sse_data_value(&line)
&& let Err(err) = push_sse_event_data(&mut line_buf, data)
{
decode_failed = true;
stream_failed = true;
yield Err(anyhow::anyhow!("{err}"));
}
}
Ok(None) => {}
+35 -12
View File
@@ -21,7 +21,7 @@ use codewhale_models::{
use super::prepared::WireDialect;
use super::role_placement::{RolePlacement, role_placement};
use super::wire::{extract_sse_data_value, next_sse_line};
use super::wire::{extract_sse_data_value, next_sse_line, push_sse_event_data};
use super::{
CodewhaleClient, ERROR_BODY_MAX_BYTES, bounded_error_text, from_api_tool_name,
system_to_instructions, to_api_tool_name,
@@ -255,6 +255,9 @@ impl CodewhaleClient {
// multi-byte UTF-8 char split across HTTP/2 DATA is never
// corrupted to U+FFFD. Genuine invalid bytes fail closed.
let mut buffer: Vec<u8> = Vec::new();
// `data:` fields of the event being assembled, dispatched at the
// blank line that ends it (or at stream end).
let mut event_data = String::new();
let mut done = false;
let mut ended = false;
let mut content_block_counter: u32 = 0;
@@ -297,25 +300,37 @@ impl CodewhaleClient {
// Process complete SSE lines, and the unterminated tail at stream end.
loop {
let line = match next_sse_line(&mut buffer, ended) {
Ok(Some(line)) => line,
let data = match next_sse_line(&mut buffer, ended) {
// A blank line ends the event.
Ok(Some(line)) if line.is_empty() => std::mem::take(&mut event_data),
Ok(Some(line)) => {
if line.starts_with(':') {
// SSE comment keep-alive: the provider is alive (#6184).
yield Ok(StreamEvent::Ping);
} else if let Some(value) = extract_sse_data_value(&line)
&& let Err(err) = push_sse_event_data(&mut event_data, value)
{
yield Err(anyhow::anyhow!("{err}"));
return;
}
continue;
}
// The final event may arrive without its blank line.
Ok(None) if ended && !event_data.is_empty() => {
std::mem::take(&mut event_data)
}
Ok(None) => break,
Err(err) => {
yield Err(anyhow::anyhow!("{err}"));
return;
}
};
if line.is_empty() {
continue;
}
if line.starts_with(':') {
// SSE comment keep-alive: the provider is alive (#6184).
yield Ok(StreamEvent::Ping);
if data.is_empty() {
continue;
}
if let Some(data) = extract_sse_data_value(&line) {
{
let data = data.as_str();
if data == "[DONE]" {
done = true;
break;
@@ -561,7 +576,15 @@ impl CodewhaleClient {
}
}
// Emit MessageStop.
// Only `[DONE]` or `response.completed`/`response.incomplete`
// proves the response is whole. A bare HTTP EOF is truncation, and
// a MessageStop here would hand the turn a partial answer as done.
if !done {
yield Err(anyhow::anyhow!(
"Responses stream closed before response.completed or [DONE]"
));
return;
}
yield Ok(StreamEvent::MessageStop);
};
+76
View File
@@ -234,6 +234,82 @@ async fn responses_stream_retries_upstream_499_before_streaming() {
assert_eq!(attempts.load(Ordering::SeqCst), 2);
}
async fn collect_responses_stream(sse_body: &'static str) -> Vec<Result<StreamEvent>> {
let server = MockServer::start().await;
Mock::given(method("POST"))
.and(path(CODEX_RESPONSES_PATH))
.respond_with(
ResponseTemplate::new(200)
.insert_header("Content-Type", "text/event-stream")
.set_body_string(sse_body),
)
.mount(&server)
.await;
let client = {
let _env_lock = crate::test_support::lock_test_env();
let _codex_token =
crate::test_support::EnvVarGuard::set("OPENAI_CODEX_ACCESS_TOKEN", "test-token");
let _legacy_codex_token = crate::test_support::EnvVarGuard::remove("CODEX_ACCESS_TOKEN");
CodewhaleClient::new(&test_codex_config(&server)).unwrap()
};
let stream = client
.handle_responses_stream(
&client
.prepare_outbound_request(minimal_responses_request(), true)
.expect("responses request prepares"),
)
.await
.expect("Responses stream opens");
tokio::time::timeout(std::time::Duration::from_secs(5), stream.collect())
.await
.expect("stream ends")
}
#[tokio::test]
async fn responses_stream_eof_without_a_terminal_event_is_an_error() {
let events = collect_responses_stream(concat!(
"data: {\"type\":\"response.output_item.added\",\"item\":{\"type\":\"message\",\"id\":\"m\"}}\n\n",
"data: {\"type\":\"response.output_text.delta\",\"delta\":\"partial\"}\n\n",
))
.await;
assert!(
!events
.iter()
.any(|event| matches!(event, Ok(StreamEvent::MessageStop))),
"a truncated stream must not report MessageStop: {events:?}"
);
assert!(
events
.last()
.is_some_and(|event| event.as_ref().is_err_and(|error| error
.to_string()
.contains("closed before response.completed"))),
"{events:?}"
);
}
#[tokio::test]
async fn responses_stream_joins_multiline_data_fields_into_one_event() {
let events = collect_responses_stream(concat!(
"data: {\"type\":\"response.output_item.added\",\"item\":{\"type\":\"message\",\"id\":\"m\"}}\n\n",
"data: {\"type\":\"response.output_text.delta\",\n",
"data: \"delta\":\"joined\"}\n\n",
"data: [DONE]\n\n",
))
.await;
assert!(
events.iter().any(|event| matches!(
event,
Ok(StreamEvent::ContentBlockDelta {
delta: Delta::TextDelta { text },
..
}) if text == "joined"
)),
"the split event was lost: {events:?}"
);
assert!(matches!(events.last(), Some(Ok(StreamEvent::MessageStop))));
}
#[tokio::test]
async fn responses_stream_finishes_on_semantic_terminal_event_without_done_marker() {
let server = MockServer::start().await;
+30
View File
@@ -98,6 +98,10 @@ pub(super) fn extract_sse_data_value(line: &str) -> Option<&str> {
/// Hard ceiling for one pending SSE line, matching the MCP frame limit.
/// This does not bound the size of a multi-line event or the whole stream.
const MAX_SSE_LINE_BYTES: usize = 8 * 1024 * 1024;
/// Hard ceiling for one assembled SSE event (its `data:` fields joined), so a
/// peer that never sends the blank line ending an event cannot grow memory
/// without bound. Same size as the single-line ceiling.
const MAX_SSE_EVENT_BYTES: usize = MAX_SSE_LINE_BYTES;
/// Invalid or oversized SSE line (or unterminated flush).
///
@@ -109,6 +113,7 @@ const MAX_SSE_LINE_BYTES: usize = 8 * 1024 * 1024;
pub(super) enum SseLineError {
InvalidUtf8 { valid_up_to: usize },
TooLong,
EventTooLong,
}
impl std::fmt::Display for SseLineError {
@@ -121,6 +126,10 @@ impl std::fmt::Display for SseLineError {
f,
"SSE line exceeded {MAX_SSE_LINE_BYTES} bytes (8 MiB) — aborting stream"
),
Self::EventTooLong => write!(
f,
"SSE event exceeded {MAX_SSE_EVENT_BYTES} bytes (8 MiB) — aborting stream"
),
}
}
}
@@ -184,6 +193,27 @@ pub(super) fn flush_sse_line(buffer: &mut Vec<u8>) -> Result<Option<String>, Sse
decoded.map(|line| (!line.is_empty()).then_some(line))
}
/// Append one `data:` field to the event being assembled. The SSE spec joins
/// the `data:` fields of one event with '\n' and dispatches the event at the
/// blank line that ends it; a provider may split one JSON payload across
/// several fields, so parsing each field alone would drop the event. Fails
/// closed (clearing the event) past [`MAX_SSE_EVENT_BYTES`].
pub(super) fn push_sse_event_data(event: &mut String, data: &str) -> Result<(), SseLineError> {
let needed = event
.len()
.saturating_add(usize::from(!event.is_empty()))
.saturating_add(data.len());
if needed > MAX_SSE_EVENT_BYTES {
event.clear();
return Err(SseLineError::EventTooLong);
}
if !event.is_empty() {
event.push('\n');
}
event.push_str(data);
Ok(())
}
/// Next decoded SSE line. When `at_end` is false, wait for `\n`. When `at_end`
/// is true, also flush an unterminated tail (stream closed).
pub(super) fn next_sse_line(
+146 -47
View File
@@ -21,7 +21,6 @@
//! provider-accepted active observation.
use std::fs;
use std::io::Write;
use std::path::{Path, PathBuf};
use std::process::Command;
use std::time::{SystemTime, UNIX_EPOCH};
@@ -288,25 +287,13 @@ impl CloudJobStore {
}
/// Persist a job atomically. Never writes credentials.
///
/// The record is written through a unique temporary file (never a fixed
/// `.json.tmp` two writers would share) and renamed into place while the
/// job's lock is held, so it cannot interleave with a
/// [`Self::save_unless_canceled`] or a cancel.
pub fn save(&self, job: &CloudJob) -> Result<()> {
fs::create_dir_all(&self.root).context("failed to create cloud-jobs directory")?;
let path = self.job_path(&job.id)?;
let tmp = path.with_extension("json.tmp");
let body = serde_json::to_vec_pretty(job).context("failed to encode cloud job")?;
{
let mut file =
fs::File::create(&tmp).context("failed to start a private cloud job write")?;
file.write_all(&body)
.context("failed to write the cloud job record")?;
file.sync_all().ok();
}
#[cfg(unix)]
{
use std::os::unix::fs::PermissionsExt;
let _ = fs::set_permissions(&tmp, fs::Permissions::from_mode(0o600));
}
fs::rename(&tmp, &path).context("failed to commit the cloud job record")?;
Ok(())
self.with_job_lock(&job.id, || self.write_record(job))
}
/// Cancel-authoritative save: refuse to overwrite a `canceled` record.
@@ -319,19 +306,53 @@ impl CloudJobStore {
/// persisted record is already `canceled`, `Ok(true)` after a normal
/// save.
///
/// This is load-check-save: the store is file-backed with no
/// cross-process lock, so the check cannot remove the load→save window
/// entirely — it narrows the clobber window from a whole phase (seconds
/// to minutes) to the span of one save, which is the single-writer
/// discipline this store assumes elsewhere.
/// The load, check and save run under the job's advisory file lock, and
/// [`cancel_job`] flips the record under the same lock, so the check and
/// the write are one step with respect to a cancel in another process.
pub fn save_unless_canceled(&self, job: &CloudJob) -> Result<bool> {
if let Ok(current) = self.load(&job.id)
&& current.status == CloudJobStatus::Canceled
{
return Ok(false);
}
self.save(job)?;
Ok(true)
self.with_job_lock(&job.id, || {
match self.load(&job.id) {
Ok(current) if current.status == CloudJobStatus::Canceled => return Ok(false),
Ok(_) => {}
Err(error)
if error
.downcast_ref::<std::io::Error>()
.is_some_and(|error| error.kind() == std::io::ErrorKind::NotFound) => {}
Err(error) => return Err(error.context("cannot verify the persisted job status")),
}
self.write_record(job)?;
Ok(true)
})
}
/// Run `f` holding the job's exclusive advisory lock
/// (`<root>/<id>.json.lock`). The lock is released when `f` returns.
///
/// Limitation: the lock is advisory, so it serializes Codewhale writers
/// only; it is not reentrant, so `f` must not call a locking method.
fn with_job_lock<T>(&self, id: &str, f: impl FnOnce() -> Result<T>) -> Result<T> {
let path = self.job_path(id)?;
fs::create_dir_all(&self.root).context("failed to create cloud-jobs directory")?;
let lock_file = fs::OpenOptions::new()
.create(true)
.truncate(false)
.read(true)
.write(true)
.open(path.with_extension("json.lock"))
.context("failed to open the cloud job lock")?;
let mut lock = fd_lock::RwLock::new(lock_file);
let _guard = lock
.write()
.context("failed to lock the cloud job record")?;
f()
}
/// Write the record through a unique private temporary file. Callers
/// hold the job lock.
fn write_record(&self, job: &CloudJob) -> Result<()> {
let path = self.job_path(&job.id)?;
let body = serde_json::to_vec_pretty(job).context("failed to encode cloud job")?;
crate::utils::write_atomic(&path, &body).context("failed to commit the cloud job record")
}
/// Load one job by id.
@@ -789,23 +810,32 @@ pub fn cancel_job(
id: &str,
launcher: &dyn DaytonaLauncher,
) -> Result<CloudJob> {
let mut job = store.load(id)?;
if matches!(
job.status,
CloudJobStatus::Canceled | CloudJobStatus::Failed | CloudJobStatus::Refused
) {
return Ok(job);
}
let had_sandbox = job.sandbox_id.is_some();
let sandbox_may_exist = had_sandbox || job.sandbox_pending;
job.status = CloudJobStatus::Canceled;
job.finished_unix = Some(unix_now());
job.note = if sandbox_may_exist {
"Canceled locally; the cloud agent sandbox is being torn down.".to_string()
} else {
"Canceled locally before a sandbox was created.".to_string()
// The flip is one locked read-modify-write, so a runner phase save cannot
// land between this load and this write (and lose either the cancel or
// the sandbox id the runner just recorded).
let flipped = store.with_job_lock(id, || {
let mut job = store.load(id)?;
if matches!(
job.status,
CloudJobStatus::Canceled | CloudJobStatus::Failed | CloudJobStatus::Refused
) {
return Ok((job, false));
}
let sandbox_may_exist = job.sandbox_id.is_some() || job.sandbox_pending;
job.status = CloudJobStatus::Canceled;
job.finished_unix = Some(unix_now());
job.note = if sandbox_may_exist {
"Canceled locally; the cloud agent sandbox is being torn down.".to_string()
} else {
"Canceled locally before a sandbox was created.".to_string()
};
store.write_record(&job)?;
Ok((job, true))
})?;
let mut job = match flipped {
(job, false) => return Ok(job),
(job, true) => job,
};
store.save(&job)?;
if let Some(sandbox_id) = job.sandbox_id.clone() {
let receipt = SandboxReceipt {
sandbox_id,
@@ -3181,6 +3211,75 @@ mod tests {
assert_eq!(persisted.finished_unix, Some(10_000_060));
}
#[test]
fn a_phase_save_preserves_an_unreadable_job_record() {
let temp = tempfile::tempdir().unwrap();
let store = CloudJobStore::from_path(temp.path().join("jobs"));
let job = stored_job(CloudJobStatus::Running, 10_000_000);
// A genuinely absent record can be created by the first phase save.
assert!(store.save_unless_canceled(&job).unwrap());
let path = store.job_path(&job.id).unwrap();
let damaged = b"{\"status\":\"canceled\", interrupted write";
fs::write(&path, damaged).unwrap();
let error = store.save_unless_canceled(&job).unwrap_err();
assert!(format!("{error:#}").contains("cannot verify the persisted job status"));
assert_eq!(fs::read(&path).unwrap(), damaged);
// An I/O failure is uncertainty too, rather than permission to replace it.
fs::remove_file(&path).unwrap();
fs::create_dir(&path).unwrap();
assert!(store.save_unless_canceled(&job).is_err());
assert!(path.is_dir());
}
#[test]
fn a_phase_save_waits_for_a_cancel_holding_the_job_lock() {
let temp = tempfile::tempdir().unwrap();
let store = CloudJobStore::from_path(temp.path().join("jobs"));
let job = stored_job(CloudJobStatus::Running, 10_000_000);
store.save(&job).unwrap();
let mut runner_copy = job.clone();
runner_copy.status = CloudJobStatus::OpeningPr;
// A cancel in progress holds the lock between its load and its write;
// the runner's check-and-save must not slip into that window.
let outcome = store
.with_job_lock(&job.id, || {
let runner_store = store.clone();
let runner = std::thread::spawn(move || {
runner_store.save_unless_canceled(&runner_copy).unwrap()
});
std::thread::sleep(std::time::Duration::from_millis(200));
assert!(
!runner.is_finished(),
"the phase save ran inside the cancel's read-modify-write"
);
let mut canceled = store.load(&job.id)?;
canceled.status = CloudJobStatus::Canceled;
store.write_record(&canceled)?;
Ok(runner)
})
.unwrap()
.join()
.unwrap();
assert!(
!outcome,
"the phase save must see the cancel and stand down"
);
assert_eq!(
store.load(&job.id).unwrap().status,
CloudJobStatus::Canceled
);
assert!(
!temp
.path()
.join("jobs")
.join(format!("{}.json.tmp", job.id))
.exists(),
"records are written through unique temporaries"
);
}
#[test]
fn sweep_fails_stale_active_jobs_and_tears_down_their_sandboxes() {
let temp = tempfile::tempdir().unwrap();
+80 -26
View File
@@ -2348,32 +2348,39 @@ pub fn set_config_value(app: &mut App, key: &str, value: &str, persist: bool) ->
}
let next_path = PathBuf::from(expand_tilde(value));
let path_changed = next_path != app.mcp_config_path;
app.mcp_config_path = next_path;
if path_changed {
app.mcp_reload_required = true;
}
let reload_note = if path_changed {
"; run /mcp reload to rebuild the live tool pool"
} else {
""
};
let message = if persist {
// Persist before touching live state (C01-08): a failed save
// must leave the session exactly as it was, not report failure
// over a path and reload flag that already moved.
let saved_to = if persist {
match persist_root_string_key(app.config_path.as_deref(), "mcp_config_path", value)
{
Ok(path) => format!(
"mcp_config_path = {} (saved to {}){}",
app.mcp_config_path.display(),
path.display(),
reload_note
),
Ok(path) => Some(path),
Err(err) => return CommandResult::error(format!("Failed to save: {err}")),
}
} else {
format!(
None
};
app.mcp_config_path = next_path;
if path_changed {
app.mcp_reload_required = true;
}
let message = match saved_to {
Some(path) => format!(
"mcp_config_path = {} (saved to {}){}",
app.mcp_config_path.display(),
path.display(),
reload_note
),
None => format!(
"mcp_config_path = {} (session only){}",
app.mcp_config_path.display(),
reload_note
)
),
};
return CommandResult::message(message);
}
@@ -2525,7 +2532,6 @@ pub fn set_config_value(app: &mut App, key: &str, value: &str, persist: bool) ->
} else {
raw
};
app.stream_chunk_timeout_secs = resolved;
let value_label = stream_chunk_timeout_value_label(raw, resolved);
if persist {
match persist_table_value_key(
@@ -2535,6 +2541,9 @@ pub fn set_config_value(app: &mut App, key: &str, value: &str, persist: bool) ->
(raw as i64).into(), // validated above: at most 3600
) {
Ok(path) => {
// Live state moves only with the engine action and
// only after the save landed (C01-08).
app.stream_chunk_timeout_secs = resolved;
return CommandResult::with_message_and_action(
format!(
"stream_chunk_timeout_secs = {value_label} (saved to {}; affects subsequent turns in this session)",
@@ -2546,6 +2555,7 @@ pub fn set_config_value(app: &mut App, key: &str, value: &str, persist: bool) ->
Err(err) => return CommandResult::error(format!("Failed to save: {err}")),
}
}
app.stream_chunk_timeout_secs = resolved;
return CommandResult::with_message_and_action(
format!(
"stream_chunk_timeout_secs = {value_label} (session only; affects subsequent turns in this session)"
@@ -2731,14 +2741,8 @@ pub fn set_config_value(app: &mut App, key: &str, value: &str, persist: bool) ->
Ok(value) => value,
Err(err) => return CommandResult::error(err),
};
match field {
"keep_header" => app.mini_window.keep_header = value,
"keep_input" => app.mini_window.keep_input = value,
"keep_todo" => app.mini_window.keep_todo = value,
"keep_sidebar" => app.mini_window.keep_sidebar = value,
"keep_footer" => app.mini_window.keep_footer = value,
_ => unreachable!("mini_window field matched above"),
}
// Persist first: a failed save leaves the live layout as it was
// (C01-08).
if persist
&& let Err(err) = crate::config_persistence::persist_mini_window_bool_key(
app.config_path.as_deref(),
@@ -2748,6 +2752,14 @@ pub fn set_config_value(app: &mut App, key: &str, value: &str, persist: bool) ->
{
return CommandResult::error(format!("Failed to persist: {err}"));
}
match field {
"keep_header" => app.mini_window.keep_header = value,
"keep_input" => app.mini_window.keep_input = value,
"keep_todo" => app.mini_window.keep_todo = value,
"keep_sidebar" => app.mini_window.keep_sidebar = value,
"keep_footer" => app.mini_window.keep_footer = value,
_ => unreachable!("mini_window field matched above"),
}
app.needs_redraw = true;
}
"composer_density" | "composer" => {
@@ -2798,15 +2810,17 @@ pub fn set_config_value(app: &mut App, key: &str, value: &str, persist: bool) ->
app.needs_redraw = true;
}
"workspace_follow_symlinks" | "follow_symlinks" => {
// Persist first: a failed save leaves the live value as it was
// (C01-08).
if persist && let Err(e) = persist_single_setting(&key, value) {
return CommandResult::error(format!("Failed to save: {e}"));
}
app.workspace_follow_symlinks = settings.workspace_follow_symlinks;
app.composer.mention_completion_cache = None;
app.composer.mention_discovery.invalidate();
app.needs_redraw = true;
// Engine tools use EngineConfig which is fixed at startup
return CommandResult::message(if persist {
if let Err(e) = persist_single_setting(&key, value) {
return CommandResult::error(format!("Failed to save: {e}"));
}
format!(
"workspace_follow_symlinks = {} (saved; restart required for engine tools)",
settings.workspace_follow_symlinks
@@ -2904,7 +2918,18 @@ pub fn set_config_value(app: &mut App, key: &str, value: &str, persist: bool) ->
let message = if persist {
if let Err(e) = persist_single_setting(&key, value) {
return CommandResult::error(format!("Failed to save: {e}"));
// C01-08: the projection above already changed live state (and
// some arms validate only while projecting, so saving first could
// persist a value that never applied). Report exactly that partial
// effect, and keep the action so the engine matches the UI instead
// of silently diverging from it.
return CommandResult {
message: Some(format!(
"Error: {key} = {display_value} applies to this session only; saving failed: {e}"
)),
action,
is_error: true,
};
}
format!("{key} = {display_value} (saved)")
} else {
@@ -5384,6 +5409,35 @@ context_window = 262144
);
}
#[test]
fn failed_save_leaves_live_config_state_untouched() {
// C01-08: a `--save` that cannot write must not report failure over
// live state that already moved.
let temp = tempfile::tempdir().unwrap();
let _guard = EnvGuard::new(temp.path());
let blocker = temp.path().join("not-a-directory");
fs::write(&blocker, "a file where the config directory should be").unwrap();
let mut app = create_test_app();
app.config_path = Some(blocker.join("config.toml"));
let mcp_before = app.mcp_config_path.clone();
app.mcp_reload_required = false;
let timeout_before = app.stream_chunk_timeout_secs;
let result = set_config_value(&mut app, "mcp_config_path", "/elsewhere/mcp.json", true);
assert!(result.is_error, "{:?}", result.message);
assert_eq!(app.mcp_config_path, mcp_before);
assert!(!app.mcp_reload_required);
let next_timeout = if timeout_before == 120 { "121" } else { "120" };
let result = set_config_value(&mut app, "stream_chunk_timeout_secs", next_timeout, true);
assert!(result.is_error, "{:?}", result.message);
assert!(
result.action.is_none(),
"no engine update for an unsaved value"
);
assert_eq!(app.stream_chunk_timeout_secs, timeout_before);
}
#[test]
fn config_command_stream_chunk_timeout_save_overrides_canonical_value() {
let nanos = SystemTime::now()
+32 -6
View File
@@ -127,9 +127,19 @@ pub async fn capture_and_transcribe(
if !is_available() {
return Err(tr(locale, MessageId::VoiceErrNoRecorder).to_string());
}
let api_key = config
.active_route_api_key()
.map_err(|_| tr(locale, MessageId::VoiceErrNoAuth).to_string())?;
// C01-10: the credential check follows the ASR backend that will run,
// not the chat provider. Local Whisper and Groq need no provider key, so
// resolve it lazily — the same contract as `crate::voice::dictate_once`
// — and only refuse before recording when provider ASR is the backend.
let provider_key = || {
config
.active_route_api_key()
.map_err(|_| tr(locale, MessageId::VoiceErrNoAuth).to_string())
};
let (asr_kind, _asr_model) = resolve_asr_choice(config);
if asr_kind == "provider" {
provider_key()?;
}
let base_url = config.active_route_base_url();
let openrouter_vendor = config
.openrouter_vendor()
@@ -143,7 +153,6 @@ pub async fn capture_and_transcribe(
// Streaming interim: poll every 700ms and show partial transcript like Grok Build's
// VoiceEvent::Interim → VoiceState::Recording{interim}. We re-transcribe the
// growing buffer (local-whisper is cheap; Groq is ~300ms; provider falls back).
let (asr_kind, _asr_model) = resolve_asr_choice(config);
let interim_enabled = true; // always show partials — feels alive like Spark
// Spawn recorder on blocking thread with a shared buffer for interim polling.
@@ -241,13 +250,30 @@ pub async fn capture_and_transcribe(
let text = match asr_kind.as_str() {
"local-whisper" => match transcribe_local_whisper(&samples).await {
Ok(v) => Ok(v),
Err(_) => transcribe(&api_key, &base_url, &samples, openrouter_vendor.as_deref()).await,
Err(_) => {
transcribe(
&provider_key()?,
&base_url,
&samples,
openrouter_vendor.as_deref(),
)
.await
}
},
"groq" => match transcribe_groq(&samples).await {
Ok(v) => Ok(v),
Err(_) => transcribe(&api_key, &base_url, &samples, openrouter_vendor.as_deref()).await,
Err(_) => {
transcribe(
&provider_key()?,
&base_url,
&samples,
openrouter_vendor.as_deref(),
)
.await
}
},
_ => {
let api_key = provider_key()?;
if app.voice_control_enabled {
process_voice_control(
&api_key,
+18 -14
View File
@@ -350,8 +350,9 @@ pub enum ComputerMeterError {
/// Interval ends before it starts.
#[error("A Computer meter interval cannot end before it starts.")]
IntervalReversed,
/// Admission has expired before the interval started.
#[error("The Computer admission expired before this interval.")]
/// Admission expired before the interval ended. An interval must lie
/// wholly inside the admission; the tail after expiry was never authorized.
#[error("The Computer admission expired before this interval ended.")]
AdmissionExpired,
/// Meter or catalog revision is not the pinned v3 revision.
#[error("{message}")]
@@ -479,7 +480,10 @@ pub fn issue_computer_meter_receipt(
return Err(ComputerMeterError::IntervalReversed);
}
let expires = parse_timestamp(&admission.expires_at, "expiresAt")?;
if started >= expires {
// `ended_at` is exclusive, so an interval ending exactly at expiry is
// wholly authorized; one that runs past it is refused rather than clipped,
// because a clipped receipt would no longer restate the provider interval.
if started >= expires || ended > expires {
return Err(ComputerMeterError::AdmissionExpired);
}
let accepted_seconds = elapsed_whole_seconds(started, ended);
@@ -539,21 +543,21 @@ pub fn issue_computer_meter_receipt(
Ok(receipt)
}
/// Exact replay of an existing receipt. Identity and digest must match.
/// Exact replay of an existing receipt. Every field must match (only the
/// `replay_of` back-reference may differ), and the incoming digest and id must
/// be the ones its own fields produce: a caller-supplied digest is never
/// trusted to stand for fields it does not cover.
pub fn assert_computer_meter_receipt_replay(
existing: &ComputerMeterReceipt,
incoming: &ComputerMeterReceipt,
) -> Result<(), ComputerMeterError> {
if existing.receipt_id == incoming.receipt_id
&& existing.binding_digest == incoming.binding_digest
&& existing.admission_id == incoming.admission_id
&& existing.account_id == incoming.account_id
&& existing.provider_event_ref == incoming.provider_event_ref
&& existing.accepted_seconds == incoming.accepted_seconds
&& existing.standard_equivalent_seconds == incoming.standard_equivalent_seconds
&& existing.profile_id == incoming.profile_id
&& existing.multiplier == incoming.multiplier
{
let self_consistent = receipt_binding_digest(incoming) == incoming.binding_digest
&& receipt_id_for(incoming) == incoming.receipt_id;
let same_terms = ComputerMeterReceipt {
replay_of: existing.replay_of.clone(),
..incoming.clone()
} == *existing;
if self_consistent && same_terms {
return Ok(());
}
Err(ComputerMeterError::ReplayConflict)
+55
View File
@@ -190,6 +190,61 @@ fn exact_replay_is_idempotent_and_a_mutated_binding_conflicts() {
);
}
#[test]
fn replay_rejects_fields_the_supplied_digest_does_not_vouch_for() {
let case = fixtures()
.into_iter()
.find(|case| case.id == "accepted-standard-8")
.unwrap();
let first = issue_case(&case).unwrap();
// Same id and digest, different interval: the digest no longer matches
// the fields it claims to bind.
let mut shifted = first.clone();
shifted.started_at = "2026-08-31T13:00:00.000Z".to_string();
shifted.ended_at = "2026-08-31T13:10:00.000Z".to_string();
assert_eq!(
assert_computer_meter_receipt_replay(&first, &shifted)
.unwrap_err()
.code(),
"computer_meter_receipt_replay_conflict"
);
// A field outside the digest must still match exactly.
let mut restated = first.clone();
restated.provider_allocation.accepted = false;
assert_eq!(
assert_computer_meter_receipt_replay(&first, &restated)
.unwrap_err()
.code(),
"computer_meter_receipt_replay_conflict"
);
}
#[test]
fn an_interval_that_runs_past_admission_expiry_is_refused() {
let case = fixtures()
.into_iter()
.find(|case| case.id == "accepted-standard-8")
.unwrap();
let mut admission_request = case.admission.clone().unwrap();
let observation = case.observation.clone().unwrap();
// Starts before expiry, ends after it.
admission_request.expires_at = "2026-08-31T12:05:00.000Z".to_string();
let admission = bind_computer_admission(admission_request.clone()).unwrap();
assert_eq!(
issue_computer_meter_receipt(&admission, observation.clone())
.unwrap_err()
.code(),
"computer_admission_expired"
);
// Ending exactly at expiry is inside the admission (end is exclusive).
admission_request.expires_at = observation.ended_at.clone();
let admission = bind_computer_admission(admission_request).unwrap();
assert!(issue_computer_meter_receipt(&admission, observation).is_ok());
}
#[test]
fn admission_binds_profile_multiplier_account_and_expiry_before_dispatch() {
let admission = bind_computer_admission(ComputerAdmissionRequest {
+115 -39
View File
@@ -912,11 +912,17 @@ pub struct Engine {
/// the channel closing (task exited with the pool) disarms it and the
/// next pool ensure respawns.
mcp_supervisor_rx: Option<mpsc::Receiver<McpSupervisorUpdate>>,
/// Abort the supervisor's reconnect handshakes when its pool is retired.
mcp_supervisor_task: Option<tokio::task::AbortHandle>,
mcp_boot_done: Option<tokio::sync::watch::Receiver<bool>>,
/// Generation owned by the currently installed boot receiver. Terminal
/// cleanup is conditional on this exact value so an older pass can never
/// clear a newer receiver.
mcp_boot_generation: Option<u64>,
/// Abort handle for the boot pass that owns `mcp_boot_generation`. A
/// session or workspace boundary drops the pool that pass is dialing
/// into, so it aborts the pass instead of letting it run on (C02-08).
mcp_boot_task: Option<tokio::task::AbortHandle>,
/// Monotonic generation for engine-authored MCP session snapshots. Boot
/// task updates retain their spawn generation so later passes can reject
/// only genuinely stale work.
@@ -2060,8 +2066,10 @@ impl Engine {
mcp_boot_in_flight: false,
mcp_boot_rx: None,
mcp_supervisor_rx: None,
mcp_supervisor_task: None,
mcp_boot_done: None,
mcp_boot_generation: None,
mcp_boot_task: None,
mcp_event_generation: 0,
plugin_registry,
extension_host,
@@ -2789,7 +2797,12 @@ impl Engine {
// Runtime-added MCP servers are conversation capabilities even when
// both conversations use the same workspace. Configured servers can
// reconnect lazily after the new session is installed.
self.mcp_pool = None;
self.drop_mcp_pool();
// C02-17: cumulative usage belongs to the conversation that spent
// it. The sync carries no prior total for the installed session, so
// it starts from zero instead of inheriting the previous session's
// tokens into its snapshot and saved `total_tokens`.
self.session.total_usage = Default::default();
self.session.id = next_session_id;
Some(previous_session_id)
}
@@ -3645,7 +3658,7 @@ impl Engine {
self.config.plugin_registry = Some(Arc::clone(&self.plugin_registry));
// A pool may contain plugin servers and authority
// receipts from the previous workspace snapshot.
self.mcp_pool = None;
self.drop_mcp_pool();
if let Some(attachment) = &self.extension_host {
attachment.set_plugins(Arc::clone(&self.plugin_registry));
attachment.sync_in_background();
@@ -3841,39 +3854,65 @@ impl Engine {
continue;
}
};
// C02-02: stage the cut. The removed exchange is kept
// until the replacement turn actually starts; a send
// that never starts puts it back (below).
let removed_exchange = self
.session
.messages
.get(idx..)
.map(<[_]>::to_vec)
.unwrap_or_default();
self.session.messages.truncate_to(idx);
self.session.bump_messages_revision();
// Now dispatch the new message as a normal send,
// reusing the engine's stored mode/model config.
let mode = self.current_mode;
self.handle_send_message(TurnSpec {
content: new_message.clone(),
mode,
route: Box::new(route),
compaction: Box::new(self.config.compaction.clone()),
initial_routed_usage: Box::new(
crate::cost_status::RuntimeUsageBatch::default(),
),
goal_objective: self.config.goal_objective.clone(),
goal_token_budget: self.config.goal_token_budget,
goal_status: self.config.goal_status,
reasoning_effort: self.session.reasoning_effort.clone(),
reasoning_effort_auto: self.session.reasoning_effort_auto,
auto_model: self.session.auto_model,
allow_shell: self.session.allow_shell,
trust_mode: self.session.trust_mode,
auto_approve: self.session.auto_approve,
approval_mode: self.session.approval_mode,
translation_enabled: self.config.translation_enabled,
allowed_tools: self.config.allowed_tools.clone(),
dynamic_tools: Vec::new(),
hook_executor: self.config.hook_executor.clone(),
verbosity: self.config.verbosity.clone(),
provenance: UserInputProvenance::ExternalUser,
images: Vec::new(),
max_output_tokens: None,
})
.await;
let outcome = self
.handle_send_message(TurnSpec {
content: new_message.clone(),
mode,
route: Box::new(route),
compaction: Box::new(self.config.compaction.clone()),
initial_routed_usage: Box::new(
crate::cost_status::RuntimeUsageBatch::default(),
),
goal_objective: self.config.goal_objective.clone(),
goal_token_budget: self.config.goal_token_budget,
goal_status: self.config.goal_status,
reasoning_effort: self.session.reasoning_effort.clone(),
reasoning_effort_auto: self.session.reasoning_effort_auto,
auto_model: self.session.auto_model,
allow_shell: self.session.allow_shell,
trust_mode: self.session.trust_mode,
auto_approve: self.session.auto_approve,
approval_mode: self.session.approval_mode,
translation_enabled: self.config.translation_enabled,
allowed_tools: self.config.allowed_tools.clone(),
dynamic_tools: Vec::new(),
hook_executor: self.config.hook_executor.clone(),
verbosity: self.config.verbosity.clone(),
provenance: UserInputProvenance::ExternalUser,
images: Vec::new(),
max_output_tokens: None,
})
.await;
if matches!(outcome, SendMessageOutcome::NotStarted { .. }) {
// Anything the failed send appended after the cut
// (a drained shell-completion notice) stays, after
// the restored exchange.
let appended = self
.session
.messages
.get(idx..)
.map(<[_]>::to_vec)
.unwrap_or_default();
self.session.messages.truncate_to(idx);
self.session.messages.push_batch(removed_exchange);
self.session.messages.push_batch(appended);
self.session.bump_messages_revision();
self.emit_session_updated().await;
}
}
Op::SetAdvisorEnabled { enabled } => {
self.config.advisor_config.enabled = enabled;
@@ -6711,29 +6750,42 @@ impl Engine {
self.ensure_mcp_supervisor();
return Ok(pool);
}
// C02-18: misconfiguration fails loud. A missing file is an ordinary
// empty config (`load_config` returns the default); an unreadable or
// malformed one still yields an empty, source-aware pool so a fixed
// file and `/mcp reload` recover in-process — but the person is told
// that the configured servers are gone, not left to find no tools.
let mut load_failure = None;
let mut pool = McpPool::from_config_path_with_workspace_and_plugins(
&self.session.mcp_config_path,
&self.session.workspace,
Arc::clone(&self.plugin_registry),
)
.unwrap_or_else(|e| {
tracing::debug!(
"MCP config unavailable: {}",
crate::mcp::format_mcp_error_for_display(&e)
);
let reason = crate::mcp::format_mcp_error_for_display(&e);
tracing::warn!("MCP config unavailable: {reason}");
load_failure = Some(reason);
McpPool::empty_with_workspace_config_sources(
&self.session.mcp_config_path,
&self.session.workspace,
Arc::clone(&self.plugin_registry),
)
.unwrap_or_else(|fallback_error| {
tracing::debug!(
tracing::warn!(
"MCP reload source setup failed: {}",
crate::mcp::format_mcp_error_for_display(&fallback_error)
);
McpPool::new(McpConfig::default())
})
});
if let Some(reason) = load_failure {
let _ = self
.tx_event
.send(Event::status(format!(
"MCP config could not be loaded, so none of its servers are available: {reason}. Fix it, then run /mcp reload."
)))
.await;
}
pool = pool.with_disallowed_tools(self.config.disallowed_tools.clone().unwrap_or_default());
if let Some(decider) = self.config.network_policy.as_ref() {
pool = pool.with_network_policy(decider.clone());
@@ -6752,8 +6804,8 @@ impl Engine {
}
/// Start the connection supervisor once per pool. The task holds only a
/// Weak: pool replacement lets the old task exit, its channel closes, the
/// run loop disarms, and the next ensure respawns against the new pool.
/// Weak between sweeps, but an in-flight handshake holds the pool. Keep
/// its abort handle so a boundary stops that work and queued diagnoses.
fn ensure_mcp_supervisor(&mut self) {
if self.mcp_supervisor_rx.is_some() {
return;
@@ -6764,11 +6816,12 @@ impl Engine {
let (tx, rx) = mpsc::channel(16);
self.mcp_supervisor_rx = Some(rx);
let weak = Arc::downgrade(pool);
spawn_supervised(
let task = spawn_supervised(
"mcp-supervisor",
std::panic::Location::caller(),
McpPool::supervise_pool(weak, tx),
);
self.mcp_supervisor_task = Some(task.abort_handle());
}
/// Apply one supervisor sweep: deaths and failures refresh the engine's
@@ -6886,12 +6939,34 @@ impl Engine {
return false;
}
self.mcp_boot_generation = None;
self.mcp_boot_task = None;
self.mcp_boot_in_flight = false;
self.mcp_boot_rx = None;
self.mcp_boot_done = None;
true
}
/// Drop the engine-owned MCP pool at a session or workspace boundary.
/// The in-flight boot pass dials into the dropped pool: abort it and
/// clear its generation, receiver and diagnoses, so the next pool starts
/// clean instead of waiting on — or adopting progress and connection
/// errors from — the previous conversation's pass (C02-08).
fn drop_mcp_pool(&mut self) {
self.mcp_pool = None;
if let Some(task) = self.mcp_boot_task.take() {
task.abort();
}
if let Some(task) = self.mcp_supervisor_task.take() {
task.abort();
}
self.mcp_supervisor_rx = None;
self.mcp_boot_generation = None;
self.mcp_boot_in_flight = false;
self.mcp_boot_rx = None;
self.mcp_boot_done = None;
self.mcp_connection_errors.clear();
}
async fn emit_mcp_session_boot(&self, generation: u64, finished: bool) {
let Ok(snapshot) = self.mcp_session_snapshot().await else {
return;
@@ -7337,7 +7412,7 @@ impl Engine {
self.emit_mcp_session_boot(generation, false).await;
let pool_for_task = Arc::clone(&pool);
spawn_supervised(
let boot_task = spawn_supervised(
"mcp-session-boot",
std::panic::Location::caller(),
async move {
@@ -7419,6 +7494,7 @@ impl Engine {
let _ = done_tx.send(true);
},
);
self.mcp_boot_task = Some(boot_task.abort_handle());
Ok(generation)
}
+24 -2
View File
@@ -287,14 +287,36 @@ impl Engine {
tool_id: &str,
request: UserInputRequest,
) -> Result<UserInputResponse, ToolError> {
let _ = self
// C02-19: a question that never reached a host has nobody to answer
// it. Fail now instead of waiting out the timeout — which by default
// is no timeout at all.
if self
.tx_event
.send(Event::UserInputRequired {
id: tool_id.to_string(),
request,
})
.await;
.await
.is_err()
{
return Err(ToolError::execution_failed(
"User input request could not reach its host, so nobody was asked. \
Continue without the answer or ask in your reply instead."
.to_string(),
));
}
// R1, as for tool approval: the per-turn wall-clock budget bounds the
// agent's own time, not how long a person takes to answer.
self.turn_wall_clock.begin_human_wait();
let response = self.await_user_input_decision(tool_id).await;
self.turn_wall_clock.end_human_wait();
response
}
async fn await_user_input_decision(
&mut self,
tool_id: &str,
) -> Result<UserInputResponse, ToolError> {
// #6003: `[tools] user_input_timeout_seconds`. Absent, or an explicit
// 0, waits until the person answers or cancels. A positive value is
// one absolute deadline for the whole wait: `select!` drops the
+1 -1
View File
@@ -38,7 +38,7 @@ impl ToolUseState {
}
}
/// Maximum total bytes of text/thinking content before aborting the stream.
/// Maximum total bytes of text, reasoning and tool-argument content before aborting the stream.
pub(super) const STREAM_MAX_CONTENT_BYTES: usize = 10 * 1024 * 1024; // 10 MB
/// Sanity backstop for total stream wall-clock duration. **Not** a routine
/// kill switch — the stream chunk idle timeout is the primary stall
+31 -1
View File
@@ -25109,7 +25109,7 @@ async fn reasoning_only_length_stop_fails_without_retry() {
/// each carry exactly one extra message over the same baseline.
#[tokio::test]
async fn the_reasoning_only_nudge_rides_one_request_and_never_joins_the_session() {
let (model, _events) = run_reasoning_only_turn_with_reprompts(usize::MAX, "stop", 3).await;
let (model, events) = run_reasoning_only_turn_with_reprompts(usize::MAX, "stop", 3).await;
let requests = model.requests.lock().expect("captured requests").clone();
assert_eq!(requests.len(), 4, "one initial request plus three retries");
@@ -25144,6 +25144,34 @@ async fn the_reasoning_only_nudge_rides_one_request_and_never_joins_the_session(
carries_nudge(&requests[2]),
"nudge present once retrying again"
);
// C02-04: model-visible means logged. Every request the nudge rides
// leaves a durable (internal) receipt carrying its exact text.
let receipts: Vec<&str> = events
.iter()
.filter_map(|event| match event {
Event::Status { message }
if message.starts_with(super::turn_loop::REQUEST_NUDGE_RECEIPT_PREFIX) =>
{
Some(message.as_str())
}
_ => None,
})
.collect();
assert_eq!(
receipts.len(),
requests
.iter()
.filter(|request| carries_nudge(request))
.count(),
"one receipt per nudged request"
);
assert!(receipts.iter().all(|receipt| receipt.ends_with(nudge)));
assert_eq!(
crate::core::events::status_visibility(receipts[0]),
crate::core::events::StatusVisibility::Internal,
"durable clients keep the receipt, collapsed"
);
}
/// A model that only ever returns reasoning is bounded: it retries up to the
@@ -27443,6 +27471,8 @@ fn engine_adopts_host_owned_session_id_from_config() {
);
}
mod admission_gates;
mod runtime_state;
mod sse_turn_recovery;
mod tool_cancellation;
@@ -0,0 +1,382 @@
//! Admission gates between a provider response and tool execution: a failed
//! stream (C02-05), the step-budget final report (C02-10), the stream content
//! cap over tool arguments (C02-13), and bounded tool errors (C02-12). Every
//! fixture counts handler entries; none runs a shell or reaches a provider.
use super::*;
use crate::llm_client::mock::{MockLlmClient, canned};
use crate::tools::spec::{ApprovalRequirement, ToolCapability, ToolSpec};
use std::sync::atomic::{AtomicUsize, Ordering};
const WRITE_TOOL: &str = "fixture_write";
/// A mutating tool that needs no approval, so a call that is admitted runs.
struct CountingWriteTool(Arc<AtomicUsize>);
#[async_trait::async_trait]
impl ToolSpec for CountingWriteTool {
fn name(&self) -> &str {
WRITE_TOOL
}
fn description(&self) -> &str {
"Count every execution of a mutating fixture."
}
fn input_schema(&self) -> Value {
json!({"type": "object"})
}
fn capabilities(&self) -> Vec<ToolCapability> {
vec![ToolCapability::WritesFiles]
}
fn approval_requirement(&self) -> ApprovalRequirement {
ApprovalRequirement::Auto
}
async fn execute(&self, _: Value, _: &ToolContext) -> Result<ToolResult, ToolError> {
self.0.fetch_add(1, Ordering::SeqCst);
Ok(ToolResult::success("wrote"))
}
}
fn write_surface(
engine: &Engine,
workspace: &Path,
executions: Arc<AtomicUsize>,
) -> ToolSurfacePolicy {
let mut registry = crate::tools::ToolRegistry::new(ToolContext::new(workspace));
registry.register(Arc::new(CountingWriteTool(executions)));
let tools = Some(registry.to_api_tools_with_cache(true));
test_tool_surface(engine, registry, tools, AppMode::Agent)
}
/// Streams a scripted response per call, `Err` items included, which the
/// queue-driven mock cannot express.
struct ScriptedStreamClient {
calls: AtomicUsize,
script: fn(usize) -> Vec<anyhow::Result<StreamEvent>>,
}
#[async_trait::async_trait]
impl crate::core::model_client::ModelClient for ScriptedStreamClient {
fn provider_name(&self) -> &str {
"scripted"
}
fn model(&self) -> &str {
"scripted-model"
}
async fn create_message(
&self,
_request: codewhale_models::MessageRequest,
) -> anyhow::Result<codewhale_models::MessageResponse> {
anyhow::bail!("scripted fixture only streams")
}
async fn create_message_stream(
&self,
_request: codewhale_models::MessageRequest,
) -> anyhow::Result<crate::llm_client::StreamEventBox> {
let call = self.calls.fetch_add(1, Ordering::SeqCst);
Ok(Box::pin(futures_util::stream::iter((self.script)(call))))
}
async fn health_check(&self) -> anyhow::Result<bool> {
Ok(true)
}
}
async fn drain_events(handle: &EngineHandle) -> Vec<Event> {
let mut rx = handle.rx_event.write().await;
std::iter::from_fn(|| rx.try_recv().ok()).collect()
}
fn not_started(result: &Result<ToolResult, ToolError>) -> bool {
result.as_ref().is_ok_and(|output| {
!output.success
&& output
.metadata
.as_ref()
.is_some_and(|metadata| metadata["side_effect_status"] == "not_started")
})
}
#[tokio::test]
async fn terminal_stream_error_after_a_complete_tool_call_never_reaches_the_handler() {
// C02-05: a complete mutating call followed by a provider error frame.
let workspace = tempdir().unwrap();
let mock = Arc::new(MockLlmClient::new(vec![
vec![
canned::message_start("tool-then-error"),
canned::tool_use_block_start(0, "call-write", WRITE_TOOL),
canned::tool_input_delta(0, r#"{"path":"target.txt"}"#),
canned::block_stop(0),
StreamEvent::Error {
error: json!({ "message": "Model not exist." }),
},
],
canned::simple_text_turn("this second request must never be issued"),
]));
let (mut engine, handle) = Engine::new_with_model_client(
deterministic_engine_config(workspace.path()),
&Config::default(),
mock.clone(),
);
engine.session.auto_approve = true;
let executions = Arc::new(AtomicUsize::new(0));
let surface = write_surface(&engine, workspace.path(), executions.clone());
let mut turn = crate::core::turn::TurnContext::new(4);
let (status, error) = engine.run_turn(&mut turn, surface, None, None).await;
assert_eq!(
executions.load(Ordering::SeqCst),
0,
"the handler must never run"
);
assert_eq!(status, TurnOutcomeStatus::Failed);
assert!(
error
.as_deref()
.is_some_and(|e| e.contains("Model not exist.")),
"{error:?}"
);
assert_eq!(
mock.call_count(),
1,
"a failed response authorizes no further request"
);
assert_eq!(
turn.stop_diagnostics.last_response_tool_calls_suppressed,
Some(1)
);
let completions: Vec<_> = drain_events(&handle)
.await
.into_iter()
.filter_map(|event| match event {
Event::ToolCallComplete { result, .. } => Some(result),
_ => None,
})
.collect();
assert_eq!(completions.len(), 1, "the call is settled exactly once");
assert!(not_started(&completions[0]), "{:?}", completions[0]);
assert!(
!engine
.session
.messages
.iter()
.flat_map(|message| &message.content)
.any(|block| matches!(
block,
ContentBlock::ToolUse { .. } | ContentBlock::ToolResult { .. }
)),
"no unpaired or fabricated tool history"
);
}
#[tokio::test]
async fn exhausted_stream_resume_never_executes_the_failed_batch() {
// C02-05, retry path: every attempt streams a complete call and then
// drops. Each resume discards its batch; the last, unretried attempt is
// refused at admission instead of executing.
fn tool_then_drop(_call: usize) -> Vec<anyhow::Result<StreamEvent>> {
vec![
Ok(canned::message_start("tool-then-drop")),
Ok(canned::tool_use_block_start(0, "call-write", WRITE_TOOL)),
Ok(canned::tool_input_delta(0, "{}")),
Ok(canned::block_stop(0)),
Err(anyhow::anyhow!(
"Stream read error: error decoding response body"
)),
]
}
let workspace = tempdir().unwrap();
let client = Arc::new(ScriptedStreamClient {
calls: AtomicUsize::new(0),
script: tool_then_drop,
});
let (mut engine, _handle) = Engine::new_with_model_client(
EngineConfig {
terminal_chrome_enabled: false,
..deterministic_engine_config(workspace.path())
},
&Config::default(),
client.clone(),
);
engine.session.auto_approve = true;
let resumes = engine.config.stream_retry_limits.max_resumes as usize;
let executions = Arc::new(AtomicUsize::new(0));
let surface = write_surface(&engine, workspace.path(), executions.clone());
let mut turn = crate::core::turn::TurnContext::new(4);
let (status, error) = engine.run_turn(&mut turn, surface, None, None).await;
assert_eq!(
executions.load(Ordering::SeqCst),
0,
"no attempt's batch may run"
);
assert_eq!(client.calls.load(Ordering::SeqCst), 1 + resumes);
assert_eq!(status, TurnOutcomeStatus::Failed);
assert!(
error
.as_deref()
.is_some_and(|e| e.contains("error decoding response body")),
"{error:?}"
);
}
#[tokio::test]
async fn tool_argument_bytes_count_toward_the_stream_content_cap() {
// C02-13: a response made only of tool-argument JSON must still hit the
// per-step content cap, and the capped call must not run.
let workspace = tempdir().unwrap();
let huge_args = format!(r#"{{"blob":"{}"}}"#, "x".repeat(4096));
let mock = Arc::new(MockLlmClient::new(vec![
canned::tool_call_turn("call-big", WRITE_TOOL, &huge_args),
canned::simple_text_turn("this second request must never be issued"),
]));
let (mut engine, _handle) = Engine::new_with_model_client(
EngineConfig {
stream_max_content_bytes: 64,
..deterministic_engine_config(workspace.path())
},
&Config::default(),
mock.clone(),
);
engine.session.auto_approve = true;
let executions = Arc::new(AtomicUsize::new(0));
let surface = write_surface(&engine, workspace.path(), executions.clone());
let mut turn = crate::core::turn::TurnContext::new(4);
let (status, _error) = engine.run_turn(&mut turn, surface, None, None).await;
assert_eq!(executions.load(Ordering::SeqCst), 0);
assert_eq!(status, TurnOutcomeStatus::Failed);
assert_eq!(mock.call_count(), 1);
}
#[tokio::test]
async fn step_budget_final_report_is_report_only() {
// C02-10: the one response granted after the step budget is spent asks
// for no tools, and a call it returns anyway is refused, not executed.
let workspace = tempdir().unwrap();
let mock = Arc::new(MockLlmClient::new(vec![
canned::tool_call_turn("call-step-1", WRITE_TOOL, "{}"),
canned::tool_call_turn("call-final-report", WRITE_TOOL, "{}"),
]));
let (mut engine, handle) = Engine::new_with_model_client(
deterministic_engine_config(workspace.path()),
&Config::default(),
mock.clone(),
);
engine.session.auto_approve = true;
let executions = Arc::new(AtomicUsize::new(0));
let surface = write_surface(&engine, workspace.path(), executions.clone());
let mut turn = crate::core::turn::TurnContext::new(1);
let (status, error) = engine.run_turn(&mut turn, surface, None, None).await;
assert_eq!(
executions.load(Ordering::SeqCst),
1,
"only the in-budget step runs"
);
assert_eq!(status, TurnOutcomeStatus::Failed);
assert!(
error
.as_deref()
.is_some_and(|e| e.contains("Maximum model steps")),
"{error:?}"
);
let requests = mock.captured_requests();
assert_eq!(requests.len(), 2);
assert_ne!(requests[0].tool_choice, Some(json!("none")));
assert_eq!(requests[1].tool_choice, Some(json!("none")));
let refused = drain_events(&handle).await.into_iter().any(|event| {
matches!(
event,
Event::ToolCallComplete {
model_call: Some(ref call),
result: Err(ToolError::PermissionDenied { ref message }),
..
} if call.provider_id == "call-final-report" && message.contains("final report")
)
});
assert!(refused, "the final-report call is refused with the reason");
}
#[test]
fn oversized_tool_errors_are_bounded_before_fanout() {
// C02-12: an error is published to the event stream and the session like
// a result, so an oversized one gets the same bounded projection.
struct HugeErrorTool {
as_result: bool,
}
#[async_trait::async_trait]
impl ToolSpec for HugeErrorTool {
fn name(&self) -> &str {
"fixture_huge_error"
}
fn description(&self) -> &str {
"Fail with an oversized message."
}
fn input_schema(&self) -> Value {
json!({"type": "object"})
}
fn capabilities(&self) -> Vec<ToolCapability> {
vec![ToolCapability::ReadOnly]
}
async fn execute(&self, _: Value, _: &ToolContext) -> Result<ToolResult, ToolError> {
let message = format!("FIRST{}LAST", "e".repeat(300_000));
if self.as_result {
Ok(ToolResult::error(message))
} else {
Err(ToolError::execution_failed(message))
}
}
}
with_artifact_home(|home| {
tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.unwrap()
.block_on(async {
for as_result in [false, true] {
let mock = Arc::new(MockLlmClient::new(vec![
canned::tool_call_turn("call-huge", "fixture_huge_error", "{}"),
canned::simple_text_turn("done"),
]));
let (mut engine, handle) = Engine::new_with_model_client(
deterministic_engine_config(home),
&Config::default(),
mock.clone(),
);
let mut registry = crate::tools::ToolRegistry::new(ToolContext::new(home));
registry.register(Arc::new(HugeErrorTool { as_result }));
let tools = Some(registry.to_api_tools_with_cache(true));
let surface = test_tool_surface(&engine, registry, tools, AppMode::Agent);
let mut turn = crate::core::turn::TurnContext::new(4);
let (status, error) = engine.run_turn(&mut turn, surface, None, None).await;
assert_eq!(status, TurnOutcomeStatus::Completed, "{error:?}");
let published = drain_events(&handle)
.await
.into_iter()
.find_map(|event| match event {
Event::ToolCallComplete { result, .. } => Some(match result {
Ok(output) => output.content,
Err(error) => error.to_string(),
}),
_ => None,
})
.expect("the call completes");
assert!(
published.len() < crate::tools::truncate::SPILLOVER_THRESHOLD_BYTES,
"as_result={as_result}: {} bytes published",
published.len()
);
assert!(published.contains("FIRST"), "the head survives");
assert!(published.contains("LAST"), "the tail survives");
assert!(
published.contains(crate::tools::truncate::SPILLOVER_RECOVERY_HINT),
"the omitted range stays recoverable"
);
}
});
});
}
@@ -0,0 +1,321 @@
//! Engine state that must stay truthful across a boundary: the live MCP
//! catalog (C02-07/C02-16), a session switch (C02-08/C02-17), a user-input
//! wait (C02-19), an `/edit` whose replacement never starts (C02-02), and a
//! malformed MCP config (C02-18). No fixture reaches a provider or a shell.
use super::*;
use crate::llm_client::mock::MockLlmClient;
fn quiet_engine(config: EngineConfig) -> (Engine, EngineHandle) {
Engine::new_with_model_client(
config,
&Config::default(),
Arc::new(MockLlmClient::new(Vec::new())),
)
}
async fn session_snapshot(handle: &EngineHandle) -> SessionSnapshot {
let (tx, rx) = tokio::sync::oneshot::channel();
handle
.send(Op::GetSessionSnapshot {
tx: Arc::new(StdMutex::new(Some(tx))),
})
.await
.unwrap();
tokio::time::timeout(model_turn_event_timeout(), rx)
.await
.expect("snapshot response")
.expect("snapshot")
}
#[test]
fn live_mcp_refresh_drops_tools_the_pool_no_longer_lists() {
// C02-07: the caller's universe is what the pool lists *now*. After a
// live 401 that is only the synthetic login tool — never the dead one.
let mut catalog = vec![api_tool("mcp_alpha_search"), api_tool("exec_shell")];
let mut active: HashSet<String> = ["mcp_alpha_search", "exec_shell"]
.into_iter()
.map(str::to_string)
.collect();
let universe: HashSet<String> = ["mcp_alpha_authenticate".to_string()].into();
let changed = replace_runtime_mcp_tools(
&mut catalog,
&mut active,
&universe,
vec![api_tool("mcp_alpha_authenticate")],
AppMode::Agent,
&HashSet::new(),
crate::model_profile::ToolSurfaceBudget::Standard,
);
let names: Vec<&str> = catalog.iter().map(|tool| tool.name.as_str()).collect();
assert_eq!(names, vec!["exec_shell", "mcp_alpha_authenticate"]);
assert!(
!active.contains("mcp_alpha_search"),
"a dead tool is not callable"
);
assert!(changed);
}
#[test]
fn live_mcp_refresh_reports_a_schema_only_edit() {
// C02-16: the same names with an edited description is a change the
// prefix check must hear about; an identical refresh is not.
let refresh = |catalog: &mut Vec<Tool>, active: &mut HashSet<String>, tool: Tool| {
replace_runtime_mcp_tools(
catalog,
active,
&["mcp_alpha_read".to_string()].into(),
vec![tool],
AppMode::Agent,
&HashSet::new(),
crate::model_profile::ToolSurfaceBudget::Standard,
)
};
let mut catalog = vec![api_tool("exec_shell")];
let mut active = HashSet::new();
refresh(&mut catalog, &mut active, api_tool("mcp_alpha_read"));
let mut edited = api_tool("mcp_alpha_read");
edited.description = "Read a resource (now paginated)".to_string();
assert!(refresh(&mut catalog, &mut active, edited.clone()));
assert!(!refresh(&mut catalog, &mut active, edited));
}
async fn engine_mid_mcp_boot_with_usage() -> (Engine, EngineHandle, tokio::task::JoinHandle<()>) {
let workspace = tempdir().unwrap();
let (mut engine, handle) = quiet_engine(deterministic_engine_config(workspace.path()));
engine.session.total_usage.input_tokens = 1_000;
engine.session.total_usage.output_tokens = 200;
let boot = tokio::spawn(std::future::pending::<()>());
engine.mcp_boot_task = Some(boot.abort_handle());
engine.mcp_boot_in_flight = true;
engine.mcp_boot_generation = Some(7);
engine.mcp_connection_errors.insert(
"previous-server".to_string(),
"previous failure".to_string(),
);
// A same-conversation re-sync keeps its usage and its boot pass.
let same = engine.session.id.clone();
assert!(engine.install_synced_session_id(same).is_none());
assert_eq!(engine.session_snapshot().total_tokens, 1_200);
assert!(engine.mcp_boot_in_flight);
(engine, handle, boot)
}
#[tokio::test]
async fn a_session_boundary_does_not_inherit_the_previous_usage() {
// C02-17: session A's tokens are not session B's.
let (mut engine, _handle, _boot) = engine_mid_mcp_boot_with_usage().await;
assert!(
engine
.install_synced_session_id("next-session".to_string())
.is_some()
);
assert_eq!(engine.session_snapshot().total_tokens, 0);
}
#[tokio::test]
async fn a_session_boundary_abandons_the_previous_mcp_boot() {
// C02-08: the dropped pool's pass is aborted and its state cleared.
let (mut engine, _handle, boot) = engine_mid_mcp_boot_with_usage().await;
assert!(
engine
.install_synced_session_id("next-session".to_string())
.is_some()
);
assert!(!engine.mcp_boot_in_flight);
assert_eq!(engine.mcp_boot_generation, None);
assert!(engine.mcp_boot_rx.is_none());
assert!(engine.mcp_connection_errors.is_empty());
let joined = tokio::time::timeout(Duration::from_secs(5), boot)
.await
.expect("the previous boot pass must stop promptly");
assert!(joined.unwrap_err().is_cancelled(), "the pass was aborted");
}
fn empty_user_input_request() -> crate::tools::user_input::UserInputRequest {
crate::tools::user_input::UserInputRequest {
questions: Vec::new(),
}
}
#[tokio::test]
async fn a_session_boundary_discards_and_stops_the_previous_mcp_supervisor() {
let workspace = tempdir().unwrap();
let (mut engine, _handle) = quiet_engine(deterministic_engine_config(workspace.path()));
let (tx, rx) = mpsc::channel(1);
tx.send(McpSupervisorUpdate {
died: vec![("previous-server".to_string(), "stale diagnosis".to_string())],
failed: Vec::new(),
recovered: Vec::new(),
parked: Vec::new(),
})
.await
.unwrap();
engine.mcp_supervisor_rx = Some(rx);
// A reconnect future retains its connection resources across await. The
// boundary must abort that future, not merely drop a Weak pool owner.
let held = Arc::new(());
let weak = Arc::downgrade(&held);
let (started_tx, started_rx) = tokio::sync::oneshot::channel();
let supervisor = tokio::spawn(async move {
let _held = held;
started_tx.send(()).unwrap();
std::future::pending::<()>().await;
});
started_rx.await.unwrap();
engine.mcp_supervisor_task = Some(supervisor.abort_handle());
assert!(
engine
.install_synced_session_id("next-session".to_string())
.is_some()
);
assert!(
engine.mcp_supervisor_rx.is_none(),
"queued diagnoses retired"
);
assert!(engine.mcp_supervisor_task.is_none());
let joined = tokio::time::timeout(Duration::from_secs(1), supervisor)
.await
.expect("the previous supervisor must stop promptly");
assert!(joined.unwrap_err().is_cancelled());
assert!(weak.upgrade().is_none(), "reconnect resources released");
assert!(tx.is_closed(), "old diagnoses cannot cross the boundary");
engine.ensure_mcp_pool().await.unwrap();
assert!(engine.mcp_supervisor_rx.is_some(), "the next pool is armed");
assert!(engine.mcp_supervisor_task.is_some());
assert!(engine.mcp_connection_errors.is_empty());
engine.drop_mcp_pool();
}
#[tokio::test]
async fn an_undeliverable_user_input_request_fails_fast() {
// C02-19: the question cannot reach a host (its event channel is closed,
// while the answer channel stays open), so nobody can ever answer it.
let workspace = tempdir().unwrap();
let (mut engine, handle) = quiet_engine(deterministic_engine_config(workspace.path()));
handle.rx_event.write().await.close();
let result = tokio::time::timeout(
Duration::from_secs(5),
engine.await_user_input("ask-undeliverable", empty_user_input_request()),
)
.await
.expect("an undeliverable question must not wait for an answer");
assert!(
matches!(result, Err(ToolError::ExecutionFailed { .. })),
"{result:?}"
);
}
#[tokio::test]
async fn an_open_user_input_question_is_not_charged_to_the_turn() {
// C02-19: a delivered question the person leaves open is human time, not
// the agent's; the per-turn wall clock does not advance across it.
let workspace = tempdir().unwrap();
let wait = Duration::from_millis(400);
let (mut engine, _handle) = quiet_engine(EngineConfig {
user_input_timeout: Some(wait),
..deterministic_engine_config(workspace.path())
});
let before = engine.turn_wall_clock.spent();
let result = engine
.await_user_input("ask-unanswered", empty_user_input_request())
.await;
assert!(
matches!(result, Err(ToolError::Timeout { .. })),
"{result:?}"
);
let charged = engine.turn_wall_clock.spent().saturating_sub(before);
assert!(
charged < wait / 2,
"the human wait was charged to the turn budget: {charged:?}"
);
}
#[tokio::test]
#[allow(clippy::await_holding_lock)]
async fn edit_last_turn_restores_the_exchange_when_the_replacement_never_starts() {
// C02-02: `/edit` cuts the last exchange, then dispatches the new text.
// A dispatch that never starts must hand the cut exchange back.
let _lock = lock_test_env();
let tmp = tempdir().unwrap();
let api_config = Config::default().with_legacy_root(
Some("test-key".to_string()),
Some("http://127.0.0.1:9".to_string()),
);
let (mut engine, handle) = Engine::new_with_model_client(
EngineConfig {
workspace: tmp.path().to_path_buf(),
model: "deepseek-v4-pro".to_string(),
snapshots_enabled: false,
subagents_enabled: false,
..Default::default()
},
&api_config,
Arc::new(MockLlmClient::new(Vec::new())),
);
// No model client: the replacement send returns NotStarted.
engine.model_client = None;
let run = tokio::spawn(engine.run());
let text = |role: Role, text: &str| Message {
role,
content: vec![ContentBlock::Text {
text: text.to_string(),
cache_control: None,
}],
};
handle
.send(Op::SyncSession {
session_id: Some("edit-restore".to_string()),
messages: vec![
text(Role::User, "original prompt"),
text(Role::Assistant, "original answer"),
],
system_prompt: None,
system_prompt_override: false,
model: "deepseek-v4-pro".to_string(),
workspace: tmp.path().to_path_buf(),
mode: AppMode::Agent,
})
.await
.unwrap();
let before = session_snapshot(&handle).await;
assert_eq!(before.messages.len(), 2);
handle
.send(Op::EditLastTurn {
new_message: "edited prompt".to_string(),
})
.await
.unwrap();
let after = session_snapshot(&handle).await;
assert_eq!(
after.messages, before.messages,
"a replacement that never started must not cost the original exchange"
);
handle.send(Op::Shutdown).await.unwrap();
run.await.unwrap();
}
#[tokio::test]
async fn malformed_mcp_config_is_reported_instead_of_silently_empty() {
// C02-18: an unreadable config still yields an empty, reloadable pool,
// but the person is told the configured servers are gone.
let workspace = tempdir().unwrap();
let config_path = workspace.path().join("mcp.json");
fs::write(&config_path, "{ this is not json").unwrap();
let (mut engine, handle) = quiet_engine(deterministic_engine_config(workspace.path()));
engine.session.mcp_config_path = config_path;
engine
.ensure_mcp_pool()
.await
.expect("a reloadable empty pool still exists");
let mut rx = handle.rx_event.write().await;
let reported = std::iter::from_fn(|| rx.try_recv().ok()).any(|event| {
matches!(
event,
Event::Status { ref message } if message.contains("MCP config could not be loaded")
)
});
assert!(reported, "a malformed MCP config must not fail silently");
}
+288 -80
View File
@@ -112,13 +112,39 @@ async fn preserve_tool_output_before_fanout(
tool_id: &str,
tool_name: &str,
) -> Result<RichToolResult, ToolError> {
let mut rich = result?;
let model = model.to_owned();
let session_id = session_id.to_owned();
let tool_id = tool_id.to_owned();
let tool_name = tool_name.to_owned();
let mut rich = match result {
Ok(rich) => rich,
// C02-12: an error is fanned out to the event stream and the session
// exactly like a result, so an oversized one gets the same bounded
// head/tail projection and saved artifact. Ordinary short errors
// stay byte-identical (the projection only engages past the
// spillover threshold).
Err(mut error) => {
if tool_error_message_mut(&mut error).is_none_or(|message| {
message.len() <= crate::tools::truncate::SPILLOVER_THRESHOLD_BYTES
}) {
return Err(error);
}
return tokio::task::spawn_blocking(move || {
bound_oversized_tool_error(error, &tool_id, &tool_name, &session_id)
})
.await
.map_err(|join_error| {
ToolError::execution_failed(format!(
"Tool output preservation failed: {join_error}"
))
})
.and_then(Err);
}
};
tokio::task::spawn_blocking(move || {
if let Some(path) = crate::tools::truncate::apply_spillover_with_artifact(
// Failed results are bounded too: the fan-out cost of a huge one is
// the same whether or not the tool called it a failure (C02-12).
if let Some(path) = crate::tools::truncate::apply_spillover_with_artifact_including_errors(
&mut rich.result,
&tool_id,
&tool_name,
@@ -155,6 +181,42 @@ async fn preserve_tool_output_before_fanout(
})
}
/// The free-form text of a tool error, when its variant carries one.
fn tool_error_message_mut(error: &mut ToolError) -> Option<&mut String> {
match error {
ToolError::InvalidInput { message }
| ToolError::ExecutionFailed { message, .. }
| ToolError::Cancelled { message }
| ToolError::NotAvailable { message }
| ToolError::PermissionDenied { message } => Some(message),
_ => None,
}
}
/// Give an oversized tool error message the bounded head/tail projection and
/// saved artifact a failed result gets (C02-12). Blocking: it may write the
/// artifact, so callers run it under `spawn_blocking`. A failed artifact
/// write degrades to the original message, as spillover does for results.
fn bound_oversized_tool_error(
mut error: ToolError,
tool_id: &str,
tool_name: &str,
session_id: &str,
) -> ToolError {
let Some(message) = tool_error_message_mut(&mut error) else {
return error;
};
let mut projected = ToolResult::error(std::mem::take(message));
crate::tools::truncate::apply_spillover_with_artifact_including_errors(
&mut projected,
tool_id,
tool_name,
session_id,
);
*message = projected.content;
error
}
fn approval_intent_summary(text: &str) -> Option<String> {
let trimmed = text.trim();
if trimmed.is_empty() {
@@ -405,6 +467,28 @@ fn incomplete_tool_result(reason: &str) -> ToolResult {
}
}
/// Status receipt carried by the one request a reasoning-only / empty-stop
/// nudge rides (C02-04). The nudge itself never joins the session.
pub(super) const REQUEST_NUDGE_RECEIPT_PREFIX: &str =
"Continuing — this retry carries a request-scoped nudge (not saved to the conversation): ";
/// The not-executed result for a call collected from a response whose stream
/// failed before it completed (C02-05). Same shape as
/// [`incomplete_tool_result`]: nothing started, so nothing needs undoing.
fn stream_failed_tool_result(error: &str) -> ToolResult {
ToolResult {
content: format!(
"Not executed: the provider stream failed before the model response completed ({error})."
),
success: false,
metadata: Some(json!({
"side_effect_status": "not_started",
"error_category": "model_stream_failed",
"model_output_incomplete": true,
})),
}
}
fn registered_tool_requires_non_bypassable_approval(tool_name: &str) -> bool {
// `rlm_eval` (and the unified `rlm` tool whose eval action inherits the
// same Required approval) must never bypass explicit approval (#3866).
@@ -414,9 +498,11 @@ fn registered_tool_requires_non_bypassable_approval(tool_name: &str) -> bool {
/// Replace the runtime-MCP slice of the tool catalog wholesale. An additive
/// merge could never remove anything: the synthetic `mcp_<server>_
/// authenticate` entry would survive its own successful login, and tools
/// killed by a live 401 would stay callable in name. `universe` is every
/// name the pool can own; entries inside it are the pool's to manage, and
/// the refreshed list is the new truth.
/// killed by a live 401 would stay callable in name. The pool owns `universe`
/// (every name it can list now) and every MCP name already in the catalog —
/// the second half is what lets a tool whose server vanished or lost its
/// authorization leave (C02-07); `universe` alone only names survivors. The
/// refreshed list is the new truth for all of them.
///
/// The refreshed slice is shaped exactly like the turn's initial catalog —
/// the same deferral pass, the same surface budget, the same always-load
@@ -425,6 +511,10 @@ fn registered_tool_requires_non_bypassable_approval(tool_name: &str) -> bool {
/// projection carries `defer_loading = false` on every tool, so pushing it
/// in unshaped put every MCP tool definition into every remaining request
/// of the turn (#5939).
///
/// Returns whether the catalog or its active set changed in any way — a
/// schema or description edit included, not only a count change (C02-16) —
/// so the caller can declare the tool-surface change to the prefix check.
pub(super) fn replace_runtime_mcp_tools(
tool_catalog: &mut Vec<Tool>,
active_tool_names: &mut std::collections::HashSet<String>,
@@ -433,11 +523,12 @@ pub(super) fn replace_runtime_mcp_tools(
mode: AppMode,
always_load: &std::collections::HashSet<String>,
surface_budget: crate::model_profile::ToolSurfaceBudget,
) -> usize {
let before = tool_catalog.len();
) -> bool {
let catalog_before = tool_catalog.clone();
let active_before = active_tool_names.clone();
let mut previously_active = std::collections::HashSet::new();
tool_catalog.retain(|tool| {
let owned = universe.contains(&tool.name);
let owned = universe.contains(&tool.name) || McpPool::is_mcp_tool(&tool.name);
if owned && active_tool_names.remove(&tool.name) {
previously_active.insert(tool.name.clone());
}
@@ -455,7 +546,7 @@ pub(super) fn replace_runtime_mcp_tools(
}
tool_catalog.push(tool);
}
tool_catalog.len().abs_diff(before)
*tool_catalog != catalog_before || *active_tool_names != active_before
}
impl Engine {
@@ -600,6 +691,18 @@ impl Engine {
None
}
/// R1: the turn-ending error once the per-turn wall-clock budget is spent.
/// Checked wherever the loop is about to authorize a provider request.
fn turn_wall_clock_exhausted_error(&self) -> Option<String> {
self.turn_wall_clock.exhausted().then(|| {
format!(
"Per-turn wall-clock budget exhausted after {}s (limit: {}s). The turn was stopped before another model request; work already done is in the transcript. Send another message to continue, or raise `[tui].turn_wall_clock_secs`.",
self.turn_wall_clock.spent().as_secs(),
self.turn_wall_clock.budget().as_secs(),
)
})
}
/// A connection completed during inference must be discoverable in this
/// turn, without widening its command policy or making every MCP tool eager.
pub(super) async fn refresh_boot_mcp_catalog(
@@ -615,23 +718,17 @@ impl Engine {
let Some(pool) = self.mcp_pool.as_ref() else {
return;
};
let (mut universe, mut refreshed) = {
let (universe, mut refreshed) = {
let pool = pool.lock().await;
let refreshed = pool.to_api_tools();
(pool.model_tool_names(&refreshed), refreshed)
};
// A config/authority change during handshake can remove a server;
// its previous names must also leave this turn's catalog.
universe.extend(
catalog
.iter()
.filter(|tool| McpPool::is_mcp_tool(&tool.name))
.map(|tool| tool.name.clone()),
);
// `replace_runtime_mcp_tools` owns every MCP name already in the
// catalog, so its previous names leave this turn's catalog too.
refreshed
.retain(|tool| policy.passes_allow_list(&tool.name) && !policy.denies_tool(&tool.name));
let before = catalog.clone();
replace_runtime_mcp_tools(
if replace_runtime_mcp_tools(
catalog,
active,
&universe,
@@ -640,8 +737,7 @@ impl Engine {
&self.config.tools_always_load,
self.turn_tool_surface_budget
.unwrap_or(crate::model_profile::ToolSurfaceBudget::Standard),
);
if *catalog != before {
) {
self.session.pending_prefix_change_reason = Some("mcp-session-boot".to_string());
}
}
@@ -997,8 +1093,11 @@ impl Engine {
// retry after that carries a nudge — attached to one outbound request
// and dropped, never added to the session. Writing it to the session
// would put a message the user never sent into the transcript, the
// exports, and every later turn's context.
let mut reasoning_only_nudge: Option<Message> = None;
// exports, and every later turn's context. It is still model-visible,
// so the request that carries it also emits a durable internal
// status receipt with its exact text (C02-04, "model-visible means
// logged"): the runtime event log can reconstruct the request.
let mut reasoning_only_nudge: Option<String> = None;
// Outer stream-retry budget: when the chunked-transfer connection
// dies mid-stream and either nothing useful was streamed (#103
// Phase 3), the host slept mid-turn (#2990), or a host hit a
@@ -1034,12 +1133,7 @@ impl Engine {
// result already produced stays in the transcript. Hitting it is
// never a clean success — the turn ends `Failed` with the limit
// named, matching how the step ceiling below reports.
if self.turn_wall_clock.exhausted() {
let error = format!(
"Per-turn wall-clock budget exhausted after {}s (limit: {}s). The turn was stopped before another model request; work already done is in the transcript. Send another message to continue, or raise `[tui].turn_wall_clock_secs`.",
self.turn_wall_clock.spent().as_secs(),
self.turn_wall_clock.budget().as_secs(),
);
if let Some(error) = self.turn_wall_clock_exhausted_error() {
let _ = self.tx_event.send(Event::status(error.clone())).await;
return (TurnOutcomeStatus::Failed, Some(error));
}
@@ -1418,6 +1512,13 @@ impl Engine {
}
}
self.finish_compaction(&compaction_id);
// C02-06: a compaction pass has its own bound, not the
// turn's. Recheck the wall clock before it can authorize the
// provider request below.
if let Some(error) = self.turn_wall_clock_exhausted_error() {
let _ = self.tx_event.send(Event::status(error.clone())).await;
return (TurnOutcomeStatus::Failed, Some(error));
}
}
// The guard measures what the compaction gate measures: the honest
@@ -1728,14 +1829,18 @@ impl Engine {
let fleet_report_response = fleet_denial_guard
.as_ref()
.is_some_and(FleetDenialGuard::report_only);
// `take` is what keeps this request-scoped: the nudge is spent
// here and never reaches `self.session.messages`.
let request_nudge = reasoning_only_nudge.take();
let mut request = prepare_primary_turn_request(PrimaryTurnRequest {
model: self.session.model.clone(),
messages: {
let mut messages = self.messages_with_turn_metadata();
// `take` is what keeps this request-scoped: the nudge is
// spent here and never reaches `self.session.messages`.
if let Some(nudge) = reasoning_only_nudge.take() {
messages.push(nudge);
if let Some(nudge) = request_nudge.as_ref() {
messages.push(self.runtime_text_message_with_turn_metadata(
nudge.clone(),
UserInputProvenance::Runtime,
));
}
messages
},
@@ -1748,10 +1853,11 @@ impl Engine {
system: self.session.system_prompt.clone(),
tools: active_tools.clone(),
tool_choice: if active_tools.is_some() {
if fleet_report_response {
if fleet_report_response || turn.budget_exhausted_final_report {
// Keep the pinned tool prefix; only this request's
// choice changes. Admission below also enforces this
// if a provider ignores the report-only request.
// if a provider ignores the report-only request
// (C02-10: the step-budget final report included).
Some(json!("none"))
} else if strict_tool_mode {
Some(json!("required"))
@@ -1820,6 +1926,16 @@ impl Engine {
snapshot: tool_request_snapshot,
})
.await;
if let Some(nudge) = request_nudge {
// The "Continuing — " prefix classifies the receipt as
// internal: durable clients keep it, collapsed.
let _ = self
.tx_event
.send(Event::status(format!(
"{REQUEST_NUDGE_RECEIPT_PREFIX}{nudge}"
)))
.await;
}
if let Some(mut route) = turn.pending_route.take() {
if let Some(billing) = route.billing.as_mut() {
// Freeze the exact provider-live row at CodeWhale's
@@ -2015,6 +2131,11 @@ impl Engine {
None,
Some(super::turn_heartbeat::PREPARING_PHASE_BOUND),
);
// C02-05: a response whose stream failed — a provider error frame,
// a transport error, a stall, or a cap — is not a complete
// response. Unless the retry below re-issues the request, nothing
// it collected may execute or continue the turn.
let response_stream_failed = stream_error.is_some();
turn_error = turn_error.or(stream_error);
turn.stop_diagnostics
.observe_provider_response(stop_reason.as_deref(), tool_uses.len());
@@ -2115,26 +2236,8 @@ impl Engine {
output_limit_truncated = Some(reason.to_string());
// Fall through to the normal content/tool dispatch below.
} else {
for tool in &tool_uses {
let _ = self
.tx_event
.send(Event::ToolCallStarted {
id: tool.execution_id.clone(),
model_call: Some(tool.model_call()),
name: tool.name.clone(),
input: final_tool_input(tool),
})
.await;
let _ = self
.tx_event
.send(Event::ToolCallComplete {
id: tool.execution_id.clone(),
model_call: Some(tool.model_call()),
name: tool.name.clone(),
result: Ok(incomplete_tool_result(reason)),
})
.await;
}
self.settle_unadmitted_tool_calls(&tool_uses, &incomplete_tool_result(reason))
.await;
// Do not emit MessageComplete: hosts must retain the visible
// fragment as interrupted/failed rather than recording it as
// a completed assistant item.
@@ -2334,6 +2437,28 @@ impl Engine {
}
}
// C02-05: the one admission authority for a failed stream. Calls
// collected before the failure are announced with an explicit
// not-executed result and never reach planning, approval or a
// handler; the visible text is kept as an interrupted fragment
// and no tool_use enters history, so the transcript stays paired.
// A retry never gets here: it `continue`d above with this batch
// dropped, and the re-issued request streams its own calls.
if response_stream_failed && !tool_uses.is_empty() {
let error = turn_error
.clone()
.unwrap_or_else(|| "provider stream failed".to_string());
self.settle_unadmitted_tool_calls(
&tool_uses,
&stream_failed_tool_result(&summarize_text(&error, 200)),
)
.await;
self.add_interrupted_assistant_text(&current_text_visible)
.await;
turn.stop_diagnostics.last_response_tool_calls_suppressed = Some(tool_uses.len());
return (TurnOutcomeStatus::Failed, Some(error));
}
for tool in &tool_uses {
let _ = self
.tx_event
@@ -2484,6 +2609,15 @@ impl Engine {
.await;
}
// C02-05: a failed stream that carried no tool call keeps its text
// (above) and ends the turn with the stream's error. It never
// runs a ```repl fence from the failed text, and never authorizes
// another request for an output-limit continuation, a steer, a
// sub-agent completion or a goal continuation.
if response_stream_failed {
break;
}
// A truncated response with no tool call cannot continue through
// tool execution: surface the truncation as a bounded observation
// and resume the loop so the model can act on it instead of the
@@ -3031,11 +3165,7 @@ impl Engine {
crate::config::DEFAULT_REASONING_ONLY_REPROMPT_MESSAGE.to_string()
});
if !text.trim().is_empty() {
reasoning_only_nudge =
Some(self.runtime_text_message_with_turn_metadata(
text,
UserInputProvenance::Runtime,
));
reasoning_only_nudge = Some(text);
}
}
let how = if nudged {
@@ -3093,11 +3223,7 @@ impl Engine {
.to_string()
});
if !text.trim().is_empty() {
reasoning_only_nudge =
Some(self.runtime_text_message_with_turn_metadata(
text,
UserInputProvenance::Runtime,
));
reasoning_only_nudge = Some(text);
}
"re-requesting the answer with a nudge"
}
@@ -3555,6 +3681,18 @@ impl Engine {
blocked_error = guard.admission_error(&tool_name, &tool_input);
}
// C02-10: the response granted after the step budget ran out is
// report-only. A provider that ignores `tool_choice: none` still
// gets no execution; the next loop pass ends the turn at the
// exhausted budget.
if blocked_error.is_none() && turn.budget_exhausted_final_report {
blocked_error = Some(ToolError::permission_denied(format!(
"Model-step budget exhausted (limit: {}, {}): this is the final report response, so no tool may execute. Report what you did, what you found, what remains, and the evidence.",
turn.max_steps,
turn.budget_source.key_label(),
)));
}
if blocked_error.is_none()
&& let Some(error) = tool.input_parse_error.clone()
{
@@ -5533,7 +5671,7 @@ impl Engine {
self.current_mode,
&self.config.tools_always_load,
surface_budget,
) > 0;
);
}
// Any of the legitimate mid-turn tool-surface changes above
// re-pin the header under a declared `change:tool_surface`
@@ -5840,18 +5978,6 @@ impl Engine {
break;
}
// Guard: max accumulated content bytes
if stream_content_bytes > max_content_bytes {
let envelope = StreamError::Overflow {
limit_bytes: max_content_bytes,
}
.into_envelope();
crate::logging::warn(&envelope.message);
stream_error.get_or_insert(envelope.message.clone());
let _ = self.tx_event.send(Event::error(envelope)).await;
break;
}
let event = match event_result {
Ok(e) => {
self.turn_heartbeat.stream_progress(
@@ -5895,6 +6021,15 @@ impl Engine {
last_progress_mono.elapsed(),
wall_elapsed,
));
// Like the network-drop resumes below, keep the real
// error as the prospective outcome: the retry clears
// it, and an exhausted resume budget then fails the
// turn with it instead of admitting the partial
// response as if it had completed.
stream_error.get_or_insert(stream_read_error_user_message(
&message,
any_content_received,
));
pending_resume = Some(StreamResume::AfterSleep);
break;
}
@@ -6040,6 +6175,24 @@ impl Engine {
}
};
// Guard: max accumulated content bytes (C02-13). Counted per
// event and checked before the event is applied, so the delta
// that crosses the cap is never forwarded or accumulated — even
// when it is the stream's last — and tool-argument JSON counts
// like text and reasoning.
stream_content_bytes =
stream_content_bytes.saturating_add(stream_event_content_bytes(&event));
if stream_content_bytes > max_content_bytes {
let envelope = StreamError::Overflow {
limit_bytes: max_content_bytes,
}
.into_envelope();
crate::logging::warn(&envelope.message);
stream_error.get_or_insert(envelope.message.clone());
let _ = self.tx_event.send(Event::error(envelope)).await;
break;
}
match event {
StreamEvent::ToolProjectionWarning {
provider,
@@ -6150,7 +6303,6 @@ impl Engine {
},
StreamEvent::ContentBlockDelta { index, delta } => match delta {
Delta::TextDelta { text } => {
stream_content_bytes = stream_content_bytes.saturating_add(text.len());
current_text_raw.push_str(&text);
let filtered =
filter_tool_call_delta_with_state(&text, &mut tool_call_filter);
@@ -6173,7 +6325,6 @@ impl Engine {
}
}
Delta::ThinkingDelta { thinking } => {
stream_content_bytes = stream_content_bytes.saturating_add(thinking.len());
current_thinking.push_str(&thinking);
if !thinking.is_empty() {
let _ = self
@@ -6338,6 +6489,32 @@ impl Engine {
}
}
/// Announce every call of a response that will not be admitted, each
/// paired with its not-executed `result`, so a host never shows a started
/// call without a completion. Nothing here plans, approves or executes.
async fn settle_unadmitted_tool_calls(&self, tool_uses: &[ToolUseState], result: &ToolResult) {
for tool in tool_uses {
let _ = self
.tx_event
.send(Event::ToolCallStarted {
id: tool.execution_id.clone(),
model_call: Some(tool.model_call()),
name: tool.name.clone(),
input: final_tool_input(tool),
})
.await;
let _ = self
.tx_event
.send(Event::ToolCallComplete {
id: tool.execution_id.clone(),
model_call: Some(tool.model_call()),
name: tool.name.clone(),
result: Ok(result.clone()),
})
.await;
}
}
/// Finalize one streamed tool call's input from its accumulated buffer.
///
/// The parse that lands here must be structurally intact: a value that
@@ -7448,6 +7625,37 @@ fn stream_event_has_actionable_content(event: &StreamEvent) -> bool {
}
}
/// Bytes an event adds to the response the engine accumulates: text,
/// reasoning, tool calls (id, name and argument JSON, whether the arguments
/// arrive whole in the block start or as `InputJsonDelta`s) and replay
/// signatures. Opaque reasoning state is provider-owned and not counted.
fn stream_event_content_bytes(event: &StreamEvent) -> usize {
fn initial_input_bytes(input: &Value) -> usize {
let empty = input.is_null() || input.as_object().is_some_and(serde_json::Map::is_empty);
if empty { 0 } else { input.to_string().len() }
}
match event {
StreamEvent::ContentBlockStart { content_block, .. } => match content_block {
ContentBlockStart::Text { text } => text.len(),
ContentBlockStart::Thinking { thinking } => thinking.len(),
ContentBlockStart::ToolUse {
id, name, input, ..
}
| ContentBlockStart::ServerToolUse { id, name, input } => {
id.len() + name.len() + initial_input_bytes(input)
}
},
StreamEvent::ContentBlockDelta { delta, .. } => match delta {
Delta::TextDelta { text } => text.len(),
Delta::ThinkingDelta { thinking } => thinking.len(),
Delta::InputJsonDelta { partial_json } => partial_json.len(),
Delta::SignatureDelta { signature } => signature.len(),
Delta::ReasoningStateDelta { .. } => 0,
},
_ => 0,
}
}
/// Sentinel reasoning-effort value meaning "let the auto-reasoning system
/// decide" (#4158).
pub(super) const REASONING_EFFORT_AUTO: &str = "auto";
+410 -85
View File
@@ -1104,11 +1104,30 @@ fn resolve_exec_resume_session_id(args: &ExecArgs, workspace: &Path) -> Result<O
}
fn load_exec_resume_session(session_id: &str) -> Result<session_manager::SavedSession> {
SessionManager::default_location()
match SessionManager::default_location()
.context("could not open session manager for resume")?
.resume_session_by_prefix(session_id)
.map(|recovery| recovery.session)
.with_context(|| exec_resume_load_error(session_id))
.attach_session_by_prefix(session_id)
{
Ok((recovery, lease)) => {
// This exec run owns the session until it exits.
lease.commit();
Ok(recovery.session)
}
// Resuming a session a TUI has open would give its document two
// writers; the TUI's next autosave would drop this run's turns.
Err(error) if error.kind() == io::ErrorKind::ResourceBusy => {
bail!(exec_resume_busy_error(session_id))
}
Err(error) => Err(error).with_context(|| exec_resume_load_error(session_id)),
}
}
fn exec_resume_busy_error(session_id: &str) -> String {
format!(
"session {} is open in another Codewhale window. Continue it there, or run \
`codewhale fork <SESSION_ID>` and resume the copy.",
exec_stream_session_ref(session_id)
)
}
/// The typed `--resume` value stays redacted in every output mode: exec runs
@@ -1312,7 +1331,7 @@ struct ScorecardArgs {
#[arg(long, value_name = "FILE")]
baseline: Option<PathBuf>,
/// Regression threshold, in percent increase over the baseline.
#[arg(long, default_value_t = 5.0)]
#[arg(long, default_value_t = 5.0, value_parser = parse_regression_threshold)]
threshold: f64,
/// Emit machine-readable JSON instead of the human summary.
#[arg(long, default_value_t = false)]
@@ -1996,7 +2015,7 @@ fn run_with_args(args: Vec<String>) -> Result<()> {
plugin_registry = Some(discovery.registry_for_workspace(&workspace));
plugin_discovery = Some(discovery);
},
warn_on_workspace_dotenv_result,
|| warn_on_workspace_dotenv_result(&workspace),
);
let plugin_discovery = plugin_discovery
.expect("plugin discovery initialization must precede workspace dotenv loading");
@@ -2962,8 +2981,8 @@ struct WorkspaceDotenvReport {
/// MCP servers, plugin trust, executable lookup, sandbox/approval posture, or
/// network destinations. Shell-exported values and config/CLI arguments remain
/// the explicit surfaces for those controls.
fn warn_on_workspace_dotenv_result() {
match load_workspace_dotenv_credentials() {
fn warn_on_workspace_dotenv_result(workspace: &Path) {
match load_workspace_dotenv_credentials(workspace) {
Ok(Some(report)) if !report.ignored.is_empty() => {
eprintln!(
"Codewhale ignored non-credential settings in {}: {}. Use config.toml, CLI flags, or the launching shell for control settings.",
@@ -3003,21 +3022,31 @@ fn display_env_key_set(keys: &BTreeSet<String>) -> String {
labels.join(", ")
}
fn load_workspace_dotenv_credentials() -> Result<Option<WorkspaceDotenvReport>> {
let Some(path) = find_workspace_dotenv()? else {
fn load_workspace_dotenv_credentials(workspace: &Path) -> Result<Option<WorkspaceDotenvReport>> {
let Some(path) = find_workspace_dotenv(workspace)? else {
return Ok(None);
};
load_workspace_dotenv_credentials_from_path(&path).map(Some)
}
fn find_workspace_dotenv() -> Result<Option<PathBuf>> {
let cwd = std::env::current_dir().context("could not resolve the current workspace")?;
let boundary = cwd
/// The nearest `.env` from the resolved launch workspace (`--workspace`, else
/// the current directory) up to its repository root. Searching from the
/// process directory instead would load another tree's credentials when the
/// two differ.
fn find_workspace_dotenv(workspace: &Path) -> Result<Option<PathBuf>> {
let start = if workspace.is_absolute() {
workspace.to_path_buf()
} else {
std::env::current_dir()
.context("could not resolve the current workspace")?
.join(workspace)
};
let boundary = start
.ancestors()
.find(|ancestor| std::fs::symlink_metadata(ancestor.join(".git")).is_ok())
.unwrap_or(cwd.as_path());
.unwrap_or(start.as_path());
for ancestor in cwd.ancestors() {
for ancestor in start.ancestors() {
let candidate = ancestor.join(".env");
match std::fs::symlink_metadata(&candidate) {
Ok(_) => return Ok(Some(candidate)),
@@ -3346,6 +3375,20 @@ fn run_eval(args: EvalArgs) -> Result<()> {
}
}
/// A regression gate threshold must be a finite percentage: `NaN` compares
/// false against every change (a gate that always passes) and infinity
/// disables the gate outright.
fn parse_regression_threshold(raw: &str) -> Result<f64, String> {
let value: f64 = raw
.trim()
.parse()
.map_err(|_| format!("`{raw}` is not a number"))?;
if !value.is_finite() {
return Err(format!("`{raw}` is not a finite percentage"));
}
Ok(value)
}
/// Score a run's token/cache/cost from recorded turns and (optionally) flag
/// regressions against a committed baseline. Offline: reads recorded usage from
/// a JSON file, reuses the pricing layer, never calls a model. Exits non-zero
@@ -11192,10 +11235,26 @@ fn run_sandbox_command(args: SandboxArgs) -> Result<()> {
.stdout(Stdio::piped())
.stderr(Stdio::piped());
child_env::apply_to_command(&mut cmd, child_env::string_map_env(&exec_env.env));
// Lead a process group so the timeout ends everything the command started.
#[cfg(unix)]
{
use std::os::unix::process::CommandExt as _;
cmd.process_group(0);
}
let mut child = cmd
.spawn()
.map_err(|e| anyhow::anyhow!("Failed to run command: {e}"))?;
// The sandbox run is the tree's lifetime: dropping `tree` on return ends
// anything the command left running, as `contained_output` does.
let tree = match crate::process_tree::ProcessTree::attach(&child) {
Ok(tree) => tree,
Err(error) => {
let _ = child.kill();
let _ = child.wait();
bail!("Failed to contain the sandboxed command: {error}");
}
};
let stdout_handle = child
.stdout
.take()
@@ -11205,48 +11264,78 @@ fn run_sandbox_command(args: SandboxArgs) -> Result<()> {
.take()
.ok_or_else(|| anyhow::anyhow!("stderr unavailable"))?;
let timeout = exec_env.timeout;
let stdout_thread = std::thread::spawn(move || {
// Output streams straight through instead of being buffered whole: a
// command's output size is unbounded, so only a bounded stderr tail is
// kept for sandbox-denial detection.
const STDERR_TAIL_BYTES: usize = 64 * 1024;
let (done_tx, done_rx) = std::sync::mpsc::channel::<()>();
let stdout_done = done_tx.clone();
std::thread::spawn(move || {
let mut reader = stdout_handle;
let mut buf = Vec::new();
let _ = reader.read_to_end(&mut buf);
buf
let _ = io::copy(&mut reader, &mut io::stdout());
let _ = stdout_done.send(());
});
let stderr_thread = std::thread::spawn(move || {
let stderr_tail = Arc::new(std::sync::Mutex::new(Vec::<u8>::new()));
let tail = Arc::clone(&stderr_tail);
std::thread::spawn(move || {
let mut reader = stderr_handle;
let mut buf = Vec::new();
let _ = reader.read_to_end(&mut buf);
buf
let mut chunk = [0_u8; 8192];
loop {
let read = match reader.read(&mut chunk) {
Ok(0) => break,
Ok(read) => read,
Err(error) if error.kind() == io::ErrorKind::Interrupted => continue,
Err(_) => break,
};
let _ = io::stderr().write_all(&chunk[..read]);
if let Ok(mut tail) = tail.lock() {
tail.extend_from_slice(&chunk[..read]);
let excess = tail.len().saturating_sub(STDERR_TAIL_BYTES);
tail.drain(..excess);
}
}
let _ = done_tx.send(());
});
if let Some(status) = child.wait_timeout(timeout)? {
let stdout = stdout_thread.join().unwrap_or_default();
let stderr = stderr_thread.join().unwrap_or_default();
let stderr_str = String::from_utf8_lossy(&stderr);
let exit_code = status.code().unwrap_or(-1);
let sandbox_type = exec_env.sandbox_type;
let sandbox_denied = SandboxManager::was_denied(sandbox_type, exit_code, &stderr_str);
if !stdout.is_empty() {
print!("{}", String::from_utf8_lossy(&stdout));
}
if !stderr.is_empty() {
eprint!("{stderr_str}");
}
if sandbox_denied {
eprintln!(
"{}",
SandboxManager::denial_message(sandbox_type, &stderr_str)
);
}
if !status.success() {
bail!("Command failed with exit code {exit_code}");
}
} else {
let timeout = exec_env.timeout;
let deadline = Instant::now() + timeout;
let Some(status) = child.wait_timeout(timeout)? else {
let _ = tree.kill();
let _ = child.kill();
let _ = child.wait();
bail!("Command timed out after {}ms", timeout.as_millis());
};
// A descendant may still hold the output pipes. Let it finish inside the
// same budget, then end the tree so the drain always reaches EOF.
let mut drained = 0;
while drained < 2 {
match done_rx.recv_timeout(deadline.saturating_duration_since(Instant::now())) {
Ok(()) => drained += 1,
Err(_) => break,
}
}
if drained < 2 {
let _ = tree.kill();
while drained < 2 && done_rx.recv_timeout(Duration::from_secs(1)).is_ok() {
drained += 1;
}
}
let stderr = stderr_tail
.lock()
.map(|tail| tail.clone())
.unwrap_or_default();
let stderr_str = String::from_utf8_lossy(&stderr);
let exit_code = status.code().unwrap_or(-1);
let sandbox_type = exec_env.sandbox_type;
if SandboxManager::was_denied(sandbox_type, exit_code, &stderr_str) {
eprintln!(
"{}",
SandboxManager::denial_message(sandbox_type, &stderr_str)
);
}
if !status.success() {
bail!("Command failed with exit code {exit_code}");
}
Ok(())
}
@@ -11400,6 +11489,20 @@ fn load_recent_checkpoints(manager: &session_manager::SessionManager) -> Vec<Rec
let refs = manager.list_checkpoints().unwrap_or_default();
let mut recent = Vec::new();
for checkpoint_ref in refs {
// A session open in another terminal refreshes its own checkpoint
// mid-turn. It is not interrupted: promoting or clearing it would
// take that session's only crash-recovery record while it runs. The
// legacy slot is checked by the session it names, before anything
// prunes, promotes or migrates it over that session's live state.
let owner = match &checkpoint_ref.source {
session_manager::CheckpointSource::Session(id) => Some(id.clone()),
session_manager::CheckpointSource::Legacy => {
manager.legacy_checkpoint_origin().ok().flatten()
}
};
if owner.is_some_and(|id| manager.is_session_live_anywhere(&id)) {
continue;
}
let Ok(age) = std::time::SystemTime::now().duration_since(checkpoint_ref.modified) else {
continue;
};
@@ -11507,6 +11610,15 @@ fn recover_interrupted_checkpoint_for_resume(launch_workspace: &Path) -> Option<
let session_id = best.session.metadata.id.clone();
// Take the session's live lease before promoting or clearing anything:
// the liveness filter above is a check, and another terminal can attach
// between it and these writes. The TUI's own attach then finds the lease
// already held by this process. Losing the race leaves every file alone.
match manager.reserve_session_for_attach(&session_id) {
Ok(lease) => lease.commit(),
Err(_) => return None,
}
// Persist the checkpoint as a regular session so the TUI can load it by
// id — unless a newer regular session file for the same id already
// exists (e.g. `--continue` ran before and the session advanced since).
@@ -11601,7 +11713,19 @@ fn preserve_interrupted_checkpoint_for_explicit_resume(launch_workspace: &Path)
/// else falls back to the global value.
#[cfg(test)]
fn merge_project_config(config: &mut Config, workspace: &Path) {
merge_project_config_with_approval_baseline(config, workspace, None);
merge_project_config_with_approval_baseline(config, workspace, None)
.expect("project config applies");
}
/// A project config that exists but cannot be applied is an error, not an
/// absent file: it may be the thing tightening approval, sandbox or shell for
/// this workspace, and launching on the looser user baseline without it would
/// fail open. The reason never quotes file contents.
fn project_config_unusable(path: &Path, reason: &str) -> anyhow::Error {
anyhow!(
"Project config {} could not be applied ({reason}), so its approval, sandbox and shell restrictions are not in effect. Fix the file, or launch with --no-project-config to ignore it.",
path.display()
)
}
/// Apply project config while evaluating approval tightening against the
@@ -11612,7 +11736,7 @@ fn merge_project_config_with_approval_baseline(
config: &mut Config,
workspace: &Path,
saved_permission_posture: Option<&str>,
) {
) -> Result<()> {
// When the workspace is the user's home directory, the project-scope
// config file is also the global config file. Skip the merge to avoid
// redundant processing and a misleading "project-scope config key
@@ -11624,46 +11748,41 @@ fn merge_project_config_with_approval_baseline(
)
&& w == h
{
return;
return Ok(());
}
// v0.8.44: prefer .codewhale/config.toml, fall back to .deepseek/
let path = workspace
let primary = workspace
.join(codewhale_config::CODEWHALE_APP_DIR)
.join("config.toml");
let raw = match read_project_config_file(&path) {
Ok(Some(r)) => r,
let (path, raw) = match read_project_config_file(&primary) {
Ok(Some(raw)) => (primary, raw),
Ok(None) => {
let legacy = workspace
.join(codewhale_config::LEGACY_APP_DIR)
.join("config.toml");
match read_project_config_file(&legacy) {
Ok(Some(r)) => r,
Ok(None) => return,
Err(err) => {
eprintln!(
"warning: failed to read project-scope config {}: {err}",
legacy.display()
);
return;
}
Ok(Some(raw)) => (legacy, raw),
Ok(None) => return Ok(()),
Err(err) => return Err(project_config_unusable(&legacy, &err.to_string())),
}
}
Err(err) => {
eprintln!(
"warning: failed to read project-scope config {}: {err}",
path.display()
);
return;
}
Err(err) => return Err(project_config_unusable(&primary, &err.to_string())),
};
let project: toml::Value = match toml::from_str(&raw) {
Ok(v) => v,
Err(_) => return,
};
let table = match project.as_table() {
Some(t) => t,
None => return,
let project: toml::Value = toml::from_str(&raw).map_err(|err| {
// Position only: the parser's message can quote the offending value.
let reason = err.span().map_or_else(
|| "invalid TOML".to_string(),
|span| {
let prefix = &raw.as_bytes()[..span.start.min(raw.len())];
let line = prefix.iter().filter(|byte| **byte == b'\n').count() + 1;
format!("invalid TOML at line {line}")
},
);
project_config_unusable(&path, &reason)
})?;
let Some(table) = project.as_table() else {
return Err(project_config_unusable(&path, "not a TOML table"));
};
// #417: dangerous keys are denied at project scope. A malicious
@@ -11779,6 +11898,7 @@ fn merge_project_config_with_approval_baseline(
(See #417.)"
);
}
Ok(())
}
/// Maximum bytes read from a project config file. Configs are kilobytes.
@@ -12012,7 +12132,7 @@ async fn run_interactive_with_notice(
&mut merged_config,
&workspace,
saved_permission_posture.as_deref(),
);
)?;
}
if resume_session_id.is_none() {
let explicit_route_override = crate::config::explicit_launch_provider_override().is_some()
@@ -18757,6 +18877,56 @@ api_key = "test-only-key"
}
}
#[test]
fn scorecard_rejects_a_threshold_that_cannot_gate() {
for bad in ["NaN", "inf", "-inf"] {
assert!(
Cli::try_parse_from([
"codewhale",
"scorecard",
"--input",
"t.json",
"--threshold",
bad
])
.is_err(),
"--threshold {bad} must be refused"
);
}
assert!(
Cli::try_parse_from([
"codewhale",
"scorecard",
"--input",
"t.json",
"--threshold",
"2.5"
])
.is_ok()
);
}
#[test]
fn workspace_dotenv_is_found_from_the_launch_workspace_not_the_process_directory() {
let workspace = tempfile::tempdir().expect("workspace tempdir");
let nested = workspace.path().join("crates/app");
std::fs::create_dir_all(&nested).expect("mkdir nested");
std::fs::create_dir_all(workspace.path().join(".git")).expect("mkdir .git");
std::fs::write(workspace.path().join(".env"), "OPENAI_API_KEY=workspace\n")
.expect("write .env");
// `--workspace <dir>` from a process directory in another tree.
assert_eq!(
find_workspace_dotenv(workspace.path()).expect("search"),
Some(workspace.path().join(".env"))
);
// Nested launch directory walks up to the repository root.
assert_eq!(
find_workspace_dotenv(&nested).expect("search"),
Some(workspace.path().join(".env"))
);
}
#[test]
fn workspace_dotenv_loads_only_provider_credentials_and_preserves_shell_values() {
let _lock = crate::test_support::lock_test_env();
@@ -19995,7 +20165,10 @@ mod project_config_tests {
..Config::default()
};
merge_project_config(&mut config, workspace.path());
let error =
merge_project_config_with_approval_baseline(&mut config, workspace.path(), None)
.expect_err("a symlinked primary project config must stop the launch");
assert!(error.to_string().contains("--no-project-config"), "{error}");
assert_eq!(
config.default_text_model.as_deref(),
@@ -20321,7 +20494,8 @@ approval_policy = "on-request"
);
let mut config = Config::default();
merge_project_config_with_approval_baseline(&mut config, tmp.path(), Some("full-access"));
merge_project_config_with_approval_baseline(&mut config, tmp.path(), Some("full-access"))
.expect("valid project config tightens the saved baseline");
assert_eq!(
config.approval_policy.as_deref(),
@@ -20602,14 +20776,22 @@ max_subagents = -3
}
#[test]
fn project_overlay_skips_malformed_toml() {
let tmp = workspace_with_project_config("this is not valid TOML !!");
fn project_overlay_refuses_malformed_toml_instead_of_dropping_its_restrictions() {
let tmp = workspace_with_project_config(
"approval_policy = \"on-request\"\nallow_shell = false\nthis is not valid TOML !!",
);
let mut config = Config {
provider: Some("codewhale".to_string()),
..Config::default()
};
merge_project_config(&mut config, tmp.path());
// Untouched on parse error — better to fall back to global than crash.
// A broken file may be the one tightening this workspace; launching on
// the looser user baseline without it would fail open.
let error = merge_project_config_with_approval_baseline(&mut config, tmp.path(), None)
.expect_err("a malformed project config must stop the launch");
let message = error.to_string();
assert!(message.contains("invalid TOML at line 3"), "{message}");
assert!(message.contains("--no-project-config"), "{message}");
assert!(!message.contains("this is not valid"), "{message}");
assert_eq!(config.provider.as_deref(), Some("codewhale"));
}
@@ -21195,6 +21377,9 @@ mod setup_helper_tests {
std::env::set_var("USERPROFILE", home);
}
let result = f();
// `--continue` recovery takes the recovered session's live lease for
// the process; release it with the temporary home it lives in.
crate::session_manager::set_live_session(None);
unsafe {
match prev_home {
Some(value) => std::env::set_var("HOME", value),
@@ -21328,6 +21513,72 @@ mod setup_helper_tests {
});
}
/// `--continue` in a second terminal must not take the session the first
/// terminal is still running: no promotion or clear of its checkpoint, no
/// silent swap to an older session, and the attach is refused by name.
#[test]
fn continue_leaves_a_session_live_in_another_terminal_alone() {
let _guard = crate::test_support::lock_test_env();
let tmp = TempDir::new().unwrap();
let workspace = tmp.path().join("workspace");
std::fs::create_dir_all(&workspace).unwrap();
with_home(tmp.path(), || {
let manager = SessionManager::default_location().expect("manager");
let message = |text: &str| {
vec![Message {
role: Role::User,
content: vec![ContentBlock::Text {
text: text.to_string(),
cache_control: None,
}],
}]
};
let older = create_saved_session(&message("older"), "test-model", &workspace, 0, None);
manager.save_session(&older).expect("save older");
std::thread::sleep(std::time::Duration::from_millis(20));
let session =
create_saved_session(&message("still running"), "test-model", &workspace, 0, None);
let session_id = session.metadata.id.clone();
manager.save_session(&session).expect("save session");
manager.save_checkpoint(&session).expect("save checkpoint");
let lease = manager.hold_live_lease_elsewhere(&session_id);
let resolved = resolve_continue_session_id(&workspace, true);
assert_eq!(
resolved.as_deref(),
Some(session_id.as_str()),
"the newest session is named, not swapped for an older one"
);
assert!(
manager
.load_session_checkpoint(&session_id)
.expect("load checkpoint")
.is_some(),
"the live session keeps its crash-recovery checkpoint"
);
let refusal = manager
.attach_session(&session_id)
.expect_err("attaching to it is refused");
assert_eq!(refusal.kind(), io::ErrorKind::ResourceBusy);
assert!(refusal.to_string().contains(&session_id), "{refusal}");
assert!(
load_exec_resume_session(&session_id)
.expect_err("exec --continue is refused too")
.to_string()
.contains("open in another Codewhale window")
);
drop(lease);
// Once that session has exited, --continue recovers it as before.
assert_eq!(
resolve_continue_session_id(&workspace, true).as_deref(),
Some(session_id.as_str())
);
crate::session_manager::set_live_session(None);
});
}
#[test]
fn continue_without_interactive_terminal_leaves_checkpoint_for_a_real_launch() {
// `codewhale --continue </dev/null` (and `run --continue`) used to
@@ -21391,6 +21642,80 @@ mod setup_helper_tests {
std::fs::write(checkpoints.join("latest.json"), content).expect("write legacy checkpoint");
}
/// The legacy `latest.json` slot names a session. When that session is
/// open in another terminal, neither `--continue` nor a plain launch may
/// promote the slot over its document, overwrite its per-session
/// checkpoint, or consume the slot.
#[test]
fn legacy_checkpoint_of_a_session_live_elsewhere_is_left_alone() {
let _guard = crate::test_support::lock_test_env();
let tmp = TempDir::new().unwrap();
let workspace = tmp.path().join("workspace");
std::fs::create_dir_all(&workspace).unwrap();
with_home(tmp.path(), || {
let manager = SessionManager::default_location().expect("manager");
let message = |text: &str| {
vec![Message {
role: Role::User,
content: vec![ContentBlock::Text {
text: text.to_string(),
cache_control: None,
}],
}]
};
let live =
create_saved_session(&message("live turn"), "test-model", &workspace, 0, None);
let session_id = live.metadata.id.clone();
let document = manager.save_session(&live).expect("save live document");
manager
.save_checkpoint(&live)
.expect("save live checkpoint");
let mut stale = live.clone();
stale.messages = message("stale legacy slot");
stale.metadata.updated_at = live.metadata.updated_at + chrono::Duration::seconds(60);
write_legacy_checkpoint(&manager, &stale);
let legacy = manager
.sessions_dir()
.join("checkpoints")
.join("latest.json");
let document_before = std::fs::read(&document).expect("document");
let checkpoint = || {
serde_json::to_string(
&manager
.load_session_checkpoint(&session_id)
.expect("checkpoint")
.expect("present")
.messages,
)
.expect("serialize")
};
let checkpoint_before = checkpoint();
let lease = manager.hold_live_lease_elsewhere(&session_id);
let recovered = recover_interrupted_checkpoint_for_resume(&workspace);
preserve_interrupted_checkpoint_for_explicit_resume(&workspace);
assert_eq!(recovered, None, "nothing is recovered over a live session");
assert!(legacy.exists(), "the legacy slot is not consumed");
assert_eq!(
std::fs::read(&document).expect("document"),
document_before,
"the live document is not overwritten"
);
assert_eq!(
checkpoint(),
checkpoint_before,
"the live checkpoint is not replaced by the legacy slot"
);
assert!(
!crate::session_manager::is_live_session(&session_id),
"no claim was taken"
);
drop(lease);
});
}
#[test]
fn continue_recovers_legacy_checkpoint_and_migrates_it() {
let _guard = crate::test_support::lock_test_env();
+11
View File
@@ -1222,6 +1222,17 @@ pub type RetryCallback = Box<dyn Fn(&LlmError, u32, Duration) + Send + Sync>;
/// * `Ok(T)` - The successful result from the operation
/// * `Err(RetryError)` - All retries exhausted or non-retryable error encountered
///
/// # Known limitation: ambiguous failures are replayed
///
/// A timeout or connection loss after the request was written is retried
/// like a connect failure, so a provider that already accepted the first
/// attempt may bill a second completion. Every caller sends model inference
/// (messages, FIM, translation, speech, provider web search): a replay costs
/// compute but has no external side effect, and the provider APIs used here
/// expose no idempotency key for these requests that could dedupe it. An
/// operation with an external side effect must not be retried through this
/// helper without an idempotency key the server honors.
///
/// # Example
///
/// ```ignore
+45 -8
View File
@@ -826,13 +826,17 @@ impl McpOAuthRuntime {
/// the rotated token while keeping our dead grant would make the next
/// `invalid_grant` compare an "unchanged" store and delete the newer
/// valid credential.
///
/// The comparison and the delete are one step under the secret store's
/// entry lock (the same lock every save takes), so a login that lands
/// between them is never the credential that gets deleted.
async fn clear_stored_tokens(&self, reason: &str) -> Result<()> {
let held = { self.inner.last_tokens.lock().await.take() };
let Some(held) = held else {
return Ok(());
};
match load_oauth_tokens(&self.inner.server_name, &self.inner.url)? {
Some(stored) if stored != held => {
match delete_oauth_tokens_if_held(&self.inner.server_name, &self.inner.url, &held)? {
Some(stored) => {
tracing::debug!(
target: "mcp",
server = %self.inner.server_name,
@@ -845,8 +849,7 @@ impl McpOAuthRuntime {
*self.inner.last_tokens.lock().await = Some(stored);
*self.inner.rejection.lock().await = None;
}
_ => {
delete_oauth_tokens(&self.inner.server_name, &self.inner.url)?;
None => {
*self.inner.rejection.lock().await = Some(reason.to_string());
}
}
@@ -863,8 +866,9 @@ impl McpOAuthRuntime {
};
let Some(credentials) = credentials else {
let mut last = self.inner.last_tokens.lock().await;
if last.take().is_some() {
delete_oauth_tokens(&self.inner.server_name, &self.inner.url)?;
if let Some(previous) = last.take() {
// Only our own credential goes; a newer login stays.
delete_oauth_tokens_if_held(&self.inner.server_name, &self.inner.url, &previous)?;
}
return Ok(());
};
@@ -1444,9 +1448,42 @@ pub(crate) fn load_oauth_tokens(
else {
return Ok(None);
};
let mut tokens = parse_stored_oauth_tokens(&serialized, server_name)?;
decode_stored_oauth_tokens(&serialized, server_name).map(Some)
}
fn decode_stored_oauth_tokens(serialized: &str, server_name: &str) -> Result<StoredMcpOAuthTokens> {
let mut tokens = parse_stored_oauth_tokens(serialized, server_name)?;
refresh_expires_in_from_timestamp(&mut tokens);
Ok(Some(tokens))
Ok(tokens)
}
/// Delete the stored credential only while it is still exactly `held`,
/// comparing and deleting under the store's entry lock. Returns the different
/// credential found in its place (left untouched), or `None` when the entry
/// was `held` and is now gone, or was already absent. An unreadable entry is
/// an error and is left in place, as a plain load would report it.
fn delete_oauth_tokens_if_held(
server_name: &str,
url: &str,
held: &StoredMcpOAuthTokens,
) -> Result<Option<StoredMcpOAuthTokens>> {
let secrets = codewhale_secrets::Secrets::auto_detect();
let key = store_key(server_name, url);
secrets
.with_entry_transaction(&key, |current| {
let Some(serialized) = current.as_deref() else {
return Ok(Ok(None));
};
Ok(match decode_stored_oauth_tokens(serialized, server_name) {
Ok(stored) if stored == *held => {
*current = None;
Ok(None)
}
Ok(stored) => Ok(Some(stored)),
Err(error) => Err(error),
})
})
.with_context(|| format!("clearing the MCP OAuth token for '{server_name}'"))?
}
fn parse_stored_oauth_tokens(serialized: &str, server_name: &str) -> Result<StoredMcpOAuthTokens> {
+58 -18
View File
@@ -28,6 +28,13 @@ pub(super) struct StdioTransport {
pub(super) authority_cancel_watch: Option<tokio::task::JoinHandle<()>>,
/// Holds reviewed executable/script handles for the process lifetime.
pub(super) _reviewed_launch: Option<super::ReviewedStdioLaunch>,
/// Everything the server started: its own process group on Unix, a Job
/// Object on Windows. Termination signals the whole tree, so a server's
/// grandchild (an `npx` wrapper's node, a shell's background job) dies
/// with it on shutdown and on plugin revocation. The last clone's drop
/// kills whatever is left. `None` only for a transport built around an
/// uncontained child (tests).
pub(super) process_tree: Option<Arc<crate::process_tree::ProcessTree>>,
}
/// How long `StdioTransport::shutdown` waits for the child to exit on SIGTERM
@@ -172,6 +179,9 @@ impl StdioTransport {
child_env::string_map_env(&expanded_env),
);
}
// Lead a process group of its own so teardown can reach descendants.
#[cfg(unix)]
cmd.process_group(0);
let mut child = cmd.spawn().map_err(|error| {
let message = if error.kind() == std::io::ErrorKind::NotFound
@@ -195,6 +205,16 @@ impl StdioTransport {
anyhow::Error::new(error).context(message)
})?;
let process_tree = match crate::process_tree::ProcessTree::attach_tokio(&child) {
Ok(tree) => Arc::new(tree),
Err(error) => {
let _ = child.start_kill();
return Err(anyhow::Error::new(error).context(format!(
"MCP server {server_name} could not be contained with its child processes"
)));
}
};
let stdin = child.stdin.take().context("Failed to get MCP stdin")?;
let stdout = child.stdout.take().context("Failed to get MCP stdout")?;
let stderr = child.stderr.take().context("Failed to get MCP stderr")?;
@@ -218,9 +238,10 @@ impl StdioTransport {
let child = Arc::new(TokioMutex::new(child));
let authority_cancel_watch = config.reviewed_plugin.as_ref().map(|_| {
let watched_child = Arc::clone(&child);
let watched_tree = Arc::clone(&process_tree);
tokio::spawn(async move {
cancel_token.cancelled().await;
terminate_child_for_authority_change(&watched_child).await;
terminate_child_for_authority_change(&watched_child, &watched_tree).await;
})
});
@@ -232,6 +253,7 @@ impl StdioTransport {
stderr_tail,
authority_cancel_watch,
_reviewed_launch: reviewed_launch,
process_tree: Some(process_tree),
})
}
}
@@ -315,19 +337,22 @@ async fn format_stderr_context(tail: &StderrTail) -> Option<String> {
))
}
/// Best-effort SIGTERM. On Unix uses `libc::kill`; on Windows there's no
/// equivalent so we let `kill_on_drop` (TerminateProcess) handle it via the
/// subsequent Drop. Returns whether a signal was actually sent.
fn send_sigterm(child: &Child) -> bool {
/// Best-effort SIGTERM. On Unix uses `libc::kill`, addressed to the child's
/// whole process group when it leads one (`contained`); on Windows there's no
/// equivalent so we let `kill_on_drop` (TerminateProcess) and the Job Object
/// handle it. Returns whether a signal was actually sent.
fn send_sigterm(child: &Child, contained: bool) -> bool {
#[cfg(unix)]
{
if let Some(pid) = child.id() {
// SAFETY: pid was just obtained from `child.id()`. `libc::kill`
// with `SIGTERM` is async-signal-safe and never observes invalid
// memory. Worst case (pid wrap / process already gone) returns
// ESRCH, which we deliberately ignore.
let pid = pid as i32;
let target = if contained { -pid } else { pid };
// SAFETY: pid was just obtained from `child.id()` of an unreaped
// child, so neither it nor the group it leads can have been
// recycled. `libc::kill` with `SIGTERM` is async-signal-safe and
// never observes invalid memory. ESRCH is deliberately ignored.
unsafe {
let _ = libc::kill(pid as i32, libc::SIGTERM);
let _ = libc::kill(target, libc::SIGTERM);
}
return true;
}
@@ -335,27 +360,34 @@ fn send_sigterm(child: &Child) -> bool {
}
#[cfg(not(unix))]
{
let _ = child;
let _ = (child, contained);
false
}
}
async fn terminate_child_for_authority_change(child: &Arc<TokioMutex<Child>>) {
async fn terminate_child_for_authority_change(
child: &Arc<TokioMutex<Child>>,
tree: &crate::process_tree::ProcessTree,
) {
let mut child = child.lock().await;
terminate_child(&mut child).await;
terminate_child(&mut child, Some(tree)).await;
}
async fn terminate_child(child: &mut Child) {
async fn terminate_child(child: &mut Child, tree: Option<&crate::process_tree::ProcessTree>) {
// Reap an already-exited child before resolving its PID. Until it is
// reaped, the OS cannot recycle that identity; after it is reaped there is
// nothing left to signal. This avoids a PID-only watcher ever targeting an
// unrelated process after rapid PID reuse.
if child.try_wait().is_ok_and(|status| status.is_some()) {
// The server is gone, but what it started may not be.
if let Some(tree) = tree {
let _ = tree.kill();
}
return;
}
#[cfg(unix)]
send_sigterm(child);
send_sigterm(child, tree.is_some());
#[cfg(not(unix))]
let _ = child.start_kill();
@@ -369,6 +401,10 @@ async fn terminate_child(child: &mut Child) {
let _ = child.wait().await;
}
}
// Descendants that outlived the grace (or ignored SIGTERM) go now.
if let Some(tree) = tree {
let _ = tree.kill();
}
}
#[async_trait::async_trait]
@@ -450,7 +486,7 @@ impl McpTransport for StdioTransport {
/// then force termination and reap the child as the backstop.
async fn shutdown(&mut self) {
let mut child = self.child.lock().await;
terminate_child(&mut child).await;
terminate_child(&mut child, self.process_tree.as_deref()).await;
}
}
@@ -466,18 +502,22 @@ impl Drop for StdioTransport {
if let Ok(runtime) = tokio::runtime::Handle::try_current() {
let child = Arc::clone(&self.child);
let reviewed_launch = self._reviewed_launch.take();
// The task owns the tree so it is not killed before the grace.
let tree = self.process_tree.take();
runtime.spawn(async move {
let _reviewed_launch = reviewed_launch;
let mut child = child.lock().await;
terminate_child(&mut child).await;
terminate_child(&mut child, tree.as_deref()).await;
});
return;
}
if let Ok(mut child) = self.child.try_lock()
&& !child.try_wait().is_ok_and(|status| status.is_some())
{
send_sigterm(&child);
send_sigterm(&child, self.process_tree.is_some());
}
// No runtime: dropping the tree below SIGKILLs the group / closes the
// job, the same backstop `kill_on_drop` gives the direct child.
}
}
+44
View File
@@ -1946,6 +1946,48 @@ async fn plugin_stdio_does_not_surface_reviewed_child_stderr() {
assert!(!error.contains("ARBITRARY_PLUGIN_CREDENTIAL"));
}
/// Shutdown reaches what the server started, not only the server: a
/// background grandchild (an `npx` wrapper's node, a shell job) must not
/// survive the transport.
#[cfg(unix)]
#[tokio::test]
async fn stdio_shutdown_also_terminates_the_servers_grandchildren() {
let mut config = test_server_config();
config.command = Some("sh".to_string());
config.args = vec!["-c".to_string(), "sleep 300 & echo $!; wait".to_string()];
let mut transport = StdioTransport::spawn(
"grandparent",
"sh",
&config,
tokio_util::sync::CancellationToken::new(),
)
.unwrap();
let line = tokio::time::timeout(Duration::from_secs(5), transport.recv())
.await
.expect("grandchild pid line")
.unwrap();
let grandchild: i32 = String::from_utf8(line).unwrap().trim().parse().unwrap();
transport.shutdown().await;
let deadline = tokio::time::Instant::now() + Duration::from_secs(3);
loop {
// SAFETY: signal 0 only probes whether the pid still exists.
if unsafe { libc::kill(grandchild, 0) } != 0 {
break;
}
if tokio::time::Instant::now() >= deadline {
// Do not leak the sleeper past a failing run.
// SAFETY: plain kill(2) of the pid this test started.
unsafe {
libc::kill(grandchild, libc::SIGKILL);
}
panic!("the MCP server's grandchild outlived the transport");
}
tokio::time::sleep(Duration::from_millis(20)).await;
}
}
/// #6187: a crashed stdio child must stop reading as "ready" before any
/// call is in flight — `is_ready` probes the child, so the pool rebuilds
/// the connection on the next use instead of handing the dead transport
@@ -5120,6 +5162,7 @@ async fn stdio_transport_shutdown_terminates_child() {
stderr_tail: StderrTail::new(),
authority_cancel_watch: None,
_reviewed_launch: None,
process_tree: None,
};
// shutdown() should send SIGTERM and complete within the grace window.
@@ -5249,6 +5292,7 @@ async fn stdio_transport_recv_error_includes_stderr_tail() {
stderr_tail,
authority_cancel_watch: None,
_reviewed_launch: None,
process_tree: None,
};
// Give the subprocess time to write its stderr line and exit.
+52 -14
View File
@@ -64,6 +64,9 @@ pub(crate) enum AutoRouterSetupIssue {
Incomplete,
/// The route is complete but its credential is missing.
MissingKey,
/// `thinking` is not a reasoning tier; the classifier call would carry
/// an effort the provider rejects or silently reinterprets.
InvalidThinking,
}
impl AutoRouterSetupIssue {
@@ -76,6 +79,9 @@ impl AutoRouterSetupIssue {
}
Self::Incomplete => "[auto.router] needs a known provider and a model",
Self::MissingKey => "no API key for the router route",
Self::InvalidThinking => {
"[auto.router] thinking must be auto, off, minimal, low, medium, high, xhigh, ultra, or max"
}
}
}
}
@@ -322,20 +328,28 @@ impl ModelInventory {
router_setup_issue = Some(AutoRouterSetupIssue::UnknownKind);
None
}
Some(AutoRouterKind::Chat) => router_provider_setting
.and_then(ApiProvider::parse)
.zip(router_model_setting)
.map(|(provider, model)| {
(
provider,
model.to_string(),
router_table
.and_then(|router| router.thinking.as_deref())
.map(str::trim)
.filter(|t| !t.is_empty())
.map(str::to_string),
)
}),
Some(AutoRouterKind::Chat) => {
let thinking = router_table
.and_then(|router| router.thinking.as_deref())
.map(str::trim)
.filter(|t| !t.is_empty());
// A typo here would otherwise ride every classifier request
// as an effort string the provider rejects, failing Auto back
// to the local fallback on every turn.
if thinking.is_some_and(|thinking| {
crate::reasoning_preference::ReasoningEffort::parse_strict(thinking).is_err()
}) {
router_setup_issue = Some(AutoRouterSetupIssue::InvalidThinking);
None
} else {
router_provider_setting
.and_then(ApiProvider::parse)
.zip(router_model_setting)
.map(|(provider, model)| {
(provider, model.to_string(), thinking.map(str::to_string))
})
}
}
Some(AutoRouterKind::Decision) => {
match router_provider_setting.map(DecisionRouterRoute::parse) {
Some(None) => {
@@ -1507,6 +1521,30 @@ mod decision_router_inventory_tests {
);
}
#[test]
fn a_chat_router_with_an_unknown_thinking_tier_is_not_configured() {
let _env = hermetic();
let chat = |thinking: &str| crate::config::AutoRouterConfig {
kind: Some("chat".to_string()),
provider: Some("openrouter".to_string()),
model: Some("openai/gpt-5-mini".to_string()),
thinking: Some(thinking.to_string()),
..Default::default()
};
let typo = ModelInventory::from_config(&with_router(chat("hgih"), true));
assert!(!typo.router_configured);
assert!(!typo.router_available);
assert_eq!(
typo.router_setup_issue,
Some(AutoRouterSetupIssue::InvalidThinking)
);
let valid = ModelInventory::from_config(&with_router(chat("low"), true));
assert!(valid.router_available);
assert_eq!(valid.router_setup_issue, None);
assert_eq!(valid.router_thinking.as_deref(), Some("low"));
}
#[test]
fn a_router_table_with_only_tuning_keys_is_declared_and_incomplete() {
let _env = hermetic();
+180 -12
View File
@@ -46,6 +46,14 @@ pub const ENV_MODELS_DEV_PATH: &str = "CODEWHALE_MODELS_DEV_PATH";
pub const ENV_DISABLE_FETCH: &str = "CODEWHALE_DISABLE_MODELS_DEV_FETCH";
const CACHE_SCHEMA_VERSION: u32 = 1;
/// Largest catalog body accepted from the network, an override file, or the
/// disk cache. The public catalog is a few MiB; anything past this is refused
/// instead of being read whole into memory.
const MAX_CATALOG_BYTES: usize = 32 * 1024 * 1024;
/// Clock skew tolerated on a cache timestamp. A `fetched_at` further in the
/// future than this is not "age zero": it is untrustworthy, so the cache is
/// published as stale and the next refresh fetches.
const MAX_CACHE_CLOCK_SKEW_SECS: u64 = 5 * 60;
/// Provenance / freshness of the Models.dev live layer for UI chips (#4187).
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
@@ -227,11 +235,16 @@ pub fn maybe_load_persisted_cache() {
let Some(cache) = load_cache_file(&path) else {
return;
};
let age = now_unix().saturating_sub(cache.fetched_at);
let freshness = if age > DEFAULT_MODELS_DEV_TTL_SECS {
ModelsDevFreshness::Stale
} else {
// Live only when the cache is recent by a trustworthy clock *and* came
// from the source this process would fetch now. A cache written for
// another mirror or override file still publishes (offline startups keep
// rows) but as stale, so the next refresh replaces it.
let freshness = if within_ttl(cache.fetched_at, now_unix())
&& cache.source_fingerprint == current_source_fingerprint()
{
ModelsDevFreshness::Live
} else {
ModelsDevFreshness::Stale
};
if let Err(err) = publish_from_body(
&cache.body,
@@ -269,7 +282,7 @@ pub async fn refresh(force_network: bool) -> Result<usize, ModelsDevRefreshError
if current.freshness == ModelsDevFreshness::Live
&& current
.fetched_at
.is_some_and(|ts| now_unix().saturating_sub(ts) < DEFAULT_MODELS_DEV_TTL_SECS)
.is_some_and(|ts| within_ttl(ts, now_unix()))
{
return Ok(current.offering_count);
}
@@ -302,6 +315,29 @@ pub async fn refresh(force_network: bool) -> Result<usize, ModelsDevRefreshError
Ok(count)
}
/// Whether a payload fetched at `fetched_at` is still inside the TTL at
/// `now`. A timestamp beyond the tolerated clock skew is never fresh: a
/// saturating age would read it as zero and suppress refresh until the clock
/// caught up.
fn within_ttl(fetched_at: u64, now: u64) -> bool {
if fetched_at > now.saturating_add(MAX_CACHE_CLOCK_SKEW_SECS) {
return false;
}
now.saturating_sub(fetched_at) <= DEFAULT_MODELS_DEV_TTL_SECS
}
/// Fingerprint of the source a refresh would read right now: the override
/// file when `CODEWHALE_MODELS_DEV_PATH` is set, else the catalog URL. Same
/// derivation `refresh` stores in the cache.
fn current_source_fingerprint() -> String {
match std::env::var(ENV_MODELS_DEV_PATH) {
Ok(path) if !path.trim().is_empty() => {
base_url_fingerprint(&format!("file:{}", Path::new(path.trim()).display()))
}
_ => base_url_fingerprint(&resolve_catalog_url()),
}
}
/// Best-effort background refresh: never panics, never blocks callers.
pub fn spawn_background_refresh() {
if env_truthy(ENV_DISABLE_FETCH) && std::env::var(ENV_MODELS_DEV_PATH).is_err() {
@@ -328,10 +364,9 @@ pub fn spawn_background_refresh() {
}
async fn refresh_from_path(path: &Path) -> Result<usize, ModelsDevRefreshError> {
let body = match tokio::fs::read_to_string(path).await {
let body = match read_catalog_file(path).await {
Ok(body) => body,
Err(err) => {
let mapped = ModelsDevRefreshError::Io(err.to_string());
Err(mapped) => {
mark_failed(mapped.clone());
return Err(mapped);
}
@@ -375,10 +410,56 @@ async fn fetch_catalog_body(url: &str) -> Result<String, ModelsDevRefreshError>
return Err(ModelsDevRefreshError::HttpStatus(status.as_u16()));
}
response
.text()
read_catalog_response(response, MAX_CATALOG_BYTES).await
}
fn oversized_catalog(limit: usize) -> ModelsDevRefreshError {
ModelsDevRefreshError::InvalidResponse(format!("catalog exceeds {limit} bytes"))
}
/// Read a catalog response body without ever holding more than `limit` bytes.
async fn read_catalog_response(
mut response: reqwest::Response,
limit: usize,
) -> Result<String, ModelsDevRefreshError> {
if response
.content_length()
.is_some_and(|length| length > limit as u64)
{
return Err(oversized_catalog(limit));
}
let mut bytes = Vec::new();
while let Some(chunk) = response
.chunk()
.await
.map_err(|err| ModelsDevRefreshError::Network(err.to_string()))
.map_err(|err| ModelsDevRefreshError::Network(err.to_string()))?
{
if bytes.len().saturating_add(chunk.len()) > limit {
return Err(oversized_catalog(limit));
}
bytes.extend_from_slice(&chunk);
}
String::from_utf8(bytes)
.map_err(|_| ModelsDevRefreshError::InvalidResponse("catalog is not UTF-8".into()))
}
/// Read an override catalog file, refusing one larger than the catalog limit.
async fn read_catalog_file(path: &Path) -> Result<String, ModelsDevRefreshError> {
use tokio::io::AsyncReadExt as _;
let file = tokio::fs::File::open(path)
.await
.map_err(|err| ModelsDevRefreshError::Io(err.to_string()))?;
let mut bytes = Vec::new();
file.take(MAX_CATALOG_BYTES as u64 + 1)
.read_to_end(&mut bytes)
.await
.map_err(|err| ModelsDevRefreshError::Io(err.to_string()))?;
if bytes.len() > MAX_CATALOG_BYTES {
return Err(oversized_catalog(MAX_CATALOG_BYTES));
}
String::from_utf8(bytes)
.map_err(|_| ModelsDevRefreshError::InvalidResponse("catalog is not UTF-8".into()))
}
fn publish_from_body(
@@ -430,7 +511,20 @@ fn mark_failed(err: ModelsDevRefreshError) {
/// with an escaped body) formats. Returns metadata and the *unescaped* body
/// without copying it in the v2 path.
fn load_cache_file(path: &Path) -> Option<PersistedModelsDevCache> {
let bytes = std::fs::read(path).ok()?;
use std::io::Read as _;
// Header line plus body; a larger file is not ours to trust or to read
// whole, so the bundled rows stand instead.
const MAX_CACHE_FILE_BYTES: u64 = MAX_CATALOG_BYTES as u64 + 64 * 1024;
let mut bytes = Vec::new();
std::fs::File::open(path)
.ok()?
.take(MAX_CACHE_FILE_BYTES + 1)
.read_to_end(&mut bytes)
.ok()?;
if bytes.len() as u64 > MAX_CACHE_FILE_BYTES {
return None;
}
// v2: single-line JSON header terminated by a newline, then the verbatim
// catalog body. One small parse, zero body copies.
if bytes.first() == Some(&b'{') && bytes.contains(&b'\n') {
@@ -735,6 +829,80 @@ mod tests {
clear_live_snapshot();
}
#[test]
fn a_future_or_foreign_cache_is_never_live() {
let now = 2_000_000_000;
assert!(within_ttl(now - 60, now));
assert!(within_ttl(now + 30, now), "small skew is tolerated");
assert!(
!within_ttl(now + 3 * 24 * 60 * 60, now),
"a future timestamp is not age zero"
);
assert!(!within_ttl(now - DEFAULT_MODELS_DEV_TTL_SECS - 1, now));
let _lock = lock_test_env();
let _live = lock_live_snapshot();
clear_live_snapshot();
let dir = tempfile::tempdir().expect("tempdir");
let home = dir.path().join("home");
let _home = EnvVarGuard::set("CODEWHALE_HOME", &home);
let _path = EnvVarGuard::remove(ENV_MODELS_DEV_PATH);
let _url = EnvVarGuard::remove(ENV_MODELS_DEV_URL);
let cache_dir = home.join("catalog");
std::fs::create_dir_all(&cache_dir).expect("mkdir");
let cache = cache_dir.join(CACHE_FILE);
// Fresh by the clock, but fetched from another source.
let foreign = PersistedModelsDevCache {
schema_version: CACHE_SCHEMA_VERSION,
fetched_at: now_unix(),
source_fingerprint: base_url_fingerprint("https://mirror.example/catalog.json"),
source_label: "https://mirror.example/catalog.json".into(),
body: FIXTURE.into(),
};
save_cache_file(&cache, &foreign).expect("save");
maybe_load_persisted_cache();
assert_eq!(status().freshness, ModelsDevFreshness::Stale);
// From the current source, but stamped far in the future.
let future = PersistedModelsDevCache {
fetched_at: now_unix() + 30 * 24 * 60 * 60,
source_fingerprint: base_url_fingerprint(MODELS_DEV_CATALOG_URL),
source_label: MODELS_DEV_CATALOG_URL.into(),
..foreign.clone()
};
save_cache_file(&cache, &future).expect("save");
maybe_load_persisted_cache();
assert_eq!(status().freshness, ModelsDevFreshness::Stale);
// The same cache stamped now is live.
let current = PersistedModelsDevCache {
fetched_at: now_unix(),
..future
};
save_cache_file(&cache, &current).expect("save");
maybe_load_persisted_cache();
assert_eq!(status().freshness, ModelsDevFreshness::Live);
clear_live_snapshot();
}
#[test]
fn an_oversized_override_catalog_is_refused() {
let dir = tempfile::tempdir().expect("tempdir");
let path = dir.path().join("huge.json");
let file = std::fs::File::create(&path).expect("create");
file.set_len(MAX_CATALOG_BYTES as u64 + 1)
.expect("sparse size");
let rt = tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.expect("runtime");
let error = rt
.block_on(read_catalog_file(&path))
.expect_err("oversized catalog must be refused");
assert!(error.to_string().contains("exceeds"), "{error}");
}
#[test]
fn network_failure_keeps_prior_rows_and_marks_failed() {
let _lock = lock_test_env();
+89 -29
View File
@@ -165,6 +165,10 @@ static MERGED_CACHE: RwLock<Option<MergedCacheEntry>> = RwLock::new(None);
/// 600+ row OpenRouter catalog for every route candidate.
static RUNTIME_RESOLVER_CACHE: RwLock<BTreeMap<String, RuntimeResolverCacheEntry>> =
RwLock::new(BTreeMap::new());
/// Most distinct endpoints the resolver cache holds at once. Entries from an
/// older catalog generation can never hit again and are dropped on insert;
/// this caps the endpoints of the current generation.
const MAX_RUNTIME_RESOLVER_CACHE_ENTRIES: usize = 64;
#[derive(Clone)]
struct RuntimeResolverCacheEntry {
@@ -408,27 +412,61 @@ pub fn live_catalog_origin(provider: ApiProvider, wire_model_id: &str) -> Option
let Ok(guard) = LIVE_SNAPSHOT.read() else {
return None;
};
let matches = |row: &CatalogOffering| {
row.provider.eq_ignore_ascii_case(catalog_id)
&& row.wire_model_id.eq_ignore_ascii_case(needle)
};
if guard
.per_provider
.get(&owner)
.is_some_and(|snap| snap.offerings.iter().any(matches))
{
return Some(LiveSource::PerProvider);
}
if guard
.models_dev
.as_ref()
.is_some_and(|snap| snap.offerings.iter().any(|row| matches(row)))
{
return Some(LiveSource::ModelsDev);
// Wire ids are opaque and case-sensitive: an exact row in either
// partition decides before a case-folded one does.
for exact in [true, false] {
let matches = |row: &CatalogOffering| {
row.provider.eq_ignore_ascii_case(catalog_id)
&& if exact {
row.wire_model_id == needle
} else {
row.wire_model_id.eq_ignore_ascii_case(needle)
}
};
if guard
.per_provider
.get(&owner)
.is_some_and(|snap| snap.offerings.iter().any(matches))
{
return Some(LiveSource::PerProvider);
}
if guard
.models_dev
.as_ref()
.is_some_and(|snap| snap.offerings.iter().any(|row| matches(row)))
{
return Some(LiveSource::ModelsDev);
}
}
None
}
/// The row whose wire id is exactly `needle`, else the one wire id equal to
/// it ignoring ASCII case. Wire ids are opaque and case-sensitive, so the
/// fallback only forgives a case slip when it cannot choose between two
/// distinct ids; otherwise it answers nothing rather than another model's
/// metadata.
fn find_wire_model<'a>(
rows: impl IntoIterator<Item = &'a CatalogOffering>,
needle: &str,
) -> Option<&'a CatalogOffering> {
let mut folded: Option<&'a CatalogOffering> = None;
let mut ambiguous = false;
for row in rows {
if row.wire_model_id == needle {
return Some(row);
}
if row.wire_model_id.eq_ignore_ascii_case(needle) {
match folded {
None => folded = Some(row),
Some(first) if first.wire_model_id != row.wire_model_id => ambiguous = true,
Some(_) => {}
}
}
}
folded.filter(|_| !ambiguous)
}
/// Serialize tests that mutate the process-wide live snapshot.
///
/// Lock ordering: this takes the test env barrier FIRST (skipped when the
@@ -938,6 +976,12 @@ pub(crate) fn runtime_catalog_resolver_for_identity(
let resolver = RouteResolver::from_offerings(route_offerings.into_values().collect());
if let Ok(mut cache) = RUNTIME_RESOLVER_CACHE.write() {
cache.retain(|_, entry| {
entry.generation == generation && entry.cloud_generation == cloud_generation
});
if cache.len() >= MAX_RUNTIME_RESOLVER_CACHE_ENTRIES && !cache.contains_key(&cache_key) {
cache.pop_first();
}
cache.insert(
cache_key,
RuntimeResolverCacheEntry {
@@ -1110,14 +1154,14 @@ pub fn catalog_offering_for_model_identity(
return None;
}
if provider == ApiProvider::Custom {
return exact_custom_offerings(catalog_id.as_ref())
.into_iter()
.find(|row| row.wire_model_id.eq_ignore_ascii_case(needle));
let rows = exact_custom_offerings(catalog_id.as_ref());
return find_wire_model(&rows, needle).cloned();
}
offerings_for_provider_identity(&merged_snapshot(), catalog_id.as_ref())
.into_iter()
.find(|row| row.wire_model_id.eq_ignore_ascii_case(needle))
.cloned()
find_wire_model(
offerings_for_provider_identity(&merged_snapshot(), catalog_id.as_ref()),
needle,
)
.cloned()
}
/// Metadata from the exact route, without borrowing another endpoint's live facts.
@@ -1252,11 +1296,11 @@ pub fn bundled_catalog_offering_for_model(
if needle.is_empty() {
return None;
}
bundled_snapshot()
.offerings_for_provider(catalog_id)
.into_iter()
.find(|row| row.wire_model_id.eq_ignore_ascii_case(needle))
.cloned()
find_wire_model(
bundled_snapshot().offerings_for_provider(catalog_id),
needle,
)
.cloned()
}
/// Count of merged-catalog models for one provider (catalog view / dashboard).
@@ -1860,6 +1904,22 @@ mod tests {
use crate::config::{DEFAULT_TOGETHER_FLASH_MODEL, DEFAULT_TOGETHER_MODEL};
use codewhale_config::catalog::CatalogSource;
#[test]
fn wire_model_lookup_is_exact_first_and_never_guesses_between_case_variants() {
let row = |id: &str| CatalogOffering {
provider: "p".to_string(),
wire_model_id: id.to_string(),
..Default::default()
};
let rows = vec![row("org/Model-A"), row("org/model-a"), row("org/Model-B")];
let found =
|needle: &str| find_wire_model(&rows, needle).map(|row| row.wire_model_id.clone());
assert_eq!(found("org/model-a").as_deref(), Some("org/model-a"));
assert_eq!(found("org/Model-A").as_deref(), Some("org/Model-A"));
assert_eq!(found("ORG/MODEL-A"), None, "two distinct ids fold together");
assert_eq!(found("org/model-b").as_deref(), Some("org/Model-B"));
}
fn catalog_test_config(first_url: &str, second_url: &str) -> Config {
use crate::config::{ProviderConfig, ProvidersConfig};
Config {
+96
View File
@@ -3290,6 +3290,9 @@ async fn relay_worker(
sync_tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
let mut runtime_retry_delay = RUNTIME_UPLOAD_RETRY_INTERVAL;
let mut runtime_retry_not_before = Instant::now();
// One credential refresh per rejection streak (see
// `RUNTIME_REFRESHED_CREDENTIAL_REFUSED`).
let mut runtime_refreshed_since_accept = false;
loop {
tokio::select! {
@@ -3412,11 +3415,15 @@ async fn relay_worker(
RuntimeFlushOutcome::Accepted { run_id, cursor } => {
runtime_retry_delay = RUNTIME_UPLOAD_RETRY_INTERVAL;
runtime_retry_not_before = Instant::now();
runtime_refreshed_since_accept = false;
event_tx
.send(RemoteEvent::RuntimeCursor { run_id, cursor })
.map_err(|_| "The terminal remote-control owner stopped.".to_string())?;
}
RuntimeFlushOutcome::AccessTokenExpired => {
if std::mem::replace(&mut runtime_refreshed_since_accept, true) {
return Err(RUNTIME_REFRESHED_CREDENTIAL_REFUSED.to_string());
}
refresh_enrollment_and_reconnect(
&client,
&mut enrollment,
@@ -3694,6 +3701,12 @@ impl RuntimeTransportOutbox {
}
}
/// A runtime event refused (401/403) again right after a successful
/// credential refresh is a terminal rejection, not an expired token: a
/// revoked runner or a run this runner may not write. Refreshing again would
/// loop at the upload retry interval forever.
const RUNTIME_REFRESHED_CREDENTIAL_REFUSED: &str = "The remote-control server refused runtime events again with a freshly refreshed credential; stopping instead of retrying.";
/// Flushes every queued runtime envelope through the server-confirmed cursor
/// before a stop may be acknowledged. Emits `RuntimeCursor` events so the
/// controller compacts its journal as acknowledgements land. Failing to drain
@@ -3710,6 +3723,7 @@ async fn drain_runtime_outbox_for_stop(
deadline: Instant,
) -> Result<(), String> {
let mut delay = RUNTIME_UPLOAD_RETRY_INTERVAL;
let mut refreshed_since_accept = false;
while !outbox.events.is_empty() {
if Instant::now() >= deadline {
return Err(
@@ -3721,6 +3735,7 @@ async fn drain_runtime_outbox_for_stop(
RuntimeFlushOutcome::Idle => break,
RuntimeFlushOutcome::Accepted { run_id, cursor } => {
delay = RUNTIME_UPLOAD_RETRY_INTERVAL;
refreshed_since_accept = false;
let _ = event_tx.send(RemoteEvent::RuntimeCursor { run_id, cursor });
}
RuntimeFlushOutcome::Retryable => {
@@ -3728,6 +3743,9 @@ async fn drain_runtime_outbox_for_stop(
delay = delay.saturating_mul(2).min(RUNTIME_UPLOAD_MAX_BACKOFF);
}
RuntimeFlushOutcome::AccessTokenExpired => {
if std::mem::replace(&mut refreshed_since_accept, true) {
return Err(RUNTIME_REFRESHED_CREDENTIAL_REFUSED.to_string());
}
refresh_enrollment_and_reconnect(client, enrollment, runner_id, start, event_tx)
.await?;
}
@@ -7938,6 +7956,84 @@ mod tests {
assert_eq!(bodies[0], bodies[1], "retries must be byte-identical");
}
#[tokio::test]
async fn stop_drain_stops_when_a_fresh_credential_is_refused_again() {
let _env = crate::test_support::lock_test_env();
let secrets_root = tempfile::tempdir().expect("isolated remote-control secrets");
let _codewhale_home =
crate::test_support::EnvVarGuard::set("CODEWHALE_HOME", secrets_root.path());
let server = MockServer::start().await;
// A terminal refusal: the runner may not write this run, and a new
// access token does not change that.
Mock::given(method("POST"))
.and(path(
"/api/local-runners/runner_fixture/runs/run_fixture/events",
))
.respond_with(ResponseTemplate::new(403))
.expect(2)
.mount(&server)
.await;
let access_token = crate::test_support::future_test_jwt(&"a".repeat(40));
Mock::given(method("POST"))
.and(path("/api/runner/enrollments/token"))
.respond_with(ResponseTemplate::new(200).set_body_json(json!({
"enrollment": {
"id": "enrollment_fixture",
"userId": "account_fixture",
"deviceId": "device_fixture",
"runtimeVersion": "0.9.6",
"runtimeCommit": "a".repeat(40),
"capabilities": CAPABILITIES,
},
"credential": { "accessToken": access_token },
})))
.expect(1)
.mount(&server)
.await;
Mock::given(method("POST"))
.and(path("/api/local-runners/connect"))
.respond_with(ResponseTemplate::new(200).set_body_json(fixture_connection_response()))
.mount(&server)
.await;
let mut enrollment = fixture_enrollment(&format!("{}/", server.uri()));
let mut runner_id = "runner_fixture".to_string();
let client = crate::tls::reqwest_client_builder()
.redirect(reqwest::redirect::Policy::none())
.build()
.expect("fixture client");
let (event_tx, _event_rx) = mpsc::unbounded_channel();
let mut outbox = RuntimeTransportOutbox::default();
outbox
.enqueue(
"run_fixture",
runtime_envelope(
1,
"turn.completed",
Some("turn_fixture"),
"2026-08-08T12:00:00Z".to_string(),
json!({ "turn": { "status": "completed", "usage": {} } }),
),
)
.expect("queue terminal envelope");
let error = drain_runtime_outbox_for_stop(
&client,
&mut enrollment,
&mut runner_id,
&fixture_start(),
&event_tx,
&mut outbox,
Instant::now() + Duration::from_secs(5),
)
.await
.expect_err("a refusal that survives a refresh must end the drain");
assert_eq!(error, RUNTIME_REFRESHED_CREDENTIAL_REFUSED);
assert!(
!outbox.events.is_empty(),
"the refused envelope stays queued for a later, authorized resend"
);
}
#[tokio::test]
async fn stop_drain_deadline_failure_refuses_to_confirm_stop() {
let server = MockServer::start().await;
+65 -5
View File
@@ -295,6 +295,7 @@ fn lighthouse_plan(inputs: &DeployInputs) -> Vec<ProvisionStep> {
fn azure_plan(inputs: &DeployInputs) -> Vec<ProvisionStep> {
let rg = format!("{}-rg", inputs.instance_name);
let vault = format!("{}-kv", inputs.instance_name);
let nsg = format!("{}-nsg", inputs.instance_name);
let provider_secret = format!("codewhale-{}-key", inputs.provider_slug);
vec![
ProvisionStep::new(
@@ -355,20 +356,40 @@ fn azure_plan(inputs: &DeployInputs) -> Vec<ProvisionStep> {
"--custom-data",
"cloud-init.yaml",
"--assign-identity",
// A Linux `vm create` otherwise adds an NSG rule admitting SSH
// from any address; the next step adds the caller-only rule.
"--nsg",
&nsg,
"--nsg-rule",
"NONE",
],
),
ProvisionStep::new(
"Scope the NSG to SSH (22) from the caller IP only; 7878 stays on 127.0.0.1",
"Allow SSH (22) from the caller IP only (replace <your-public-ip>); 7878 stays on 127.0.0.1",
"az",
&[
"vm",
"open-port",
"network",
"nsg",
"rule",
"create",
"--resource-group",
&rg,
"--nsg-name",
&nsg,
"--name",
&inputs.instance_name,
"--port",
"allow-ssh-from-caller",
"--priority",
"1000",
"--direction",
"Inbound",
"--access",
"Allow",
"--protocol",
"Tcp",
"--destination-port-ranges",
"22",
"--source-address-prefixes",
"<your-public-ip>/32",
],
),
]
@@ -540,6 +561,45 @@ mod tests {
);
}
#[test]
fn azure_plan_admits_ssh_from_the_caller_only() {
let steps = (AZURE.plan)(&DeployInputs::default());
let arg_after = |step: &ProvisionStep, flag: &str| {
let at = step.args.iter().position(|arg| arg == flag)?;
step.args.get(at + 1).cloned()
};
// `open-port` has no source restriction: it admits the whole internet.
assert!(
steps
.iter()
.all(|s| !s.args.iter().any(|a| a == "open-port")),
"Azure plan must not open a port to every source"
);
let create = steps
.iter()
.find(|s| {
s.args
.starts_with(&["vm".to_string(), "create".to_string()])
})
.expect("vm create step");
assert_eq!(arg_after(create, "--nsg-rule").as_deref(), Some("NONE"));
let ssh_rules: Vec<_> = steps
.iter()
.filter(|s| arg_after(s, "--destination-port-ranges").as_deref() == Some("22"))
.collect();
assert_eq!(ssh_rules.len(), 1, "exactly one SSH rule");
let source = arg_after(ssh_rules[0], "--source-address-prefixes").unwrap();
assert!(
source.ends_with("/32") && !source.starts_with('*') && !source.starts_with("0.0.0.0"),
"SSH source must be a single caller address, got {source}"
);
assert_eq!(
arg_after(ssh_rules[0], "--nsg-name"),
arg_after(create, "--nsg"),
"the rule must land on the NSG the VM was created with"
);
}
#[test]
fn display_command_redacts_secret_args() {
let mut step =
+42 -1
View File
@@ -355,7 +355,14 @@ fn classify(
) -> BillingPresentation {
match provider {
ApiProvider::Ollama | ApiProvider::Sglang | ApiProvider::Vllm => BillingPresentation::Local,
ApiProvider::OpenaiCodex => BillingPresentation::Subscription("Codex OAuth quota"),
// The Codex OAuth quota is a fact of the ChatGPT backend, not of the
// provider name: a custom `[providers.openai_codex] base_url` (a
// gateway, a proxy) sells its own terms, so it is Unknown rather than
// a subscription that would hide metered spend.
ApiProvider::OpenaiCodex if is_chatgpt_codex_backend(base_url) => {
BillingPresentation::Subscription("Codex OAuth quota")
}
ApiProvider::OpenaiCodex => BillingPresentation::Unknown,
ApiProvider::OpencodeGo => BillingPresentation::Subscription("OpenCode Go quota"),
// StepFun already reduces an endpoint to a non-secret billing surface
// and fails closed on anything it does not recognize.
@@ -608,6 +615,20 @@ fn stepfun_billing_for_endpoint(base_url: Option<&str>) -> BillingPresentation {
}
}
/// The ChatGPT backend the Codex OAuth route ships with
/// (`https://chatgpt.com/backend-api`, or a path under it).
fn is_chatgpt_codex_backend(base_url: &str) -> bool {
let Ok(url) = reqwest::Url::parse(base_url.trim()) else {
return false;
};
let path = url.path().trim_end_matches('/');
url.scheme() == "https"
&& url.host_str() == Some("chatgpt.com")
&& url.port().is_none()
&& url.username().is_empty()
&& (path == "/backend-api" || path.starts_with("/backend-api/"))
}
fn is_zai_coding_plan_endpoint(base_url: &str) -> bool {
base_url
.trim()
@@ -2401,6 +2422,26 @@ mod tests {
),
BillingPresentation::Subscription("Codex OAuth quota")
);
// The OAuth token pointed anywhere else proves no Codex quota.
for elsewhere in [
"https://codex-gateway.example.com/backend-api",
"https://chatgpt.com.example.net/backend-api",
"http://chatgpt.com/backend-api",
"https://chatgpt.com/v1",
"",
] {
assert_eq!(
for_dispatched_route(
&config,
DispatchedRoute {
provider: ApiProvider::OpenaiCodex,
base_url: elsewhere,
},
),
BillingPresentation::Unknown,
"{elsewhere:?}"
);
}
}
#[test]
+9 -1
View File
@@ -105,10 +105,18 @@ pub(super) struct WorkspaceGitMetadata {
pub(super) dirty: bool,
}
/// Status is several blocking `git` spawns, so it runs off the async workers
/// (#6149) like every other git read on this surface. It deliberately uses
/// normal Git (filters, untracked settings) rather than the review command,
/// so its counts match `git status` and the operator writes (see `git.rs`).
pub(super) async fn workspace_status(
State(state): State<RuntimeApiState>,
) -> Result<Json<WorkspaceStatusResponse>, ApiError> {
Ok(Json(collect_workspace_status(&state.workspace)))
let workspace = state.workspace.clone();
tokio::task::spawn_blocking(move || collect_workspace_status(&workspace))
.await
.map(Json)
.map_err(|_| ApiError::internal("workspace status failed"))
}
pub(super) fn collect_workspace_status(workspace: &FsPath) -> WorkspaceStatusResponse {
+59 -2
View File
@@ -648,12 +648,22 @@ impl ScorecardMetrics {
/// Flag metrics that grew more than `threshold_pct` over `baseline`. Cost
/// and token counts are "lower is better", so only *increases* are
/// regressions. (Cache-hit ratio is the opposite, reported separately.)
///
/// Fails closed on values that cannot be compared: a non-finite threshold
/// admits no growth at all (it is treated as 0%), and a metric whose
/// change is `NaN` counts as regressed. Every `NaN` comparison is false,
/// so without this a `NaN` threshold would pass any run.
#[must_use]
pub fn regressions_against(
&self,
baseline: &ScorecardMetrics,
threshold_pct: f64,
) -> Vec<Regression> {
let threshold_pct = if threshold_pct.is_finite() {
threshold_pct
} else {
0.0
};
let mut out = Vec::new();
// A partial/unknown subtotal is not comparable to a complete baseline,
// but losing completeness is itself a regression. Otherwise removing
@@ -711,7 +721,7 @@ impl ScorecardMetrics {
let drop_pct = (baseline.cache_hit_ratio - self.cache_hit_ratio)
/ baseline.cache_hit_ratio
* 100.0;
if drop_pct > threshold_pct {
if drop_pct.is_nan() || drop_pct > threshold_pct {
out.push(Regression {
metric: "cache_hit_ratio_drop".to_string(),
baseline: baseline.cache_hit_ratio,
@@ -733,7 +743,7 @@ fn push_regression(
) {
if base > 0.0 {
let pct = (cur - base) / base * 100.0;
if pct > threshold_pct {
if pct.is_nan() || pct > threshold_pct {
out.push(Regression {
metric: metric.to_string(),
baseline: base,
@@ -1908,6 +1918,53 @@ mod tests {
assert!(!names.contains(&"total_input_tokens")); // under threshold
}
#[test]
fn a_non_finite_threshold_cannot_pass_a_regressed_run() {
let baseline = ScorecardMetrics {
turns: 1,
money_metered_turns: 1,
unpriced_turns: 0,
cny_unpriced_turns: 0,
cost_complete: true,
cny_cost_complete: true,
unpriced_classes: Vec::new(),
total_input_tokens: 1000,
total_output_tokens: 1000,
total_cache_read_tokens: 0,
total_cache_write_tokens: 0,
total_reasoning_tokens: 0,
total_cost_usd: 0.10,
total_cost_cny: 0.7,
cache_hit_ratio: 0.5,
};
let current = ScorecardMetrics {
total_output_tokens: 2000,
..baseline.clone()
};
for threshold in [f64::NAN, f64::INFINITY] {
let names: Vec<String> = current
.regressions_against(&baseline, threshold)
.into_iter()
.map(|r| r.metric)
.collect();
assert!(
names.iter().any(|name| name == "total_output_tokens"),
"threshold {threshold} passed a +100% regression: {names:?}"
);
}
// A metric whose change cannot be computed is not a pass either.
let unknown = ScorecardMetrics {
cache_hit_ratio: f64::NAN,
..baseline.clone()
};
assert!(
unknown
.regressions_against(&baseline, 5.0)
.iter()
.any(|r| r.metric == "cache_hit_ratio_drop")
);
}
#[test]
fn regression_flags_loss_of_cost_completeness_without_comparing_subtotals() {
let baseline = ScorecardMetrics {
+556 -16
View File
@@ -44,7 +44,7 @@ const MAX_EMPTY_SESSION_STUBS: usize = 10;
/// Maximum session title length, in `char`s. Matches the bound the session
/// picker's rename prompt has always enforced.
pub const MAX_SESSION_TITLE_CHARS: usize = 100;
const WORK_GRAPH_IMPORT_ARCHIVE_DIR: &str = ".work-graph-import-archive";
pub(crate) const WORK_GRAPH_IMPORT_ARCHIVE_DIR: &str = ".work-graph-import-archive";
const SESSION_GOALS_DIR: &str = ".goals";
const CURRENT_SESSION_GOAL_SCHEMA_VERSION: u32 = 2;
const MAX_SESSION_GOAL_OBJECTIVE_CHARS: usize = 8_192;
@@ -445,6 +445,47 @@ pub fn set_live_session(session_id: Option<&str>) {
static LIVE_SESSION_LEASE: std::sync::Mutex<Option<(String, Option<fs::File>)>> =
std::sync::Mutex::new(None);
/// Lock attempts [`SessionManager::reserve_session_for_attach`] makes
/// before it treats contention as a running owner.
const LIVE_LEASE_ATTACH_ATTEMPTS: u32 = 3;
/// The refusal for attaching to a session another process has open.
pub(crate) fn session_open_elsewhere(session_id: &str) -> std::io::Error {
std::io::Error::new(
std::io::ErrorKind::ResourceBusy,
format!(
"session {session_id} is open in another Codewhale window. \
Continue it there, or run `codewhale fork {session_id}` to work on a copy"
),
)
}
/// A reservation of a session's cross-process live lease, taken by
/// [`SessionManager::reserve_session_for_attach`] before a surface attaches.
/// Dropping it releases the reservation; [`Self::commit`] makes the session
/// this process's live session and releases the previous one's lease.
#[must_use = "an uncommitted reservation is released when dropped"]
#[derive(Debug)]
pub struct SessionLease {
id: String,
/// `None` when this process already holds the session's lease.
file: Option<fs::File>,
}
impl SessionLease {
/// Make the reserved session this process's live session. Called after
/// the attach has been validated and applied, so a failed attach never
/// costs the session this process already owns its lease.
pub fn commit(self) {
set_live_session(Some(&self.id));
if let Some(file) = self.file
&& let Ok(mut lease) = LIVE_SESSION_LEASE.lock()
{
*lease = Some((self.id, Some(file)));
}
}
}
/// Is this session currently owned by **this process's** interactive surface?
///
/// The registry is process-local. Reclamation must not treat a missing entry
@@ -1503,19 +1544,26 @@ impl SessionManager {
let Ok(mut lease) = LIVE_SESSION_LEASE.lock() else {
return;
};
if lease.as_ref().is_some_and(|(held, _)| held == id) {
return;
}
// A lease already held is kept. One that could not be taken is tried
// again: another process's liveness probe briefly holds the same lock,
// and a claim that lost that race once must not run unleased for the
// rest of the session.
let retry = match lease.as_ref() {
Some((held, Some(_))) if held == id => return,
Some((held, None)) => held == id,
_ => false,
};
let file = self.live_lease_path(id, true).and_then(|path| {
let file = open_private_lock_file(&path)?;
Ok(crate::runtime_threads::try_lock_file_exclusive(&file)?.then_some(file))
});
match &file {
Ok(Some(_)) => {}
Ok(None) => tracing::warn!(
Ok(None) if !retry => tracing::warn!(
session_id = id,
"another Codewhale process already holds this session open"
),
Ok(None) => {}
Err(error) => {
tracing::debug!(session_id = id, %error, "session live lease unavailable");
}
@@ -1523,6 +1571,59 @@ impl SessionManager {
*lease = Some((id.to_string(), file.ok().flatten()));
}
/// Reserve `session_id` before attaching to it (`resume`, `--continue`,
/// `exec --resume`, the session picker, `/load`). Unlike
/// [`Self::claim_live_session`], which records whatever it gets, this
/// refuses with `ResourceBusy` when another process holds the session
/// open: attaching anyway gives the document two autosaving writers, and
/// the last save silently drops the other's turns. Taking the lock is the
/// check, so nothing can claim the session between a check and the attach.
///
/// The reservation does not touch the session this process owns now: that
/// claim and its lease stay until [`SessionLease::commit`], which the
/// caller runs only once the new session is loaded and applied. Dropping
/// the reservation (a load or apply that failed) releases it and leaves
/// the current session leased.
///
/// It fails closed: a lease that cannot be opened or locked is an error,
/// not an unguarded attach.
pub fn reserve_session_for_attach(&self, session_id: &str) -> io::Result<SessionLease> {
let id = self.validated_session_id(session_id)?;
if LIVE_SESSION_LEASE
.lock()
.is_ok_and(|lease| matches!(lease.as_ref(), Some((held, Some(_))) if held == id))
{
return Ok(SessionLease {
id: id.to_string(),
file: None,
});
}
let lease_unavailable = |error: io::Error| {
io::Error::new(
error.kind(),
format!("could not take session {id}'s live lease: {error}"),
)
};
let file = self
.live_lease_path(id, true)
.and_then(|path| open_private_lock_file(&path))
.map_err(lease_unavailable)?;
// A liveness probe from another process holds this lock for a moment;
// a few short retries tell that apart from a running owner.
for attempt in 0..LIVE_LEASE_ATTACH_ATTEMPTS {
if crate::runtime_threads::try_lock_file_exclusive(&file).map_err(lease_unavailable)? {
return Ok(SessionLease {
id: id.to_string(),
file: Some(file),
});
}
if attempt + 1 < LIVE_LEASE_ATTACH_ATTEMPTS {
std::thread::sleep(std::time::Duration::from_millis(10));
}
}
Err(session_open_elsewhere(id))
}
/// Is `session_id` open in an interactive session in this process *or any
/// other*? External writers (the Runtime API, retention) check this before
/// rewriting or deleting a document, because the process holding it would
@@ -1547,6 +1648,16 @@ impl SessionManager {
)
}
/// Hold `session_id`'s live lease the way another running Codewhale
/// process does: an OS lock on its own open file description.
#[cfg(test)]
pub(crate) fn hold_live_lease_elsewhere(&self, session_id: &str) -> fs::File {
let path = self.live_lease_path(session_id, true).expect("lease path");
let lease = open_private_lock_file(&path).expect("open lease");
assert!(crate::runtime_threads::try_lock_file_exclusive(&lease).expect("lock lease"));
lease
}
/// Whether a saved document exists for `session_id`.
#[must_use]
pub fn session_document_exists(&self, session_id: &str) -> bool {
@@ -2270,7 +2381,7 @@ impl SessionManager {
self.read_checkpoint_file(&path)
}
fn legacy_checkpoint_origin(&self) -> io::Result<Option<String>> {
pub(crate) fn legacy_checkpoint_origin(&self) -> io::Result<Option<String>> {
use std::io::Read as _;
let path = self.checkpoints_dir().join(LEGACY_CHECKPOINT_FILE);
@@ -2384,7 +2495,12 @@ impl SessionManager {
let CheckpointSource::Session(id) = checkpoint.source else {
continue;
};
if Some(id.as_str()) == exclude || !self.session_from_prior_instance(&id) {
// A checkpoint another running session is still refreshing is
// that session's in-flight work, not a prior crash.
if Some(id.as_str()) == exclude
|| !self.session_from_prior_instance(&id)
|| self.is_session_live_anywhere(&id)
{
continue;
}
// One malformed id or unreadable record must not hide a later
@@ -2640,6 +2756,49 @@ impl SessionManager {
Ok(recovery)
}
/// [`Self::resume_session`] for a surface that will own and autosave the
/// session: it first reserves the session's live lease, and refuses with
/// `ResourceBusy` when another process has it open
/// ([`Self::reserve_session_for_attach`]). The caller commits the
/// returned lease once the session is applied.
pub fn attach_session(&self, id: &str) -> std::io::Result<(SessionRecovery, SessionLease)> {
let lease = self.reserve_session_for_attach(id)?;
let recovery = self.resume_session(id)?;
Ok((recovery, lease))
}
/// [`Self::attach_session`] with a partial-ID prefix.
pub fn attach_session_by_prefix(
&self,
prefix: &str,
) -> std::io::Result<(SessionRecovery, SessionLease)> {
self.attach_session(&self.resolve_session_id_prefix(prefix)?)
}
/// [`Self::attach_session`] for a session file `/load` has already read.
/// Either way the surface becomes the writer of the id the file names, so
/// a managed record and a foreign file take the same contract: the id's
/// live lease is reserved first and refused while another window has it
/// open. A managed record then resumes through the manager so its repair
/// is persisted in place; a foreign file is not ours to rewrite, so its
/// journal projection and repair stay in memory.
pub fn attach_session_file(
&self,
parsed: SavedSession,
path: &Path,
) -> std::io::Result<(SavedSession, SessionLease)> {
if self.owns_session_path(&parsed.metadata.id, path) {
return self
.attach_session(&parsed.metadata.id)
.map(|(recovery, lease)| (recovery.session, lease));
}
let lease = self.reserve_session_for_attach(&parsed.metadata.id)?;
let mut session = parsed;
session.ensure_journal();
repair_recovered_session(&mut session);
Ok((session, lease))
}
/// [`Self::resume_session`] with a partial-ID prefix.
pub fn resume_session_by_prefix(&self, prefix: &str) -> std::io::Result<SessionRecovery> {
self.resume_session(&self.resolve_session_id_prefix(prefix)?)
@@ -2867,6 +3026,36 @@ impl SessionManager {
self.remove_session(id, SessionRemoval::Explicit)
}
/// Remove the pre-import copy of `session_path` that the first
/// graph-bearing write kept (see `archive_before_first_graph_write`): it is
/// the same transcript, so a deleted session must not survive in it.
///
/// A linked archive directory is not followed, as the late-usage store
/// refuses one: deletion must not reach a same-named file elsewhere.
fn remove_import_archive_copy(&self, session_path: &Path) -> io::Result<()> {
let Some(name) = session_path.file_name() else {
return Ok(());
};
let dir = self.sessions_dir.join(WORK_GRAPH_IMPORT_ARCHIVE_DIR);
match fs::symlink_metadata(&dir) {
Ok(metadata) if crate::plugins::metadata_is_link_or_reparse(&metadata) => {
tracing::warn!(
path = %dir.display(),
"work-graph import archive is a link; its copies were not removed"
);
return Ok(());
}
Ok(_) => {}
Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(()),
Err(error) => return Err(error),
}
match fs::remove_file(dir.join(name)) {
Ok(()) => Ok(()),
Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(()),
Err(error) => Err(error),
}
}
fn remove_session(&self, id: &str, removal: SessionRemoval) -> std::io::Result<()> {
let path = self.validated_session_path(id)?;
// Older ordinary snapshots may use a name reserved by the checkpoint
@@ -2901,11 +3090,13 @@ impl SessionManager {
// Retention owns the ordinary snapshot, not crash recovery. Keep
// the origin and its accounting/evidence writable for resume.
// An unreadable legacy origin cannot justify retiring any id.
return match fs::remove_file(&path) {
Ok(()) => Ok(()),
Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(()),
Err(error) => Err(error),
};
match fs::remove_file(&path) {
Ok(()) => {}
Err(error) if error.kind() == io::ErrorKind::NotFound => {}
Err(error) => return Err(error),
}
// The pre-import copy is an older ordinary snapshot, not recovery.
return self.remove_import_archive_copy(&path);
}
// The store this document is bound to is named by its binding, not by
// its id: a host store usually sits under another conversation's
@@ -2944,6 +3135,7 @@ impl SessionManager {
Err(error) => return Err(error),
}
}
self.remove_import_archive_copy(&path)?;
self.clear_session_boot_owner(id);
let session_dir = self.sessions_dir.join(id.trim());
if session_dir.exists() {
@@ -3755,6 +3947,25 @@ fn json_object_end(bytes: &[u8], start: usize) -> Option<usize> {
None
}
/// The longest valid UTF-8 prefix of `buf`. A fixed-size read prefix can end
/// inside a multi-byte character (routine for CJK or emoji transcripts, which
/// serde writes unescaped); rejecting the whole buffer then forced a full-file
/// read of every such session on each listing.
///
/// Only a character cut at the very end is trimmed. An invalid byte anywhere
/// else means a damaged file, and an empty prefix sends it down the full read,
/// which reports it.
fn utf8_prefix(buf: &[u8]) -> &str {
match std::str::from_utf8(buf) {
Ok(s) => s,
// `valid_up_to` marks a char boundary, so this slice is valid.
Err(error) if error.error_len().is_none() => {
std::str::from_utf8(&buf[..error.valid_up_to()]).unwrap_or_default()
}
Err(_) => "",
}
}
/// String-scan a JSON byte buffer for the top-level `"metadata":{...}`
/// block and return it parsed. Returns `None` if no balanced metadata
/// object is present in the buffer.
@@ -3764,7 +3975,7 @@ fn json_object_end(bytes: &[u8], start: usize) -> Option<usize> {
/// `}` appearing inside a string literal doesn't perturb the depth
/// count.
fn extract_top_level_metadata(buf: &[u8]) -> Option<SessionMetadata> {
let s = std::str::from_utf8(buf).ok()?;
let s = utf8_prefix(buf);
let bytes = s.as_bytes();
const KEY: &[u8] = b"\"metadata\"";
let start = json_value_start(bytes, find_json_key(bytes, KEY)?, KEY.len(), b'{')?;
@@ -3776,9 +3987,7 @@ fn extract_top_level_metadata(buf: &[u8]) -> Option<SessionMetadata> {
/// whether the array was seen to end. A message the prefix cut in half is
/// simply absent; nothing is reconstructed.
fn extract_leading_messages(buf: &[u8], max: usize) -> (Vec<Message>, bool) {
let Ok(s) = std::str::from_utf8(buf) else {
return (Vec::new(), false);
};
let s = utf8_prefix(buf);
let bytes = s.as_bytes();
const KEY: &[u8] = b"\"messages\"";
let Some(key_offset) = find_json_key(bytes, KEY) else {
@@ -6640,6 +6849,103 @@ mod tests {
assert!(!sessions_dir.join(&session_id).exists());
}
#[test]
fn delete_session_removes_its_work_graph_import_archive() {
let tmp = tempdir().expect("tempdir");
let sessions_dir = tmp.path().join("sessions");
let manager = SessionManager::new(sessions_dir.clone()).expect("new");
let session = create_saved_session(
&[make_test_message("user", "archived transcript")],
"test-model",
tmp.path(),
0,
None,
);
let session_id = session.metadata.id.clone();
let path = manager.save_session(&session).expect("save");
let archive_dir = sessions_dir.join(WORK_GRAPH_IMPORT_ARCHIVE_DIR);
fs::create_dir_all(&archive_dir).expect("archive dir");
let archive = archive_dir.join(path.file_name().expect("file name"));
fs::copy(&path, &archive).expect("archive copy");
let other = archive_dir.join("other-session.json");
fs::write(&other, "{}").expect("other archive");
manager.delete_session(&session_id).expect("delete");
assert!(
!archive.exists(),
"the deleted transcript's copy is removed"
);
assert!(other.exists(), "other sessions' copies are untouched");
}
/// Retention keeps a session's crash-recovery checkpoint but retires the
/// ordinary snapshot, and the pre-import copy is an ordinary snapshot.
#[test]
fn retention_removes_the_import_archive_copy_of_a_recoverable_session() {
let tmp = tempdir().expect("tempdir");
let sessions_dir = tmp.path().join("sessions");
let manager = SessionManager::new(sessions_dir.clone()).expect("new");
let session = create_saved_session(
&[make_test_message("user", "archived transcript")],
"test-model",
tmp.path(),
0,
None,
);
let session_id = session.metadata.id.clone();
let path = manager.save_session(&session).expect("save");
manager.save_checkpoint(&session).expect("checkpoint");
let archive_dir = sessions_dir.join(WORK_GRAPH_IMPORT_ARCHIVE_DIR);
fs::create_dir_all(&archive_dir).expect("archive dir");
let archive = archive_dir.join(path.file_name().expect("file name"));
fs::copy(&path, &archive).expect("archive copy");
manager
.remove_session(&session_id, SessionRemoval::Retention)
.expect("retention");
assert!(!path.exists(), "the ordinary snapshot is retired");
assert!(!archive.exists(), "so is its pre-import copy");
assert!(
manager
.load_session_checkpoint(&session_id)
.expect("checkpoint")
.is_some(),
"crash recovery is kept"
);
}
/// Deletion does not follow a linked archive directory to a same-named
/// file somewhere else.
#[cfg(unix)]
#[test]
fn delete_does_not_follow_a_linked_import_archive() {
let tmp = tempdir().expect("tempdir");
let sessions_dir = tmp.path().join("sessions");
let manager = SessionManager::new(sessions_dir.clone()).expect("new");
let session = create_saved_session(
&[make_test_message("user", "transcript")],
"test-model",
tmp.path(),
0,
None,
);
let session_id = session.metadata.id.clone();
let path = manager.save_session(&session).expect("save");
let elsewhere = tmp.path().join("elsewhere");
fs::create_dir_all(&elsewhere).expect("elsewhere");
let unrelated = elsewhere.join(path.file_name().expect("file name"));
fs::write(&unrelated, "{}").expect("unrelated file");
std::os::unix::fs::symlink(&elsewhere, sessions_dir.join(WORK_GRAPH_IMPORT_ARCHIVE_DIR))
.expect("link");
manager.delete_session(&session_id).expect("delete");
assert!(!path.exists());
assert!(unrelated.exists(), "a file behind the link is untouched");
}
#[test]
fn test_session_id_rejects_invalid_characters() {
let tmp = tempdir().expect("tempdir");
@@ -7419,6 +7725,205 @@ mod tests {
);
}
fn hold_live_lease(manager: &SessionManager, id: &str) -> fs::File {
manager.hold_live_lease_elsewhere(id)
}
#[test]
fn interrupted_workspace_session_skips_a_session_live_elsewhere() {
let tmp = tempdir().expect("tempdir");
let manager = SessionManager::new(tmp.path().join("sessions")).expect("manager");
let workspace = tmp.path().join("ws");
fs::create_dir_all(&workspace).expect("workspace");
write_prior_interrupted_session(&manager, "sess-crashed", &workspace);
std::thread::sleep(std::time::Duration::from_millis(20));
write_prior_interrupted_session(&manager, "sess-running", &workspace);
let lease = hold_live_lease(&manager, "sess-running");
assert_eq!(
manager
.interrupted_workspace_session(&workspace, None)
.map(|meta| meta.id),
Some("sess-crashed".to_string()),
"a checkpoint another terminal is refreshing is not a crash"
);
drop(lease);
assert_eq!(
manager
.interrupted_workspace_session(&workspace, None)
.map(|meta| meta.id),
Some("sess-running".to_string())
);
}
/// Attaching to a session another process has open is refused, naming
/// it, rather than giving the document a second autosaving writer.
#[test]
fn attach_refuses_a_session_open_in_another_process() {
let _env = crate::test_support::lock_test_env();
let tmp = tempdir().expect("tempdir");
let manager = SessionManager::new(tmp.path().join("sessions")).expect("manager");
let workspace = tmp.path().join("ws");
fs::create_dir_all(&workspace).expect("workspace");
write_prior_interrupted_session(&manager, "sess-open", &workspace);
let lease = hold_live_lease(&manager, "sess-open");
let error = manager
.reserve_session_for_attach("sess-open")
.expect_err("a session open elsewhere is refused");
assert_eq!(error.kind(), io::ErrorKind::ResourceBusy);
assert!(error.to_string().contains("sess-open"), "{error}");
assert!(!is_live_session("sess-open"), "nothing was claimed");
drop(lease);
let (_, reserved) = manager
.attach_session("sess-open")
.expect("attach once the other window has closed");
assert!(
!try_lock_elsewhere(&manager, "sess-open"),
"the attach holds the lease from the start"
);
reserved.commit();
assert!(is_live_session("sess-open"));
assert!(!try_lock_elsewhere(&manager, "sess-open"));
set_live_session(None);
}
/// Switching from session A to B must not give up A until B is applied:
/// a reservation that is dropped (B failed to load or apply) leaves A
/// claimed and leased, and B free.
#[test]
fn a_failed_attach_keeps_the_current_sessions_lease() {
let _env = crate::test_support::lock_test_env();
let tmp = tempdir().expect("tempdir");
let manager = SessionManager::new(tmp.path().join("sessions")).expect("manager");
let workspace = tmp.path().join("ws");
fs::create_dir_all(&workspace).expect("workspace");
write_prior_interrupted_session(&manager, "sess-a", &workspace);
write_prior_interrupted_session(&manager, "sess-b", &workspace);
manager.claim_live_session("sess-a");
assert!(!try_lock_elsewhere(&manager, "sess-a"), "A is leased");
let reserved = manager
.reserve_session_for_attach("sess-b")
.expect("reserve B");
assert!(is_live_session("sess-a"), "A stays claimed while B loads");
assert!(!try_lock_elsewhere(&manager, "sess-a"), "and leased");
drop(reserved);
assert!(is_live_session("sess-a"));
assert!(!try_lock_elsewhere(&manager, "sess-a"), "A is still leased");
assert!(try_lock_elsewhere(&manager, "sess-b"), "B was released");
// A B whose document cannot be loaded is refused the same way.
let b_path = manager.validated_session_path("sess-b").expect("path");
fs::write(&b_path, "{ not json").expect("corrupt B");
manager
.attach_session("sess-b")
.expect_err("a corrupt B fails to attach");
assert!(is_live_session("sess-a"));
assert!(!try_lock_elsewhere(&manager, "sess-a"), "A is still leased");
assert!(try_lock_elsewhere(&manager, "sess-b"), "B was released");
set_live_session(None);
}
/// `/load` takes the same contract as `resume`: a managed record and a
/// foreign file naming a session open in another window are both
/// refused, and neither is claimed.
#[test]
fn load_of_a_session_open_elsewhere_is_refused_for_managed_and_foreign_files() {
let _env = crate::test_support::lock_test_env();
let tmp = tempdir().expect("tempdir");
let manager = SessionManager::new(tmp.path().join("sessions")).expect("manager");
let workspace = tmp.path().join("ws");
fs::create_dir_all(&workspace).expect("workspace");
let session = write_prior_interrupted_session(&manager, "sess-load", &workspace);
let managed = manager.validated_session_path("sess-load").expect("path");
let foreign = tmp.path().join("exported.json");
fs::write(&foreign, serde_json::to_vec(&session).expect("json")).expect("export");
let lease = hold_live_lease(&manager, "sess-load");
for path in [&managed, &foreign] {
let error = manager
.attach_session_file(session.clone(), path)
.expect_err("a session open elsewhere is not loaded");
assert_eq!(
error.kind(),
io::ErrorKind::ResourceBusy,
"{}",
path.display()
);
}
assert!(!is_live_session("sess-load"), "nothing was claimed");
drop(lease);
for path in [&managed, &foreign] {
let (loaded, reserved) = manager
.attach_session_file(session.clone(), path)
.expect("loads once the other window has closed");
assert_eq!(loaded.metadata.id, "sess-load");
assert!(
!try_lock_elsewhere(&manager, "sess-load"),
"the load holds the lease"
);
drop(reserved);
}
set_live_session(None);
}
/// A lease that cannot be taken is an error, not an unguarded attach.
#[cfg(unix)]
#[test]
fn attach_fails_closed_when_the_lease_cannot_be_taken() {
let _env = crate::test_support::lock_test_env();
let tmp = tempdir().expect("tempdir");
let sessions_dir = tmp.path().join("sessions");
let manager = SessionManager::new(sessions_dir.clone()).expect("manager");
let workspace = tmp.path().join("ws");
fs::create_dir_all(&workspace).expect("workspace");
write_prior_interrupted_session(&manager, "sess-x", &workspace);
// The lease directory is a regular file: no lock file can be opened.
let _ = fs::remove_dir_all(sessions_dir.join(LATE_USAGE_DIR));
fs::write(sessions_dir.join(LATE_USAGE_DIR), "").expect("block lease dir");
let error = manager
.attach_session("sess-x")
.expect_err("no lease, no attach");
assert!(error.to_string().contains("live lease"), "{error}");
assert!(!is_live_session("sess-x"), "nothing was claimed");
}
/// A liveness probe from another process briefly holds the lease lock. A
/// claim that lost that race once must take the lease on a later save,
/// not run unleased for the whole session.
#[test]
fn a_claim_that_lost_a_probe_race_takes_the_lease_later() {
let _env = crate::test_support::lock_test_env();
let tmp = tempdir().expect("tempdir");
let manager = SessionManager::new(tmp.path().join("sessions")).expect("manager");
let probe = hold_live_lease(&manager, "sess-a");
manager.claim_live_session("sess-a");
drop(probe);
assert!(
try_lock_elsewhere(&manager, "sess-a"),
"the first claim lost"
);
manager.claim_live_session("sess-a");
assert!(
!try_lock_elsewhere(&manager, "sess-a"),
"the next claim takes the lease"
);
set_live_session(None);
}
/// Whether another open file description can lock `id`'s lease now.
fn try_lock_elsewhere(manager: &SessionManager, id: &str) -> bool {
let lease = open_private_read_file(&manager.live_lease_path(id, false).expect("path"))
.expect("lease file");
crate::runtime_threads::try_lock_file_exclusive(&lease).expect("lock")
}
#[test]
fn interrupted_workspace_session_ignores_settled_sessions() {
let tmp = tempdir().expect("tempdir");
@@ -7839,6 +8344,41 @@ mod tests {
assert_eq!(extracted.total_tokens, 4096);
}
/// A 64 KB read prefix routinely ends inside a multi-byte character in a
/// CJK or emoji transcript. The metadata ahead of the cut is intact and
/// must still be read from the prefix, not force a full-file read.
#[test]
fn extract_top_level_metadata_survives_a_prefix_cut_inside_a_character() {
let json = format!(
r#"{{"schema_version":1,"metadata":{{"id":"cjk-1","title":"会话","created_at":"2026-01-01T00:00:00Z","updated_at":"2026-01-02T00:00:00Z","message_count":3,"total_tokens":10,"model":"m","workspace":"/tmp"}},"messages":[{{"role":"user","content":"{}"}}]}}"#,
"中文".repeat(100)
);
let bytes = json.as_bytes();
let body_start = json.find("中文").expect("body");
// One byte into a three-byte character.
let cut = &bytes[..body_start + 1];
assert!(
std::str::from_utf8(cut).is_err(),
"the cut splits a character"
);
let extracted = extract_top_level_metadata(cut).expect("metadata from the cut prefix");
assert_eq!(extracted.id, "cjk-1");
assert_eq!(extracted.title, "会话");
}
/// Only a cut at the end of the prefix is trimmed. An invalid byte ahead
/// of it is a damaged file, left to the full read to report.
#[test]
fn extract_top_level_metadata_rejects_invalid_utf8_inside_the_prefix() {
let json = r#"{"schema_version":1,"metadata":{"id":"bad-1","title":"t","created_at":"2026-01-01T00:00:00Z","updated_at":"2026-01-02T00:00:00Z","message_count":3,"total_tokens":10,"model":"m","workspace":"/tmp"},"messages":[{"role":"user","content":"AB"}]}"#;
let mut bytes = json.as_bytes().to_vec();
let body = json.find("AB").expect("body");
bytes[body] = 0xFF;
assert!(extract_top_level_metadata(&bytes).is_none());
}
#[test]
fn extract_top_level_metadata_handles_braces_inside_strings() {
// A title containing `{` and `}` inside the metadata block must
+96
View File
@@ -56,6 +56,15 @@ pub(crate) fn session_files(
if let Some(checkpoints) = canonical_store_root(&root.join("checkpoints"), unreadable) {
dirs.push(checkpoints);
}
// Pre-import transcript copies hold the same tool output. Session
// deletion does not follow a linked archive directory, so neither does
// the scrub: it is reported as not covered instead.
let archive = root.join(crate::session_manager::WORK_GRAPH_IMPORT_ARCHIVE_DIR);
match std::fs::symlink_metadata(&archive) {
Ok(metadata) if metadata.is_dir() => dirs.push(archive),
Err(error) if error.kind() == io::ErrorKind::NotFound => {}
_ => unreadable.push(archive),
}
for session in directory_entries(&root, unreadable) {
if !entry_has_type(&session, true, unreadable) {
continue;
@@ -181,6 +190,17 @@ pub(crate) fn configured_secrets(
))
}
/// Whether `path` sits directly in `manager`'s pre-import archive directory
/// (never a linked one; see [`session_files`]).
fn is_import_archive(path: &Path, manager: &crate::session_manager::SessionManager) -> bool {
let archive = manager
.sessions_dir()
.join(crate::session_manager::WORK_GRAPH_IMPORT_ARCHIVE_DIR);
std::fs::symlink_metadata(&archive).is_ok_and(|metadata| metadata.is_dir())
&& std::fs::canonicalize(&archive)
.is_ok_and(|archive| path.parent() == Some(archive.as_path()))
}
fn runtime_root(path: &Path) -> Option<&Path> {
let parent = path.parent()?;
matches!(parent.file_name()?.to_str()?, "items" | "events")
@@ -230,6 +250,18 @@ pub(crate) fn scrub_files(
.with_session_file_lock(id, || scrub_file(path, true, secrets).map(|_| ()))
}) {
Some(Ok(Some(()))) => FileScan::Dirty(redacted),
// A pre-import copy an earlier build left behind when it
// deleted the session. Finish that deletion: rewriting it
// would keep a deleted transcript, and could put back a
// copy a concurrent delete had just removed.
Some(Ok(None)) if is_import_archive(path, manager) => {
match std::fs::remove_file(path) {
Ok(()) => {}
Err(error) if error.kind() == io::ErrorKind::NotFound => {}
Err(error) => return Err(error),
}
FileScan::Dirty(redacted)
}
// A deleted session is not resurrected by a rewrite.
Some(Ok(None)) => FileScan::Clean,
// No lockable session id: leave the file untouched.
@@ -437,6 +469,70 @@ mod tests {
);
}
/// The pre-import copy of a transcript holds the same tool output. It is
/// scrubbed with the sessions. A copy an earlier build left behind for a
/// session it deleted is removed, finishing that deletion.
#[test]
fn work_graph_import_archive_copies_are_scrubbed() {
let dir = tempfile::tempdir().expect("tempdir");
let output = format!("{{\"access_token\": \"{TOKEN}\"}}");
let manager =
crate::session_manager::SessionManager::new(dir.path().to_path_buf()).expect("manager");
std::fs::write(
dir.path().join("gone.json"),
session_with_tool_output("nothing").to_string(),
)
.unwrap();
manager.delete_session("gone").expect("delete");
let archive_dir = dir
.path()
.join(crate::session_manager::WORK_GRAPH_IMPORT_ARCHIVE_DIR);
std::fs::create_dir_all(&archive_dir).unwrap();
let kept = archive_dir.join("kept.json");
let orphan = archive_dir.join("gone.json");
for path in [&kept, &orphan] {
std::fs::write(path, session_with_tool_output(&output).to_string()).unwrap();
}
let files = session_files(dir.path(), &dir.path().join("standalone"), &mut Vec::new());
let report = scrub_files(&files, Some(&manager), &[]).expect("scrub");
assert_eq!(report.flagged_files.len(), 2, "{report:?}");
let text = std::fs::read_to_string(&kept).unwrap();
assert!(!text.contains(TOKEN), "{text}");
assert!(!orphan.exists(), "the deleted session's copy is removed");
}
/// A linked archive directory is not followed, matching session deletion.
#[cfg(unix)]
#[test]
fn a_linked_import_archive_is_reported_not_followed() {
let dir = tempfile::tempdir().expect("tempdir");
let sessions = dir.path().join("sessions");
let elsewhere = dir.path().join("elsewhere");
std::fs::create_dir_all(&sessions).unwrap();
std::fs::create_dir_all(&elsewhere).unwrap();
let outside = elsewhere.join("gone.json");
let output = format!("{{\"access_token\": \"{TOKEN}\"}}");
std::fs::write(&outside, session_with_tool_output(&output).to_string()).unwrap();
std::os::unix::fs::symlink(
&elsewhere,
sessions.join(crate::session_manager::WORK_GRAPH_IMPORT_ARCHIVE_DIR),
)
.unwrap();
let mut unreadable = Vec::new();
let files = session_files(&sessions, &dir.path().join("standalone"), &mut unreadable);
assert!(
!files.iter().any(|file| file.ends_with("gone.json")),
"{files:?}"
);
assert_eq!(unreadable.len(), 1, "{unreadable:?}");
assert!(unreadable[0].ends_with(crate::session_manager::WORK_GRAPH_IMPORT_ARCHIVE_DIR));
assert!(std::fs::read_to_string(&outside).unwrap().contains(TOKEN));
}
#[test]
fn busy_runtime_does_not_abort_scrub() {
let dir = tempfile::tempdir().unwrap();
+16 -7
View File
@@ -32,14 +32,15 @@ pub fn tool_output_status(messages: &[Message], artifacts: &[ArtifactRecord]) ->
for message in messages {
for block in &message.content {
if let ContentBlock::ToolResult { content, .. } = block {
if looks_like_receipt(content) {
// The prefix is text any tool output can carry, so it never
// exempts a result from the size check: a "receipt" above the
// threshold is raw pressure like any other large result.
let chars = content.chars().count();
if chars > RAW_TOOL_OUTPUT_RECEIPT_THRESHOLD_CHARS {
status.raw_large_count += 1;
status.raw_large_chars = status.raw_large_chars.saturating_add(chars);
} else if looks_like_receipt(content) {
status.receipt_count += 1;
} else {
let chars = content.chars().count();
if chars > RAW_TOOL_OUTPUT_RECEIPT_THRESHOLD_CHARS {
status.raw_large_count += 1;
status.raw_large_chars = status.raw_large_chars.saturating_add(chars);
}
}
}
}
@@ -151,4 +152,12 @@ mod tests {
assert!(rendered.contains("compact receipt"));
assert!(rendered.contains("artifact"));
}
#[test]
fn a_receipt_prefix_does_not_hide_a_large_raw_result() {
let forged = format!("[TOOL_OUTPUT_RECEIPT]\n{}", "RAW\n".repeat(4_000));
let status = tool_output_status(&[tool_result_message("call-forged", &forged)], &[]);
assert_eq!(status.raw_large_count, 1);
assert_eq!(status.receipt_count, 0);
}
}
+3 -1
View File
@@ -11277,7 +11277,9 @@ async fn spawn_subagent_from_input(
)));
}
if let Some(remaining) = crate::retry_status::rate_limit_remaining() {
if let Some(remaining) =
crate::retry_status::rate_limit_remaining(&runtime.client.rate_limit_scope())
{
let seconds = remaining.as_secs() + u64::from(remaining.subsec_nanos() > 0);
return Err(ToolError::execution_failed(format!(
"Provider is rate-limiting; sub-agent spawning is paused for {seconds}s. \
+42 -4
View File
@@ -12605,11 +12605,14 @@ async fn rate_limit_pause_blocks_subagent_spawn() {
let _clear = ClearRateLimitOnDrop;
crate::retry_status::clear();
crate::retry_status::clear_rate_limit();
crate::retry_status::note_rate_limit(Duration::from_secs(30));
let tmp = tempdir().expect("tempdir");
let mut runtime = stub_runtime();
runtime.context = ToolContext::new(tmp.path().to_path_buf());
crate::retry_status::note_rate_limit(
&runtime.client.rate_limit_scope(),
Duration::from_secs(30),
);
let manager = new_shared_subagent_manager(tmp.path().to_path_buf(), 2);
let err = spawn_subagent_from_input(
@@ -18453,6 +18456,32 @@ async fn spawn_budget_capped_worker(
String,
Arc<AtomicUsize>,
tokio::task::JoinHandle<()>,
) {
spawn_budget_capped_worker_with_pause(
workspace,
prompt_tokens,
completion_tokens,
max_steps,
wall_time,
None,
)
.await
}
/// As [`spawn_budget_capped_worker`], optionally opening a rate-limit pause
/// on the worker's own route before it starts.
async fn spawn_budget_capped_worker_with_pause(
workspace: &Path,
prompt_tokens: u64,
completion_tokens: u64,
max_steps: u32,
wall_time: Duration,
pause: Option<Duration>,
) -> (
Arc<RwLock<SubAgentManager>>,
String,
Arc<AtomicUsize>,
tokio::task::JoinHandle<()>,
) {
let manager = Arc::new(RwLock::new(SubAgentManager::new(
workspace.to_path_buf(),
@@ -18502,6 +18531,9 @@ async fn spawn_budget_capped_worker(
launch_gate: None,
_foreground_child_registration: None,
};
if let Some(pause) = pause {
crate::retry_status::note_rate_limit(&runtime.client.rate_limit_scope(), pause);
}
let task_handle = tokio::spawn(run_subagent_task(task));
(manager, agent_id, calls, task_handle)
}
@@ -18781,11 +18813,17 @@ async fn worker_is_not_stranded_by_transient_global_rate_limit_window() {
// in-flight requests promptly.
let _guard = crate::retry_status::test_guard();
let _clear = ClearRateLimitOnDrop;
crate::retry_status::note_rate_limit(Duration::from_secs(30));
let tmp = tempdir().expect("tempdir");
let (manager, agent_id, _calls, task_handle) =
spawn_budget_capped_worker(tmp.path(), 60, 40, 4, DEFAULT_CHILD_WALL_TIME).await;
let (manager, agent_id, _calls, task_handle) = spawn_budget_capped_worker_with_pause(
tmp.path(),
60,
40,
4,
DEFAULT_CHILD_WALL_TIME,
Some(Duration::from_secs(30)),
)
.await;
// Simulate the concurrent test finishing: the window closes shortly
// after the worker's first request has already observed it.
+38 -1
View File
@@ -47,6 +47,10 @@ pub(crate) enum StatusToastKind {
BehavioralTip(crate::tui::behavioral_tips::BehavioralTip),
PluginSuggestion,
ContextPressure(crate::context_budget::PressureLevel),
/// A condition that holds until its owner retires it by event id (a
/// session save that keeps failing). It never times out: expiring it
/// would report recovery that has not happened.
Standing,
}
impl StatusToast {
@@ -67,6 +71,24 @@ impl StatusToast {
self
}
/// A notice for a standing condition, withdrawn only through
/// [`App::retire_event_notices`]`(event_id)` once the condition clears.
#[must_use]
pub(crate) fn standing(
text: impl Into<String>,
level: StatusToastLevel,
event_id: impl Into<String>,
) -> Self {
Self {
text: text.into(),
level,
created_at: Instant::now(),
ttl_ms: None,
kind: StatusToastKind::Standing,
event_id: Some(event_id.into()),
}
}
pub(crate) fn for_action(mut self, request_id: impl Into<String>) -> Self {
self.kind = StatusToastKind::ActionRequired;
self.event_id = Some(request_id.into());
@@ -130,8 +152,15 @@ impl App {
return;
}
self.status_toasts.push_back(toast);
// Overflow drops the oldest notice that is not a standing condition;
// those leave only when their condition clears.
while self.status_toasts.len() > 24 {
self.status_toasts.pop_front();
let oldest = self
.status_toasts
.iter()
.position(|toast| toast.kind != StatusToastKind::Standing)
.unwrap_or(0);
self.status_toasts.remove(oldest);
}
self.needs_redraw = true;
}
@@ -147,6 +176,14 @@ impl App {
self.needs_redraw |= self.status_toasts.len() != before;
}
/// Retire the notices pushed with [`StatusToast::for_event`]`(event_id)`.
pub(crate) fn retire_event_notices(&mut self, event_id: &str) {
let before = self.status_toasts.len();
self.status_toasts
.retain(|toast| toast.event_id.as_deref() != Some(event_id));
self.needs_redraw |= self.status_toasts.len() != before;
}
/// Gate transitions retire their own guidance or failed-write receipt,
/// independently of translated text and unrelated requests/errors.
pub(crate) fn retire_redaction_gate_notice(&mut self, notice: RedactionGateNotice) {
+34
View File
@@ -431,7 +431,11 @@ impl<'a> AutoReviewContext<'a> {
let tool_name = tool_name.to_owned();
let params = params.clone();
let workspace = workspace.map(std::path::Path::to_path_buf);
#[cfg(test)]
let env_ticket = crate::test_support::env_scope_ticket();
tokio::task::spawn_blocking(move || {
#[cfg(test)]
let _membership = crate::test_support::join_env_scope(env_ticket);
let trusted = workspace
.as_deref()
.is_some_and(crate::config::is_workspace_trusted);
@@ -2689,6 +2693,36 @@ mod tests {
)
}
#[tokio::test(flavor = "current_thread")]
async fn auto_review_worker_keeps_the_callers_sealed_environment() {
use crate::test_support::{EnvVarGuard, lock_test_env};
use std::time::Duration;
let _lock = lock_test_env();
let home = tempfile::tempdir().expect("test home");
let workspace = tempfile::tempdir().expect("workspace");
let _home = EnvVarGuard::set("CODEWHALE_HOME", home.path());
crate::config::save_workspace_trust(workspace.path()).expect("save trust");
let context = tokio::time::timeout(
Duration::from_secs(2),
AutoReviewContext::from_tool_call_async(
"read_file",
&json!({ "path": "README.md" }),
RunOrigin::Interactive,
ApprovalMode::Auto,
Some(workspace.path()),
),
)
.await
.expect("worker must not wait for its awaiting caller's environment lock")
.expect("evidence");
assert!(
context.workspace_trusted,
"worker must read the sealed trust file"
);
}
#[cfg(unix)]
#[tokio::test(flavor = "current_thread")]
async fn auto_review_filesystem_evidence_does_not_park_runtime() {
+145 -12
View File
@@ -157,6 +157,7 @@ struct SharedPending {
struct PersistRequestSender {
shared: Arc<std::sync::Mutex<SharedPending>>,
cmd_tx: mpsc::Sender<ActorCommand>,
health: SessionSaveHealth,
}
impl std::fmt::Debug for PersistRequestSender {
@@ -169,6 +170,7 @@ impl std::fmt::Debug for PersistRequestSender {
struct PersistRequestReceiver {
shared: Arc<std::sync::Mutex<SharedPending>>,
cmd_rx: mpsc::Receiver<ActorCommand>,
health: SessionSaveHealth,
}
/// Single construction seam for the production persistence request channel.
@@ -179,12 +181,18 @@ struct PersistRequestReceiver {
fn persistence_request_channel() -> (PersistRequestSender, PersistRequestReceiver) {
let (cmd_tx, cmd_rx) = mpsc::channel(ACTOR_COMMAND_CAPACITY);
let shared = Arc::new(std::sync::Mutex::new(SharedPending::default()));
let health = SessionSaveHealth::default();
(
PersistRequestSender {
shared: Arc::clone(&shared),
cmd_tx,
health: health.clone(),
},
PersistRequestReceiver {
shared,
cmd_rx,
health,
},
PersistRequestReceiver { shared, cmd_rx },
)
}
@@ -213,6 +221,11 @@ pub struct PersistActorHandle {
}
impl PersistActorHandle {
/// Whether the latest save of each session document landed.
pub(crate) fn session_save_health(&self) -> SaveHealthReading {
self.tx.health.reading()
}
/// Queue a persistence request without blocking. The request is
/// coalesced into the shared pending state immediately (latest-wins per
/// session), so repeated snapshots of one session retain only the
@@ -357,6 +370,12 @@ fn request_label(request: &PersistRequest) -> &'static str {
/// Queue persistence and report whether the actor accepted ownership. Work
/// Graph projections use this acknowledgement as their publish boundary.
/// [`PersistActorHandle::session_save_health`] of the global actor; `None`
/// before it starts.
pub(crate) fn session_save_health() -> Option<SaveHealthReading> {
ACTOR_TX.get().map(PersistActorHandle::session_save_health)
}
pub fn try_persist(request: PersistRequest) -> bool {
ACTOR_TX
.get()
@@ -390,8 +409,9 @@ pub fn spawn_persistence_actor(
manager: &SessionManager,
pending: &mut PendingState,
unreported: &mut FlushReport,
health: &SessionSaveHealth,
) {
let cycle = flush_inner(manager, pending);
let cycle = flush_inner(manager, pending, health);
log_flush_failures(&cycle);
unreported.merge(cycle);
}
@@ -404,15 +424,15 @@ pub fn spawn_persistence_actor(
match command {
ActorCommand::WorkReady => {
if !pending.is_empty() {
flush_cycle(&manager, &mut pending, &mut unreported);
flush_cycle(&manager, &mut pending, &mut unreported, &rx.health);
}
}
ActorCommand::FlushAndReport { reply } => {
flush_cycle(&manager, &mut pending, &mut unreported);
flush_cycle(&manager, &mut pending, &mut unreported, &rx.health);
let _ = reply.send(std::mem::take(&mut unreported));
}
ActorCommand::Shutdown => {
flush_cycle(&manager, &mut pending, &mut unreported);
flush_cycle(&manager, &mut pending, &mut unreported, &rx.health);
return;
}
}
@@ -420,7 +440,7 @@ pub fn spawn_persistence_actor(
// Every sender is gone — final flush and exit. Each absorb
// guarantees a queued WorkReady, so this is normally empty.
let mut pending = rx.take_pending();
flush_cycle(&manager, &mut pending, &mut unreported);
flush_cycle(&manager, &mut pending, &mut unreported, &rx.health);
},
);
@@ -561,7 +581,11 @@ impl PendingState {
/// commit clears its session's checkpoint only after that session's own
/// write succeeded, so a failed save always leaves the crash-recovery
/// checkpoint in place.
fn flush_inner(manager: &SessionManager, pending: &mut PendingState) -> FlushReport {
fn flush_inner(
manager: &SessionManager,
pending: &mut PendingState,
health: &SessionSaveHealth,
) -> FlushReport {
let mut report = FlushReport::default();
let mut record = |what: String, result: std::io::Result<()>| match result {
Ok(()) => report.completed += 1,
@@ -569,13 +593,13 @@ fn flush_inner(manager: &SessionManager, pending: &mut PendingState) -> FlushRep
};
for (session_id, session) in std::mem::take(&mut pending.sessions) {
record(
format!("session:{session_id}"),
manager.save_session_owned(session).map(|_| ()),
);
let result = manager.save_session_owned(session).map(|_| ());
health.record(&session_id, &result);
record(format!("session:{session_id}"), result);
}
for (session_id, session) in std::mem::take(&mut pending.completed_commits) {
let commit_result = manager.save_session_owned(session);
health.record(&session_id, &commit_result);
let save_succeeded = commit_result.is_ok();
record(
format!("completed-commit:{session_id}"),
@@ -620,6 +644,61 @@ fn flush_inner(manager: &SessionManager, pending: &mut PendingState) -> FlushRep
report
}
/// Which session documents' latest save failed. Only document saves count: a
/// failed checkpoint, queue or cleanup write does not lose the conversation.
/// A later successful save of the same session clears its entry, so a
/// transient failure does not leave a standing alarm.
#[derive(Debug, Clone, Default)]
pub(crate) struct SessionSaveHealth(Arc<std::sync::Mutex<SaveHealthState>>);
#[derive(Debug, Default)]
struct SaveHealthState {
/// Bumped whenever `failing` changes, so a poller can tell what is new.
generation: u64,
failing: BTreeMap<String, std::io::ErrorKind>,
}
/// One reading of [`SessionSaveHealth`].
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) struct SaveHealthReading {
pub(crate) generation: u64,
/// A session whose latest save failed, and how, while any has.
pub(crate) failing: Option<(String, std::io::ErrorKind)>,
}
impl SessionSaveHealth {
fn record<T>(&self, session_id: &str, result: &std::io::Result<T>) {
let mut state = self
.0
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner());
let changed = match result {
Ok(_) => state.failing.remove(session_id).is_some(),
Err(error) => {
state.failing.insert(session_id.to_string(), error.kind()) != Some(error.kind())
}
};
if changed {
state.generation += 1;
}
}
fn reading(&self) -> SaveHealthReading {
let state = self
.0
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner());
SaveHealthReading {
generation: state.generation,
failing: state
.failing
.iter()
.next()
.map(|(id, kind)| (id.clone(), *kind)),
}
}
}
/// Surface flush failures in the log for write cycles that have no caller
/// waiting on a [`FlushReport`].
fn log_flush_failures(report: &FlushReport) {
@@ -917,6 +996,60 @@ mod tests {
task.await.expect("persistence actor join");
}
/// The health the UI polls follows real saves through the actor: a failed
/// checkpoint does not count as a lost conversation, a failed document
/// save does, and a later successful save of that session clears it.
#[tokio::test]
async fn session_save_health_follows_document_saves_through_the_actor() {
let tmp = tempfile::tempdir().expect("tempdir");
let sessions_dir = tmp.path().join("sessions");
let manager = SessionManager::new(sessions_dir.clone()).expect("manager");
std::fs::write(sessions_dir.join("checkpoints"), b"not a directory")
.expect("block checkpoints dir");
let session = crate::session_manager::create_saved_session_with_mode(
&[],
"deepseek-v4-pro",
tmp.path(),
0,
None,
Some("agent"),
);
let session_id = session.metadata.id.clone();
// A directory where the document goes makes its save fail.
let blocker = sessions_dir.join(format!("{session_id}.json"));
std::fs::create_dir(&blocker).expect("block session document");
let (handle, task) = spawn_persistence_actor(manager);
let flush = |handle: &PersistActorHandle| {
let (reply_tx, reply_rx) = tokio::sync::oneshot::channel();
handle.try_send(PersistRequest::FlushAndReport { reply: reply_tx });
reply_rx
};
handle.try_send(PersistRequest::SaveCheckpoint {
session: session.clone(),
});
flush(&handle).await.expect("flush");
assert_eq!(handle.session_save_health().failing, None);
handle.try_send(PersistRequest::SessionSnapshot(session.clone()));
flush(&handle).await.expect("flush");
let failed = handle.session_save_health();
assert_eq!(
failed.failing.as_ref().map(|(id, _)| id.as_str()),
Some(session_id.as_str())
);
std::fs::remove_dir(&blocker).expect("unblock");
handle.try_send(PersistRequest::SessionSnapshot(session));
flush(&handle).await.expect("flush");
let healed = handle.session_save_health();
assert_eq!(healed.failing, None, "a later save clears the failure");
assert!(healed.generation > failed.generation);
handle.try_send(PersistRequest::Shutdown);
task.await.expect("persistence actor join");
}
#[tokio::test]
async fn flush_and_report_propagates_write_failures() {
let tmp = tempfile::tempdir().expect("tempdir");
@@ -1227,7 +1360,7 @@ mod tests {
});
drop(lease); // The old window changed session before the actor ran.
assert!(manager.acquire_offline_queue_lease("session-A").is_err());
let report = flush_inner(&manager, &mut pending);
let report = flush_inner(&manager, &mut pending, &SessionSaveHealth::default());
assert!(report.failures.is_empty(), "draft write failed: {report:?}");
assert_eq!(report.completed, 1);
let _next_editor = manager
+9 -1
View File
@@ -90,7 +90,12 @@ class PetNative {
this.world.voices = [];
return this.world.frame.timeMs;
}
observeEngine(metadataJSON, timeMs) { this.engine.observe(JSON.parse(metadataJSON), timeMs); }
/** A rejected observation leaves the reducer exactly as it was, like a batch. */
observeEngine(metadataJSON, timeMs) {
const next = this.engine.clone();
next.observe(JSON.parse(metadataJSON), timeMs);
this.engine = next;
}
observeEngineBatch(metadataJSON, timeMs) {
const events = JSON.parse(metadataJSON);
if (!Array.isArray(events) || events.length > 64)
@@ -1846,6 +1851,9 @@ class PetEngineTelemetry {
next.completedTurns = new Set(this.completedTurns);
return next;
}
/** Mutates in place: clock, freshness and the event window change before
* type-specific fields are checked. Callers that continue after a throw must
* observe on a clone() and swap it in on success, as PetNative does. */
observe(value, at) {
if (!Number.isFinite(at) || at < this.lastTime || at > pet_sim_js_1.PET_MAX_SECONDS * 1000)
throw new Error('Invalid Engine pet clock.');
+66
View File
@@ -510,6 +510,19 @@ impl SessionPickerView {
fn delete_selected(&mut self) -> Option<ViewEvent> {
let session = self.selected_session().cloned()?;
let manager = SessionManager::default_location().ok()?;
// A deleted document the live session still owns refuses every later
// save, so the conversation on screen would silently stop persisting.
let refusal = if self.current_session_id.as_deref() == Some(session.id.as_str()) {
Some(MessageId::SessionsDeleteOpenHere)
} else if manager.is_session_live_anywhere(&session.id) {
Some(MessageId::SessionsDeleteOpenElsewhere)
} else {
None
};
if let Some(reason) = refusal {
self.status = Some(tr(self.locale, reason).into_owned());
return None;
}
if let Err(err) = manager.delete_session(&session.id) {
self.status = Some(
tr(self.locale, MessageId::SessionsDeleteFailed)
@@ -1633,6 +1646,59 @@ mod tests {
view
}
/// Deleting the session open in this window would leave it unable to
/// save anything afterwards, with nothing on screen saying so.
#[test]
fn delete_refuses_the_session_open_here() {
let _lock = crate::test_support::lock_test_env();
let tmp = tempfile::tempdir().expect("tempdir");
let _home = crate::test_support::EnvVarGuard::set("CODEWHALE_HOME", tmp.path());
let manager = SessionManager::default_location().expect("session manager");
let mut saved = saved_session_with_messages(vec![text_message("user", "hello")]);
saved.metadata.id = "session-open-here".to_string();
manager.save_session(&saved).expect("save session");
let mut view = picker_with(vec![saved.metadata.clone()], None);
view.current_session_id = Some("session-open-here".to_string());
assert!(view.delete_selected().is_none(), "no deletion event");
assert_eq!(
view.status.as_deref(),
Some(tr(view.locale, MessageId::SessionsDeleteOpenHere).as_ref())
);
assert_eq!(view.sessions.len(), 1, "the row stays listed");
assert!(
manager.load_session("session-open-here").is_ok(),
"the document is not deleted"
);
}
/// A session another Codewhale window holds open would be written back
/// by that window's next autosave, so deleting it here is refused.
#[test]
fn delete_refuses_a_session_open_in_another_window() {
let _lock = crate::test_support::lock_test_env();
let tmp = tempfile::tempdir().expect("tempdir");
let _home = crate::test_support::EnvVarGuard::set("CODEWHALE_HOME", tmp.path());
let manager = SessionManager::default_location().expect("session manager");
let mut saved = saved_session_with_messages(vec![text_message("user", "hello")]);
saved.metadata.id = "session-open-elsewhere".to_string();
manager.save_session(&saved).expect("save session");
let _lease = manager.hold_live_lease_elsewhere("session-open-elsewhere");
let mut view = picker_with(vec![saved.metadata.clone()], None);
assert!(view.delete_selected().is_none(), "no deletion event");
assert_eq!(
view.status.as_deref(),
Some(tr(view.locale, MessageId::SessionsDeleteOpenElsewhere).as_ref())
);
assert!(
manager.load_session("session-open-elsewhere").is_ok(),
"the document is not deleted"
);
}
#[test]
fn rename_selected_persists_and_emits_saved_metadata() {
let _lock = crate::test_support::lock_test_env();
+17 -23
View File
@@ -1482,28 +1482,19 @@ async fn apply_command_result_inner(
return Ok(false);
}
};
// A managed record resumes through the manager so its repair is
// hydrated, applied, and persisted in place. A foreign `/load`
// file is not ours to rewrite: hydrate its journal projection
// and repair in memory only.
let session = match SessionManager::default_location() {
Ok(manager) if manager.owns_session_path(&parsed.metadata.id, &path) => {
match manager.resume_session(&parsed.metadata.id) {
Ok(recovery) => recovery.session,
Err(err) => {
crate::tui::ui::session_state::surface_session_load_failure(
app,
format!("Failed to resume session {}: {err}", path.display()),
);
return Ok(false);
}
}
}
_ => {
let mut session = parsed;
session.ensure_journal();
crate::session_manager::repair_recovered_session(&mut session);
session
// `/load` shares the attach contract of `resume` and the
// picker (`SessionManager::attach_session_file`); the lease is
// committed only once the session is applied.
let attached = SessionManager::default_location()
.and_then(|manager| manager.attach_session_file(parsed, &path));
let (session, lease) = match attached {
Ok(attached) => attached,
Err(err) => {
crate::tui::ui::session_state::surface_session_load_failure(
app,
format!("Failed to resume session {}: {err}", path.display()),
);
return Ok(false);
}
};
let fresh_config =
@@ -1527,7 +1518,10 @@ async fn apply_command_result_inner(
fresh_config,
true,
) {
Ok(outcome) => outcome,
Ok(outcome) => {
lease.commit();
outcome
}
Err(err) => {
crate::tui::ui::session_state::surface_session_load_failure(
app,
+122 -25
View File
@@ -577,6 +577,8 @@ pub(crate) fn coalesce_scroll_burst(
latest = *next;
}
_ => {
// Back to the head, not the tail: `pending` may already hold
// later input (typed text, Enter), and this event came first.
pending.push_front(next_observed);
break;
}
@@ -600,6 +602,76 @@ pub(crate) fn transcript_cadence_tier(
)
}
/// Fold every queued `Resize` behind the one in hand into the final size, so
/// one clear and redraw serves the whole drag (#65).
///
/// The first non-resize event ends the fold and goes back to the head of the
/// queue, ahead of any later input already drained into `pending`.
pub(crate) fn coalesce_resize_burst(
width: u16,
height: u16,
input: &TerminalInputPump,
pending: &mut VecDeque<ObservedTerminalEvent>,
) -> std::io::Result<(u16, u16)> {
let (mut final_w, mut final_h) = (width, height);
while let Some(next_observed) = try_next_terminal_event(input, pending)? {
if let Event::Resize(w, h) = next_observed.event {
final_w = w;
final_h = h;
} else {
pending.push_front(next_observed);
break;
}
}
Ok((final_w, final_h))
}
/// Toast identity for a failing session save, so recovery can retire it.
const SESSION_SAVE_FAILURE_TOAST: &str = "session-save-failure";
/// Keep the save-failure notice in step with the persistence actor's session
/// save health. Saves run off the UI thread, so a full disk or an unwritable
/// sessions directory used to reach only the log while the user kept working
/// unsaved. The notice stays while a session's latest save is failing and is
/// withdrawn once a later save lands.
pub(crate) fn surface_session_save_health(
app: &mut App,
reading: Option<crate::tui::persistence_actor::SaveHealthReading>,
seen: &mut u64,
) {
let Some(reading) = reading.filter(|reading| reading.generation != *seen) else {
return;
};
*seen = reading.generation;
app.retire_event_notices(SESSION_SAVE_FAILURE_TOAST);
if let Some((session_id, kind)) = reading.failing {
let text = app
.tr(MessageId::SessionSaveFailed)
.replace("{id}", crate::session_manager::truncate_id(&session_id))
.replace("{error}", &kind.to_string());
// Standing, not timed: it must outlast the failure, and only the
// recovery reading above withdraws it.
app.push_status_toast_record(StatusToast::standing(
text,
StatusToastLevel::Error,
SESSION_SAVE_FAILURE_TOAST,
));
}
}
/// The exit line when a session's latest save failed and was never replaced
/// by a successful one.
pub(crate) fn shutdown_persistence_notice(
locale: codewhale_localization::Locale,
reading: &crate::tui::persistence_actor::SaveHealthReading,
) -> Option<String> {
reading.failing.as_ref().map(|(session_id, kind)| {
codewhale_localization::tr(locale, MessageId::SessionSaveFailedAtExit)
.replace("{id}", session_id)
.replace("{error}", &kind.to_string())
})
}
/// Run the interactive TUI event loop.
///
/// # Examples
@@ -820,28 +892,38 @@ pub async fn run_tui(
&& let Ok(manager) = SessionManager::default_location()
{
// Try to load by prefix or full ID
let load_result: std::io::Result<Option<crate::session_manager::SavedSession>> =
let load_result: std::io::Result<
Option<(
crate::session_manager::SavedSession,
crate::session_manager::SessionLease,
)>,
> =
// `attach_*` reserves the session's live lease first, and refuses
// a session another window has open instead of becoming its second
// autosaving writer. The lease is committed once the session is
// applied.
if session_id == "latest" {
// Special case: resume the most recent session in this workspace.
match manager.get_latest_session_for_workspace(&options.workspace) {
Ok(Some(meta)) => manager
.resume_session(&meta.id)
.map(|recovery| Some(recovery.session)),
.attach_session(&meta.id)
.map(|(recovery, lease)| Some((recovery.session, lease))),
Ok(None) => Ok(None),
Err(e) => Err(e),
}
} else {
manager
.resume_session_by_prefix(session_id)
.map(|recovery| Some(recovery.session))
.attach_session_by_prefix(session_id)
.map(|(recovery, lease)| Some((recovery.session, lease)))
};
match load_result {
Ok(Some(saved)) => match manager.load_session_goal(&saved.metadata.id) {
Ok(Some((saved, lease))) => match manager.load_session_goal(&saved.metadata.id) {
Ok(goal) => {
let saved_id = saved.metadata.id.clone();
match apply_loaded_session_with_goal(&mut app, config, saved, goal.as_ref()) {
Ok(()) => {
lease.commit();
app.status_message = Some(format!(
"Resumed session: {}",
crate::session_manager::truncate_id(&saved_id)
@@ -1173,6 +1255,7 @@ pub async fn run_tui(
// applied), the checkpoint is the only durable record of that work:
// clearing it here unconditionally could erase in-flight progress that
// never reached a snapshot, so it survives for startup recovery review.
let mut shutdown_save_health = None;
if let Some((handle, task)) = persistence_runtime {
// A quit key can leave the frame before its usual queue comparison.
// Capture the final edited draft before the shutdown durability barrier.
@@ -1201,6 +1284,9 @@ pub async fn run_tui(
"session persistence reported write failures during shutdown",
);
}
// Read after the final flush: whether each session's latest save
// landed, not every failure this run has ever seen.
shutdown_save_health = Some(handle.session_save_health());
handle.try_send(PersistRequest::Shutdown);
let _ = task.await;
}
@@ -1260,7 +1346,24 @@ pub async fn run_tui(
}
}
if let Some(notice) = shutdown_save_health
.as_ref()
.and_then(|reading| shutdown_persistence_notice(app.ui_locale, reading))
{
// Primary screen, like the settings failures above.
#[allow(clippy::print_stderr)]
{
eprintln!("{notice}");
}
}
// `codewhale resume <id>` for a document that never reached disk (every
// save failed, or nothing was ever saved) only fails with NotFound.
let session_document_exists = app.current_session_id.as_deref().is_some_and(|id| {
SessionManager::default_location().is_ok_and(|manager| manager.session_document_exists(id))
});
if result.is_ok()
&& session_document_exists
&& let Some(hint) = resume_hint_text(
app.ui_locale,
app.current_session_id.as_deref(),
@@ -1677,6 +1780,7 @@ pub(crate) async fn run_event_loop(
}
let mut pending_subagent_list_refresh = false;
let mut session_save_health_seen = 0u64;
loop {
// #6169: first statement of every iteration. The job-control handler can
@@ -2024,6 +2128,12 @@ pub(crate) async fn run_event_loop(
deliver_constitution_draft_result(app, model_label, draft_locale, outcome);
}
surface_session_save_health(
app,
crate::tui::persistence_actor::session_save_health(),
&mut session_save_health_seen,
);
// Discovery and callback delivery never park terminal input.
poll_mcp_login(app);
poll_mcp_retries(app);
@@ -4747,25 +4857,12 @@ pub(crate) async fn run_event_loop(
// common "stale art on the right edge" symptom (#65) caused by
// the diff renderer skipping cells that match a stale back
// buffer between intermediate sizes.
let mut final_w = width;
let mut final_h = height;
while let Some(next_observed) =
try_next_terminal_event(&terminal_input, &mut pending_terminal_events)?
{
match next_observed.event {
Event::Resize(w, h) => {
final_w = w;
final_h = h;
}
other => {
pending_terminal_events.push_front(ObservedTerminalEvent::new(
other,
next_observed.observed_at,
));
break;
}
}
}
let (final_w, final_h) = coalesce_resize_burst(
width,
height,
&terminal_input,
&mut pending_terminal_events,
)?;
if final_w == 0 || final_h == 0 {
tracing::debug!(
+11 -3
View File
@@ -1684,8 +1684,10 @@ pub(crate) async fn handle_view_events(
}
};
match manager.resume_session(&session_id) {
Ok(recovery) => {
// Another window's open session is refused, not attached
// as a second autosaving writer.
match manager.attach_session(&session_id) {
Ok((recovery, lease)) => {
let session = recovery.session;
let next_config = config.clone();
let message_count = session.metadata.message_count;
@@ -1699,7 +1701,13 @@ pub(crate) async fn handle_view_events(
next_config,
false,
) {
Ok(outcome) => outcome,
Ok(outcome) => {
// Only now does this window give up the
// session it had open; a failed restore
// above keeps that session's lease.
lease.commit();
outcome
}
Err(err) => {
crate::tui::ui::session_state::surface_session_load_failure(
app,
+163
View File
@@ -31084,6 +31084,169 @@ fn a_scroll_burst_is_folded_into_one_frame() {
));
}
fn observed_key(c: char) -> ObservedTerminalEvent {
ObservedTerminalEvent::new(
Event::Key(KeyEvent::new(KeyCode::Char(c), KeyModifiers::NONE)),
Instant::now(),
)
}
fn idle_input_pump() -> TerminalInputPump {
let (_tx, rx) = std::sync::mpsc::channel();
TerminalInputPump {
rx,
stop: std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)),
paused: std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)),
paused_ack: std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)),
handle: None,
last_alive_at: std::cell::Cell::new(Instant::now()),
}
}
fn pending_key_chars(pending: &VecDeque<ObservedTerminalEvent>) -> String {
pending
.iter()
.filter_map(|observed| match &observed.event {
Event::Key(KeyEvent {
code: KeyCode::Char(c),
..
}) => Some(*c),
_ => None,
})
.collect()
}
/// Typing while the wheel scrolls: the loop has already drained the burst and
/// the keys into `pending`. The key that ends the burst goes back to the head,
/// so the composer receives "abc", not "bca".
#[test]
fn a_scroll_burst_keeps_following_input_in_order() {
let mut app = create_test_app();
app.launch.visible = false;
app.viewport.last_transcript_area = Some(Rect::new(0, 0, 80, 20));
let scroll = crossterm::event::MouseEvent {
kind: crossterm::event::MouseEventKind::ScrollDown,
column: 10,
row: 5,
modifiers: KeyModifiers::NONE,
};
let input = idle_input_pump();
let mut pending: VecDeque<ObservedTerminalEvent> = VecDeque::new();
pending.push_back(ObservedTerminalEvent::new(
Event::Mouse(scroll),
Instant::now(),
));
for c in ['a', 'b', 'c'] {
pending.push_back(observed_key(c));
}
super::event_loop::coalesce_scroll_burst(&mut app, scroll, &input, &mut pending)
.expect("coalesce");
assert_eq!(pending_key_chars(&pending), "abc");
}
#[test]
fn a_resize_burst_keeps_following_input_in_order() {
let input = idle_input_pump();
let mut pending: VecDeque<ObservedTerminalEvent> = VecDeque::new();
pending.push_back(ObservedTerminalEvent::new(
Event::Resize(120, 40),
Instant::now(),
));
pending.push_back(observed_key('a'));
pending.push_back(observed_key('b'));
let size =
super::event_loop::coalesce_resize_burst(100, 30, &input, &mut pending).expect("coalesce");
assert_eq!(size, (120, 40), "the final queued size wins");
assert_eq!(pending_key_chars(&pending), "ab");
}
/// A failing session save reaches the user, not only the log, and the notice
/// is withdrawn once a later save of that session lands.
#[test]
fn a_failing_session_save_shows_an_error_until_a_save_lands() {
use crate::tui::persistence_actor::SaveHealthReading;
let mut app = create_test_app();
let mut seen = 0;
let failing = SaveHealthReading {
generation: 1,
failing: Some((
"toast-probe".to_string(),
std::io::ErrorKind::PermissionDenied,
)),
};
let save_errors = |app: &App| {
app.status_toasts
.iter()
.filter(|toast| toast.level == StatusToastLevel::Error)
.count()
};
super::event_loop::surface_session_save_health(&mut app, Some(failing.clone()), &mut seen);
assert_eq!(seen, 1);
assert_eq!(save_errors(&app), 1, "a failing save is visible");
assert!(
app.status_toasts
.iter()
.any(|toast| toast.text.contains("toast-pr")),
"{:?}",
app.status_toasts
);
// Polling the same reading again does not repeat it.
super::event_loop::surface_session_save_health(&mut app, Some(failing), &mut seen);
assert_eq!(save_errors(&app), 1);
// The failure outlives any toast lifetime: well past the sticky TTL, and
// behind a full queue of newer notices, it is still there and shown once
// they expire, because nothing has recovered.
for toast in app.status_toasts.iter_mut() {
toast.created_at = std::time::Instant::now()
- std::time::Duration::from_millis(App::STICKY_ERROR_TTL_MS * 10);
}
for i in 0..30 {
app.push_status_toast(format!("newer {i}"), StatusToastLevel::Info, Some(1));
}
std::thread::sleep(std::time::Duration::from_millis(5));
let shown = app
.active_status_toast(crate::tui::underwater::ShellPhase::Idle)
.expect("a toast is shown");
assert!(shown.text.contains("toast-pr"), "{}", shown.text);
assert_eq!(save_errors(&app), 1, "a standing failure does not expire");
let healed = SaveHealthReading {
generation: 2,
failing: None,
};
super::event_loop::surface_session_save_health(&mut app, Some(healed), &mut seen);
assert_eq!(save_errors(&app), 0, "a later successful save withdraws it");
}
#[test]
fn shutdown_reports_only_a_save_that_is_still_failing() {
use crate::tui::persistence_actor::SaveHealthReading;
let locale = codewhale_localization::Locale::En;
let healthy = SaveHealthReading {
generation: 4,
failing: None,
};
assert_eq!(
super::event_loop::shutdown_persistence_notice(locale, &healthy),
None,
"failures later replaced by a successful save are not reported"
);
let failing = SaveHealthReading {
generation: 5,
failing: Some(("session-a".to_string(), std::io::ErrorKind::StorageFull)),
};
let notice = super::event_loop::shutdown_persistence_notice(locale, &failing)
.expect("a failing save produces an exit notice");
assert!(notice.contains("session-a"), "{notice}");
}
// ---------------------------------------------------------------------------
// Startup type-ahead integrity (#5925).
//
+43 -8
View File
@@ -112,15 +112,16 @@ fn reasoning_effort_for_route_selection(
}
}
/// Settle the classifier usage of a turn whose route then failed, into the
/// cost scope that was current when the classifier call was made. A fresh
/// token taken here would bill it to whatever session is current by now:
/// after a `/new` or session load during the call, a different one.
fn settle_failed_parent_route(
cost_scope: crate::cost_status::CostScopeToken,
error: String,
initial_routed_usage: &crate::cost_status::RuntimeUsageBatch,
) -> String {
crate::cost_status::report_runtime_usage_batch(
crate::cost_status::scope_token(),
None,
initial_routed_usage,
);
crate::cost_status::report_runtime_usage_batch(cost_scope, None, initial_routed_usage);
error
}
@@ -141,6 +142,9 @@ fn settle_failed_parent_route(
pub(crate) async fn plan_turn_route(
request: TurnRoutePlanRequest<'_>,
) -> Result<PlannedTurnRoute, String> {
// Taken before the classifier call so its usage settles into the scope
// it was spent in, not the one current when a failure is noticed.
let cost_scope = crate::cost_status::scope_token();
let mut auto_selection = if request.should_auto_resolve {
Some(
crate::model_routing::resolve_auto_route_with_inventory_for_session_and_cache_policy(
@@ -210,6 +214,7 @@ pub(crate) async fn plan_turn_route(
Ok(route) => route,
Err(err) => {
return Err(settle_failed_parent_route(
cost_scope,
err.to_string(),
&initial_routed_usage,
));
@@ -219,7 +224,11 @@ pub(crate) async fn plan_turn_route(
match turn_route.preflight() {
Ok(route) => route,
Err(err) => {
return Err(settle_failed_parent_route(err, &initial_routed_usage));
return Err(settle_failed_parent_route(
cost_scope,
err,
&initial_routed_usage,
));
}
}
} else {
@@ -357,17 +366,43 @@ mod tests {
dropped_records: 1,
};
let scope = crate::cost_status::scope_token();
assert_eq!(
settle_failed_parent_route("route failed".to_string(), &batch),
settle_failed_parent_route(scope, "route failed".to_string(), &batch),
"route failed"
);
settle_failed_parent_route("route failed".to_string(), &batch);
settle_failed_parent_route(scope, "route failed".to_string(), &batch);
let pending = crate::cost_status::drain();
assert_eq!(
pending.usage_source_fingerprints.len(),
2,
"both exact classifier outcomes persist, and replay is idempotent"
);
// The classifier ran in one session; `/new` closed it before the
// route failed. Its cost belongs to the closed session, never the new.
let spent_in = crate::cost_status::scope_token();
let _closed = crate::cost_status::close_current_scope();
let batch = crate::cost_status::RuntimeUsageBatch {
records: batch
.records
.iter()
.cloned()
.map(|mut record| {
record.source_id = "auto-router:plan-usage-after-new".to_string();
record
})
.collect(),
drop_records: Vec::new(),
dropped_records: 0,
};
settle_failed_parent_route(spent_in, "route failed".to_string(), &batch);
assert!(
crate::cost_status::drain()
.usage_source_fingerprints
.is_empty(),
"classifier cost leaked into the session opened after it was spent"
);
}
#[test]
+62 -23
View File
@@ -421,7 +421,16 @@ pub(crate) async fn transcribe_local_whisper(audio_samples: &[i16]) -> Result<St
}
fn transcribe_local_whisper_blocking(wav: &[u8]) -> Result<String, String> {
let tmp = std::env::temp_dir().join(format!("cw-voice-{}.wav", std::process::id()));
// C01-10: a fresh, exclusively created file per call. The old
// `cw-voice-<pid>.wav` name was shared by every transcription in the
// process (an interim pass, the final pass, a runtime-API dictation) and
// predictable in a shared temp dir. Dropping `audio` removes it.
let audio = tempfile::Builder::new()
.prefix("cw-voice-")
.suffix(".wav")
.tempfile()
.map_err(|e| e.to_string())?;
let tmp = audio.path().to_path_buf();
std::fs::write(&tmp, wav).map_err(|e| e.to_string())?;
// Try each local binary until one succeeds; whisper.cpp outputs to stdout or file.
for bin in LOCAL_WHISPER_BINS {
@@ -437,7 +446,6 @@ fn transcribe_local_whisper_blocking(wav: &[u8]) -> Result<String, String> {
&& out.status.success()
{
let txt = String::from_utf8_lossy(&out.stdout).trim().to_string();
let _ = std::fs::remove_file(&tmp);
if !txt.is_empty() {
return Ok(txt);
}
@@ -445,14 +453,12 @@ fn transcribe_local_whisper_blocking(wav: &[u8]) -> Result<String, String> {
let sidecar = tmp.with_extension("txt");
if let Ok(s) = std::fs::read_to_string(&sidecar) {
let _ = std::fs::remove_file(&sidecar);
let _ = std::fs::remove_file(&tmp);
if !s.trim().is_empty() {
return Ok(s.trim().to_string());
}
}
}
}
let _ = std::fs::remove_file(&tmp);
Err("local whisper not available".into())
}
@@ -467,28 +473,39 @@ pub(crate) async fn transcribe_groq(audio_samples: &[i16]) -> Result<String, Str
}
/// Resolve ASR model/provider preference.
/// Priority: explicit config `voice.asr_model` > env `CODEWHALE_ASR_MODEL` > auto-detect (local-whisper > groq > xiaomi).
/// Priority: env `CODEWHALE_ASR_MODEL` > auto-detect (local-whisper > groq >
/// provider). Known limitation (C01-10): there is no config-file ASR setting
/// yet — `config` is accepted for the day one exists and is not read, so the
/// env var is the only explicit override.
pub(crate) fn resolve_asr_choice(_config: &Config) -> (String, String) {
// Check explicit env override first (free, cross-platform)
if let Ok(m) = std::env::var("CODEWHALE_ASR_MODEL") {
let m = m.trim().to_ascii_lowercase();
if m.contains("groq") || m.contains("whisper") {
return ("groq".into(), GROQ_ASR_MODEL.into());
}
if m.contains("local") || m.contains("whisper.cpp") {
return ("local-whisper".into(), "tiny".into());
}
if m.contains("mimo") || m.contains("xiaomi") {
return ("provider".into(), ASR_MODEL.into());
}
let explicit = std::env::var("CODEWHALE_ASR_MODEL")
.ok()
.and_then(|value| explicit_asr_choice(&value));
// Auto-detect best free: local whisper (offline, no key) > Groq free
// tier > provider ASR (needs key).
let (kind, model) = explicit.unwrap_or_else(|| match detect_free_asr() {
"local-whisper" => ("local-whisper", "tiny"),
"groq" => ("groq", GROQ_ASR_MODEL),
_ => ("provider", ASR_MODEL),
});
(kind.to_string(), model.to_string())
}
/// Map an explicit `CODEWHALE_ASR_MODEL` value to (kind, model). Local
/// spellings are tested first: `local-whisper` and `whisper.cpp` both contain
/// "whisper", which on its own names Groq's hosted Whisper (C01-10).
fn explicit_asr_choice(value: &str) -> Option<(&'static str, &'static str)> {
let m = value.trim().to_ascii_lowercase();
if m.contains("local") || m.contains("whisper.cpp") || m.contains("whisper-cpp") {
return Some(("local-whisper", "tiny"));
}
// Auto-detect best free: local whisper (offline, no key) > Groq free tier > Xiaomi ASR (needs key)
let free = detect_free_asr();
match free {
"local-whisper" => ("local-whisper".into(), "tiny".into()),
"groq" => ("groq".into(), GROQ_ASR_MODEL.into()),
_ => ("provider".into(), ASR_MODEL.into()),
if m.contains("groq") || m.contains("whisper") {
return Some(("groq", GROQ_ASR_MODEL));
}
if m.contains("mimo") || m.contains("xiaomi") {
return Some(("provider", ASR_MODEL));
}
None
}
// --- Headless capture (HTTP/native-client path) ----------------------------
@@ -717,6 +734,28 @@ mod tests {
assert!(independent.get("provider").is_none());
}
#[test]
fn explicit_local_whisper_is_not_routed_to_groq() {
// C01-10: every local spelling contains "whisper"; none may reach the
// hosted Groq backend (which needs a key and sends audio off-host).
for local in ["local-whisper", "whisper.cpp", "whisper-cpp", " LOCAL "] {
assert_eq!(
explicit_asr_choice(local),
Some(("local-whisper", "tiny")),
"{local}"
);
}
for hosted in ["groq", "whisper", "whisper-large-v3-turbo"] {
assert_eq!(
explicit_asr_choice(hosted),
Some(("groq", GROQ_ASR_MODEL)),
"{hosted}"
);
}
assert_eq!(explicit_asr_choice("mimo"), Some(("provider", ASR_MODEL)));
assert_eq!(explicit_asr_choice("something-else"), None);
}
#[test]
fn wav_encoding_produces_valid_header() {
let samples = vec![0i16; 16000]; // 1 second of silence
+1
View File
@@ -41,6 +41,7 @@ notes, and relevant issue/PR comments.
**Merged or adapted contributions**
- **[zhuowp](https://github.com/zhuowp)** — supplied the process-scoped PowerShell execution-policy repair adapted for Codewhale; machine and user Group Policy still take precedence ([#6745](https://github.com/Hmbown/Codewhale/issues/6745)).
- **[gaord](https://github.com/gaord)** — let a client fork a thread at a named turn ([#6580](https://github.com/Hmbown/Codewhale/pull/6580)), let undo roll back files for the turn it is undoing ([#6483](https://github.com/Hmbown/Codewhale/pull/6483)), stopped resume and fork from duplicating threads and sessions ([#6406](https://github.com/Hmbown/Codewhale/pull/6406)), exposed user-defined provider routes to native clients ([#6404](https://github.com/Hmbown/Codewhale/pull/6404)), and kept a fork going when a turn lost its tool call ([#6664](https://github.com/Hmbown/Codewhale/pull/6664)).
- **[Lstarsky0](https://github.com/Lstarsky0)** — moved the docs/work, legal, digest and FAQ pages onto the dictionary spine ([#6405](https://github.com/Hmbown/Codewhale/pull/6405), [#6417](https://github.com/Hmbown/Codewhale/pull/6417), [#6499](https://github.com/Hmbown/Codewhale/pull/6499), [#6574](https://github.com/Hmbown/Codewhale/pull/6574)), tightened the Chinese-branching ceiling to 18 ([#6403](https://github.com/Hmbown/Codewhale/pull/6403)), and made Fleet publish without a two-link window ([#6431](https://github.com/Hmbown/Codewhale/pull/6431)). Also moved the constitution page onto the dictionary spine and kept its install link in the selected locale ([#6733](https://github.com/Hmbown/Codewhale/pull/6733)).
- **[aboimpinto](https://github.com/aboimpinto)** — restored a green Linux full-workspace test gate without loosening any test, twice ([#6581](https://github.com/Hmbown/Codewhale/pull/6581), [#6666](https://github.com/Hmbown/Codewhale/pull/6666)).
+1 -1
View File
@@ -54,7 +54,7 @@ For the read-only record of what a thread or turn did, see
| `codewhale app-server --mobile` | HTTP/SSE on loopback + `/mobile` | Runtime API + local mobile control page |
| `codewhale app-server --stdio` | JSON-RPC 2.0 over stdio | Local SDK / control probe (no listener) |
| `codewhale app-server --socket [--socket-path P]` | JSON-RPC 2.0 over a `0600` unix domain socket | Desktop daemon: multi-client, peer-uid checked, `daemon/attach` claim handshake (macOS/Linux; Windows named pipe reserved, not implemented) |
| `codewhale app-server` | HTTP on `127.0.0.1:8787` | Legacy in-process app-server (`/healthz`, `/thread`, `/app`, `/prompt`, `/jobs`, `/mcp/startup`); `/prompt` and `/thread` messages execute real turns via the runtime bridge. There is no direct `/tool` route: tools run only inside Engine turns, under the Engine's tool catalog and approval posture. This legacy server does not surface approvals: its bridge forwards only text deltas and the turn's completion, and it has no decision route, so an approval-gated call waits unanswered. Drive approval-gated work through the Runtime API (`/v1/threads/*` events and `POST /v1/approvals/{approval_id}`) |
| `codewhale app-server` | HTTP on `127.0.0.1:8787` | Legacy in-process app-server (`/healthz`, `/thread`, `/app`, `/prompt`, `/jobs`); `/prompt` and `/thread` messages execute real turns via the runtime bridge. There is no direct `/tool` route: tools run only inside Engine turns, under the Engine's tool catalog and approval posture. This legacy server does not surface approvals: its bridge forwards only text deltas and the turn's completion, and it has no decision route, so an approval-gated call waits unanswered. Drive approval-gated work through the Runtime API (`/v1/threads/*` events and `POST /v1/approvals/{approval_id}`) |
| `codewhale serve --http` / `--mobile` | same server as `app-server --http`/`--mobile` | Compatibility aliases |
`app-server --http` and `--mobile` launch the same mature runtime API server
+1 -1
View File
@@ -51,7 +51,7 @@ local supervisor / SDK / automation harness
| `codewhale app-server --mobile` | 回环上的 HTTP/SSE + `/mobile` | 运行时 API + 本地移动端控制页 |
| `codewhale app-server --stdio` | 基于 stdio 的 JSON-RPC 2.0 | 本地 SDK / 控制探针(不监听端口) |
| `codewhale app-server --socket [--socket-path P]` | 基于权限为 `0600` 的 unix 域套接字的 JSON-RPC 2.0 | 桌面守护进程:多客户端、对端 uid 校验、`daemon/attach` 认领握手(macOS/Linux;Windows 命名管道已预留但未实现) |
| `codewhale app-server` | 在 `127.0.0.1:8787` 上的 HTTP | 旧式进程内 app-server(`/healthz`、`/thread`、`/app`、`/prompt`、`/jobs`、`/mcp/startup`);`/prompt` 与 `/thread` 消息会通过运行时桥执行真实回合。它没有直接的 `/tool` 路由:工具只在 Engine 回合内部、在 Engine 的工具目录与审批姿态下运行。这个旧式服务器不呈现审批:它的桥只转发文本增量和回合的完成,并且没有决策路由,所以一个受审批门控的调用会一直等不到答复。受审批门控的工作请通过运行时 API 驱动(`/v1/threads/*` 事件与 `POST /v1/approvals/{approval_id}`) |
| `codewhale app-server` | 在 `127.0.0.1:8787` 上的 HTTP | 旧式进程内 app-server(`/healthz`、`/thread`、`/app`、`/prompt`、`/jobs`);`/prompt` 与 `/thread` 消息会通过运行时桥执行真实回合。它没有直接的 `/tool` 路由:工具只在 Engine 回合内部、在 Engine 的工具目录与审批姿态下运行。这个旧式服务器不呈现审批:它的桥只转发文本增量和回合的完成,并且没有决策路由,所以一个受审批门控的调用会一直等不到答复。受审批门控的工作请通过运行时 API 驱动(`/v1/threads/*` 事件与 `POST /v1/approvals/{approval_id}`) |
| `codewhale serve --http` / `--mobile` | 与 `app-server --http`/`--mobile` 相同的服务器 | 兼容别名 |
`app-server --http` 与 `--mobile` 启动的是历史上经由 `serve --http` 访问的同一个
+71 -5
View File
@@ -1,4 +1,4 @@
import { chmod, mkdir, readFile, rename, writeFile } from "node:fs/promises";
import { chmod, mkdir, open, readFile, rename, rm } from "node:fs/promises";
import path from "node:path";
import { randomBytes } from "node:crypto";
@@ -20,6 +20,39 @@ async function chmodBestEffort(filePath, mode) {
}
}
/**
* Replace `filePath` so a crash leaves either the old bytes or the new ones:
* a unique temporary name (two processes sharing a state dir never write the
* same temp file), fsync before the rename, and a best-effort directory fsync
* after it so the rename itself survives power loss where the OS allows.
*/
export async function writeFileDurable(filePath, contents, { mode = 0o600 } = {}) {
const dir = path.dirname(filePath);
const tmp = path.join(dir, `.${path.basename(filePath)}.${process.pid}.${randomBytes(6).toString("hex")}.tmp`);
let handle = null;
try {
handle = await open(tmp, "wx", mode);
await handle.writeFile(contents);
await handle.sync();
await handle.close();
handle = null;
await rename(tmp, filePath);
} catch (error) {
await handle?.close().catch(() => {});
await rm(tmp, { force: true }).catch(() => {});
throw error;
}
let dirHandle = null;
try {
dirHandle = await open(dir, "r");
await dirHandle.sync();
} catch {
// Windows and some filesystems cannot fsync a directory.
} finally {
await dirHandle?.close().catch(() => {});
}
}
export class ThreadStore {
static async open(filePath, options = {}) {
const store = new ThreadStore(filePath, options);
@@ -48,6 +81,9 @@ export class ThreadStore {
if (this.options.messageLimit > 0 && !Array.isArray(this.data.messages)) {
this.data.messages = [];
}
if (this.options.messageLimit > 0 && (!this.data.inflight || typeof this.data.inflight !== "object" || Array.isArray(this.data.inflight))) {
this.data.inflight = {};
}
if (this.options.actions && (!this.data.actions || typeof this.data.actions !== "object")) {
this.data.actions = {};
}
@@ -66,6 +102,39 @@ export class ThreadStore {
}
}
/**
* Claim one inbound message before acting on it. "new": handle it now (it
* is recorded as in flight first, durably). "done": a finished duplicate.
* "interrupted": an earlier process claimed it and stopped before
* completeMessage, so its effect is unknown; the caller reports that
* instead of running it again. Callers handle messages sequentially — a
* key still in flight in this process is not a restart.
*/
async claimMessage(messageKey) {
if (!messageKey || this.options.messageLimit <= 0) return "new";
this.ensureShape();
const inflight = this.data.inflight;
if (this.data.messages.includes(messageKey)) {
if (!Object.hasOwn(inflight, messageKey)) return "done";
delete inflight[messageKey];
await this.save();
return "interrupted";
}
this.data.messages.push(messageKey);
this.data.messages = this.data.messages.slice(-this.options.messageLimit);
for (const key of Object.keys(inflight)) if (!this.data.messages.includes(key)) delete inflight[key];
inflight[messageKey] = new Date().toISOString();
await this.save();
return "new";
}
/** Mark a claimed message handled (whether it succeeded or failed in-process). */
async completeMessage(messageKey) {
if (!messageKey || !this.data.inflight || !Object.hasOwn(this.data.inflight, messageKey)) return;
delete this.data.inflight[messageKey];
await this.save();
}
async recordMessage(messageKey) {
if (!messageKey || this.options.messageLimit <= 0) return false;
this.ensureShape();
@@ -180,10 +249,7 @@ export class ThreadStore {
const dir = path.dirname(this.filePath);
await mkdir(dir, { recursive: true, mode: 0o700 });
if (this.options.privateMode) await chmodBestEffort(dir, 0o700);
const tmp = `${this.filePath}.tmp`;
await writeFile(tmp, `${JSON.stringify(this.data, null, 2)}\n`, { mode: 0o600 });
if (this.options.privateMode) await chmodBestEffort(tmp, 0o600);
await rename(tmp, this.filePath);
await writeFileDurable(this.filePath, `${JSON.stringify(this.data, null, 2)}\n`, { mode: 0o600 });
if (this.options.privateMode) await chmodBestEffort(this.filePath, 0o600);
}
}
+35 -2
View File
@@ -1,6 +1,6 @@
import test from "node:test";
import assert from "node:assert/strict";
import { mkdtemp, rm } from "node:fs/promises";
import { mkdtemp, readdir, readFile, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import path from "node:path";
@@ -19,7 +19,8 @@ import {
readSse,
splitMessage,
stripGroupPrefix,
ThreadStore
ThreadStore,
writeFileDurable
} from "../src/lib.mjs";
test("env and primitive parsers handle bridge env conventions", () => {
@@ -255,3 +256,35 @@ test("ThreadStore persists numeric cursors", async () => {
await rm(dir, { recursive: true, force: true });
}
});
test("writeFileDurable replaces the file through unique temp names and leaves none behind", async () => {
const dir = await mkdtemp(path.join(tmpdir(), "codewhale-bridge-core-"));
try {
const target = path.join(dir, "sync-buf.txt");
// Concurrent writers used to share one fixed `.tmp` path and race on it.
await Promise.all(Array.from({ length: 8 }, (_, index) => writeFileDurable(target, `cursor-${index}`)));
assert.match(await readFile(target, "utf8"), /^cursor-[0-7]$/);
assert.deepEqual(await readdir(dir), ["sync-buf.txt"]);
} finally {
await rm(dir, { recursive: true, force: true });
}
});
test("ThreadStore claims survive a restart: finished messages skip, an in-flight one is reported", async () => {
const dir = await mkdtemp(path.join(tmpdir(), "codewhale-bridge-core-"));
try {
const statePath = path.join(dir, "thread-map.json");
const store = await ThreadStore.open(statePath, { messageLimit: 10 });
assert.equal(await store.claimMessage("u:1"), "new");
await store.completeMessage("u:1");
assert.equal(await store.claimMessage("u:2"), "new");
// The process dies here, before completeMessage("u:2").
const restarted = await ThreadStore.open(statePath, { messageLimit: 10 });
assert.equal(await restarted.claimMessage("u:1"), "done");
assert.equal(await restarted.claimMessage("u:2"), "interrupted");
assert.equal(await restarted.claimMessage("u:2"), "done", "reported once, then an ordinary duplicate");
assert.equal(await restarted.claimMessage("u:3"), "new");
} finally {
await rm(dir, { recursive: true, force: true });
}
});
+72 -70
View File
@@ -23,9 +23,10 @@ import {
latestRunningTurn,
activeTurnBlock,
helpText,
processUpdateBatch,
} from "./lib.mjs";
import { renderQrToText } from "./qr.mjs";
import { ThreadStore as CoreThreadStore } from "../../bridge-core/src/lib.mjs";
import { ThreadStore as CoreThreadStore, writeFileDurable } from "../../bridge-core/src/lib.mjs";
// ============================================================================
// ThreadStore — JSON 文件持久化(与 feishu/telegram/wechat bridge 一致)
@@ -59,11 +60,7 @@ async function loadAccount(stateDir) {
async function saveAccount(stateDir, account) {
const p = resolveAccountPath(stateDir);
await fs.mkdir(path.dirname(p), { recursive: true, mode: 0o700 });
const tmp = `${p}.tmp`;
await fs.writeFile(tmp, `${JSON.stringify(account, null, 2)}\n`, {
mode: 0o600,
});
await fs.rename(tmp, p);
await writeFileDurable(p, `${JSON.stringify(account, null, 2)}\n`, { mode: 0o600 });
}
// ============================================================================
@@ -718,9 +715,57 @@ async function saveSyncBuf(stateDir, buf) {
// The state dir may not exist yet on a first run whose first persisted write
// is the poll cursor rather than account.json.
await fs.mkdir(path.dirname(p), { recursive: true, mode: 0o700 });
const tmp = `${p}.tmp`;
await fs.writeFile(tmp, buf, { mode: 0o600 });
await fs.rename(tmp, p);
await writeFileDurable(p, buf, { mode: 0o600 });
}
/** One claimed inbound message: context token, text, allowlist, command. */
async function handleInbound(msg) {
const fromUser = msg.from_user_id || "";
// 保存 context_token
if (msg.context_token) {
await threadStore.patchChat(fromUser, {
contextToken: msg.context_token,
updatedAt: new Date().toISOString(),
});
}
// 提取文本
const text = extractText(msg.item_list);
if (!text) {
await sendText(
fromUser,
"仅支持文本消息。图片/语音/视频/文件暂不支持。"
);
return;
}
console.log(
`[inbound] from=${fromUser} text=${text.slice(0, 100)}`
);
// 白名单检查
if (!isAllowed(fromUser)) {
await sendText(
fromUser,
[
"This WeChat user is not in WEIXIN_CHAT_ALLOWLIST.",
`user_id=${fromUser}`,
"",
"For first pairing, add this user_id to WEIXIN_CHAT_ALLOWLIST, or temporarily set WEIXIN_ALLOW_UNLISTED=true.",
].join("\n")
);
return;
}
// 命令路由
const command = parseCommand(text);
await handleCommand(fromUser, command).catch((error) => {
console.error(
`failed to handle command from=${fromUser} text=${text.slice(0, 100)}`,
error
);
});
}
async function monitorLoop() {
@@ -775,69 +820,26 @@ async function monitorLoop() {
consecutiveFailures = 0;
// 保存游标
if (resp.get_updates_buf) {
getUpdatesBuf = resp.get_updates_buf;
await saveSyncBuf(config.stateDir, getUpdatesBuf);
}
// 处理消息
const msgs = resp.msgs || [];
for (const msg of msgs) {
const fromUser = msg.from_user_id || "";
const messageId = String(msg.message_id || "");
if (!fromUser) continue;
const msgKey = `${fromUser}:${messageId}`;
if (await threadStore.recordMessage(msgKey)) continue;
// 保存 context_token
if (msg.context_token) {
await threadStore.patchChat(fromUser, {
contextToken: msg.context_token,
updatedAt: new Date().toISOString(),
});
}
// 提取文本
const text = extractText(msg.item_list);
if (!text) {
// 先处理消息,再保存游标:进程崩溃时整批从 iLink 重放,
// 已完成的消息按 key 跳过,崩溃时正在处理的消息只提示、不重跑。
await processUpdateBatch({
messages: resp.msgs || [],
nextCursor: resp.get_updates_buf,
store: threadStore,
keyOf: (msg) => (msg.from_user_id ? `${msg.from_user_id}:${String(msg.message_id || "")}` : null),
interrupted: async (msg) => {
if (!isAllowed(msg.from_user_id)) return;
await sendText(
fromUser,
"仅支持文本消息。图片/语音/视频/文件暂不支持。"
msg.from_user_id,
"桥接进程在处理这条消息时重启,未重复执行;如仍需要请重新发送。\nThe bridge restarted while handling this message and did not run it again; send it again if you still need it."
);
continue;
}
console.log(
`[inbound] from=${fromUser} text=${text.slice(0, 100)}`
);
// 白名单检查
if (!isAllowed(fromUser)) {
await sendText(
fromUser,
[
"This WeChat user is not in WEIXIN_CHAT_ALLOWLIST.",
`user_id=${fromUser}`,
"",
"For first pairing, add this user_id to WEIXIN_CHAT_ALLOWLIST, or temporarily set WEIXIN_ALLOW_UNLISTED=true.",
].join("\n")
);
continue;
}
// 命令路由
const command = parseCommand(text);
await handleCommand(fromUser, command).catch((error) => {
console.error(
`failed to handle command from=${fromUser} text=${text.slice(0, 100)}`,
error
);
});
}
},
handle: handleInbound,
commitCursor: async (buf) => {
getUpdatesBuf = buf;
await saveSyncBuf(config.stateDir, buf);
},
});
} catch (error) {
if (error.name === "AbortError" || error.message?.includes("abort")) {
// 长轮询超时是正常的,立即重试
+27
View File
@@ -35,6 +35,33 @@ export {
splitMessage,
};
/**
* Handle one getUpdates batch, then advance the poll cursor. The cursor moves
* only after every message in the batch was handled, so a crash replays the
* batch from iLink instead of dropping a prompt. Each message is claimed
* durably first: on replay, finished messages are skipped, and one a crashed
* process was still handling is reported through `interrupted` rather than
* run twice (its effect is unknown).
*/
export async function processUpdateBatch({ messages, nextCursor, store, keyOf, handle, interrupted, commitCursor }) {
for (const msg of messages || []) {
const key = keyOf(msg);
if (!key) continue;
const claim = await store.claimMessage(key);
if (claim === "done") continue;
if (claim === "interrupted") {
await interrupted(msg);
continue;
}
try {
await handle(msg);
} finally {
await store.completeMessage(key);
}
}
if (nextCursor) await commitCursor(nextCursor);
}
export function randomUin() {
const uint32 = crypto.randomBytes(4).readUInt32BE(0);
return Buffer.from(String(uint32), "utf-8").toString("base64");
@@ -1,5 +1,10 @@
import test from "node:test";
import assert from "node:assert/strict";
import { mkdtemp, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import path from "node:path";
import { ThreadStore } from "../../bridge-core/src/lib.mjs";
import {
activeTurnBlock,
@@ -10,6 +15,7 @@ import {
parseCommand,
parseList,
preservedChatStateFields,
processUpdateBatch,
splitMessage
} from "../src/lib.mjs";
@@ -51,3 +57,44 @@ test("shared state and runtime helpers preserve Weixin bridge behavior", () => {
message: "Thread already has active turn turn-1. Wait for it to finish or send /interrupt."
});
});
test("a crash mid-batch replays the batch without losing or re-running prompts", async () => {
const dir = await mkdtemp(path.join(tmpdir(), "codewhale-weixin-bridge-"));
try {
const statePath = path.join(dir, "thread-map.json");
const batch = [1, 2, 3].map((id) => ({ from_user_id: "u", message_id: id }));
const keyOf = (msg) => `${msg.from_user_id}:${msg.message_id}`;
const handled = [];
const cursors = [];
const crash = new Error("process killed");
const first = await ThreadStore.open(statePath, { messageLimit: 50 });
let afterCrash = null;
await assert.rejects(processUpdateBatch({
messages: batch, nextCursor: "buf-2", store: first, keyOf,
handle: async (msg) => {
if (msg.message_id === 2) {
// What a restarted process finds on disk while message 2 runs.
afterCrash = await ThreadStore.open(statePath, { messageLimit: 50 });
throw crash;
}
handled.push(msg.message_id);
},
interrupted: async () => assert.fail("nothing is interrupted on the first pass"),
commitCursor: async (buf) => cursors.push(buf),
}), crash);
assert.deepEqual(cursors, [], "the cursor does not move past unhandled messages");
const reported = [];
await processUpdateBatch({
messages: batch, nextCursor: "buf-2", store: afterCrash, keyOf,
handle: async (msg) => handled.push(msg.message_id),
interrupted: async (msg) => reported.push(msg.message_id),
commitCursor: async (buf) => cursors.push(buf),
});
assert.deepEqual(handled, [1, 3], "message 1 is not re-run; message 3 is not lost");
assert.deepEqual(reported, [2], "the in-flight message is reported, not re-run");
assert.deepEqual(cursors, ["buf-2"]);
} finally {
await rm(dir, { recursive: true, force: true });
}
});
+5 -3
View File
@@ -88,8 +88,8 @@ The Rust particle implementation lives in the product's
`crates/tui/src/tui/ambient_life/pet_sim.rs`; this package's runner imports it.
Swift and Kotlin particle ports must match its conformance digests. Generated
native bundles are committed so the product Rust build needs no Node compiler.
Run `npm run sync` after changing core source; the generated-byte check in CI
rejects a stale bundle. Local Whalesong consumers use aliases to these same
Run `npm run sync` after changing core source; `npm run check` (run by CI)
rejects a stale bundle or `shared.html` copy. Local Whalesong consumers use aliases to these same
canonical files, not separately maintained source copies.
The original Whalesong importer, signal model, schema and browser storage code
@@ -129,7 +129,9 @@ Live recording continues in segments. At 216,000 buckets (24 hours) or 64 MiB,
the recorder syncs the completed file, preserves it as
`OUTPUT.segment-000001.jsonl` (then `000002`, etc.), and atomically replaces the
same live pathname. Each segment starts at sequence zero and replays independently.
Use `--segment-buckets=N` to rotate sooner. Followers establish a new baseline
Use `--segment-buckets=N` to rotate sooner. A stall or suspension shorter than
one segment is recorded as unknown buckets; a longer one starts a new segment
whose first bucket is unknown, as `--resume` does, and only reports its length. Followers establish a new baseline
after replacement, then accept subsequent appends as current observations.
The live importer retains unfinished lifetimes and 16 seconds of completed
Loaded 100 of 158 files, more files were not shown because too many files have changed in this diff. Show more