mirror of
https://github.com/codewhale-hq/Codewhale.git
synced 2026-11-08 08:04:25 -10:00
Merge commit '5ef2e3a50c4d2e4861899c977bcf65f706c1789e' into opus/0101-conformance-harness
Signed-off-by: CodeWhale Bot <[email protected]>
This commit is contained in:
commit
afe46fca0a
158 files changed
+6783
-1109
No files matched your search
@@ -498,6 +498,24 @@ assert.equal(
|
||||
);
|
||||
assert.match(release, /overwrite_files:\s*false/);
|
||||
assert.match(release, /fail_on_unmatched_files:\s*true/);
|
||||
// Assets land in a draft; the whole verified set becomes public at once, and
|
||||
// derived channels (the tagged container manifest) follow the canonical release.
|
||||
assert.match(release, /files: artifacts\/\*\n\s+draft: true\n/);
|
||||
assert.match(
|
||||
namedStep(release, "Verify the draft's exact asset set, then publish it at once"),
|
||||
/verify-release-inventory\.js \\\n\s+--draft --asset-dir artifacts --publish/,
|
||||
);
|
||||
const releaseJob = release.match(/\n release:\n([\s\S]*?)\n npm:\n/);
|
||||
assert.ok(releaseJob, "public release must retain its release job");
|
||||
assert.match(releaseJob[1], /^ needs: \[artifacts, docker-build, resolve\]$/m);
|
||||
const dockerJob = release.match(/\n docker:\n([\s\S]*?)\n release:\n/);
|
||||
assert.ok(dockerJob, "public release must retain its container manifest job");
|
||||
assert.match(dockerJob[1], /^ needs: \[docker-build, release, resolve\]$/m);
|
||||
assert.match(dockerJob[1], /needs\.release\.result == 'success'/);
|
||||
assert.match(
|
||||
namedStep(republish, "Require a complete published release"),
|
||||
/verify-release-inventory\.js --manifest/,
|
||||
);
|
||||
|
||||
assert.match(release, /^ docker-build:\n/m);
|
||||
assert.match(release, /^ docker:\n/m);
|
||||
|
||||
@@ -269,6 +269,7 @@ jobs:
|
||||
node .github/scripts/release-workflows.test.js
|
||||
node --test scripts/release/assemble-release-assets.test.js
|
||||
node --test scripts/release/ensure-release-assets-absent.test.js
|
||||
node --test scripts/release/verify-release-inventory.test.js
|
||||
- name: Run runtime web client tests
|
||||
# crates/tui/tests/runtime_web_client.test.mjs exercises the embedded
|
||||
# web client's event/snapshot state machine; it ran nowhere before.
|
||||
|
||||
@@ -75,21 +75,17 @@ jobs:
|
||||
echo "version=${INPUT_VERSION}"
|
||||
} >> "${GITHUB_OUTPUT}"
|
||||
echo "Resolved ${tag} -> ${sha}"
|
||||
- name: Require an existing published release
|
||||
- name: Require a complete published release
|
||||
# The inverse of release.yml's immutability guard. That one refuses to
|
||||
# run when assets exist; this one refuses to run when they do not,
|
||||
# because there is nothing to derive a channel from.
|
||||
# run when assets exist; this one refuses to run unless the published
|
||||
# release carries exactly the assets its own checksum manifest lists,
|
||||
# each fully uploaded, because a channel derived from a partial set
|
||||
# would advertise bytes that are not there. The tag may predate the
|
||||
# current inventory, so its manifest, not today's list, is the check.
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
TAG: ${{ steps.release.outputs.tag }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
count="$(gh release view "${TAG}" --repo "${GITHUB_REPOSITORY}" --json assets --jq '.assets | length')"
|
||||
if [[ "${count}" -eq 0 ]]; then
|
||||
echo "::error::${TAG} has no published assets. Run release.yml, not this recovery workflow." >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "${TAG} has ${count} published assets."
|
||||
run: node scripts/release/verify-release-inventory.js --manifest "${GITHUB_REPOSITORY}" "${TAG}"
|
||||
- name: Verify the remote tag still points at this commit
|
||||
env:
|
||||
EXPECTED_SHA: ${{ steps.release.outputs.sha }}
|
||||
|
||||
@@ -266,10 +266,15 @@ jobs:
|
||||
retention-days: 1
|
||||
overwrite: true
|
||||
|
||||
# Derived channel: the tagged/latest manifest is published only after the
|
||||
# canonical GitHub Release exists. docker-build pushes by digest (untagged)
|
||||
# and smokes each image before the release, so a broken image still stops
|
||||
# the release, but no public tag can point at a version whose GitHub
|
||||
# Release was never published.
|
||||
docker:
|
||||
timeout-minutes: 30
|
||||
needs: [docker-build, resolve]
|
||||
if: ${{ !cancelled() && needs.docker-build.result == 'success' }}
|
||||
needs: [docker-build, release, resolve]
|
||||
if: ${{ !cancelled() && needs.docker-build.result == 'success' && needs.release.result == 'success' }}
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -373,8 +378,8 @@ jobs:
|
||||
|
||||
release:
|
||||
timeout-minutes: 30
|
||||
needs: [artifacts, docker, resolve]
|
||||
if: ${{ !cancelled() && needs.artifacts.result == 'success' && needs.docker.result == 'success' }}
|
||||
needs: [artifacts, docker-build, resolve]
|
||||
if: ${{ !cancelled() && needs.artifacts.result == 'success' && needs.docker-build.result == 'success' }}
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: write
|
||||
@@ -412,14 +417,25 @@ jobs:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
TAG: ${{ needs.resolve.outputs.tag }}
|
||||
run: node repo/scripts/release/ensure-release-assets-absent.js "${GITHUB_REPOSITORY}" "${TAG}"
|
||||
# Upload into a DRAFT: 34 sequential uploads that die halfway must leave
|
||||
# nothing public (and nothing a rerun's immutability guard refuses).
|
||||
- uses: softprops/action-gh-release@efb35369e0ad2afab669f228072c1b0d510eae64 # v3
|
||||
with:
|
||||
tag_name: ${{ needs.resolve.outputs.tag }}
|
||||
files: artifacts/*
|
||||
draft: true
|
||||
prerelease: false
|
||||
body_path: release-body.md
|
||||
overwrite_files: false
|
||||
fail_on_unmatched_files: true
|
||||
- name: Verify the draft's exact asset set, then publish it at once
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
TAG: ${{ needs.resolve.outputs.tag }}
|
||||
run: |
|
||||
node repo/scripts/release/verify-release-inventory.js \
|
||||
--draft --asset-dir artifacts --publish \
|
||||
"${GITHUB_REPOSITORY}" "${TAG}"
|
||||
|
||||
npm:
|
||||
timeout-minutes: 20
|
||||
|
||||
@@ -77,6 +77,7 @@ quieter, and Fleet runs can be checked before they spend anything.
|
||||
|
||||
### Contributors
|
||||
|
||||
- **[@zhuowp](https://github.com/zhuowp)** — supplied the process-scoped PowerShell execution-policy repair adapted for Codewhale, preserving machine and user Group Policy precedence ([#6745](https://github.com/Hmbown/Codewhale/issues/6745)).
|
||||
- **[@gaord](https://github.com/gaord)** — let a client fork a thread at a named turn ([#6580](https://github.com/Hmbown/Codewhale/pull/6580)), let undo roll back files for the turn it is undoing ([#6483](https://github.com/Hmbown/Codewhale/pull/6483)), stopped resume and fork from duplicating threads and sessions ([#6406](https://github.com/Hmbown/Codewhale/pull/6406)), exposed user-defined provider routes to native clients ([#6404](https://github.com/Hmbown/Codewhale/pull/6404)), and kept a fork going when a turn lost its tool call ([#6664](https://github.com/Hmbown/Codewhale/pull/6664)).
|
||||
- **[@Lstarsky0](https://github.com/Lstarsky0)** — moved the docs/work, legal, digest and FAQ pages onto the dictionary spine ([#6405](https://github.com/Hmbown/Codewhale/pull/6405), [#6417](https://github.com/Hmbown/Codewhale/pull/6417), [#6499](https://github.com/Hmbown/Codewhale/pull/6499), [#6574](https://github.com/Hmbown/Codewhale/pull/6574)), tightened the Chinese-branching ceiling to 18 ([#6403](https://github.com/Hmbown/Codewhale/pull/6403)), and made Fleet publish without a two-link window ([#6431](https://github.com/Hmbown/Codewhale/pull/6431)). Also moved the constitution page onto the dictionary spine and kept its install link in the selected locale ([#6733](https://github.com/Hmbown/Codewhale/pull/6733)).
|
||||
- **[@aboimpinto](https://github.com/aboimpinto)** — restored a green Linux full-workspace test gate without loosening any test, twice ([#6581](https://github.com/Hmbown/Codewhale/pull/6581), [#6666](https://github.com/Hmbown/Codewhale/pull/6666)).
|
||||
|
||||
Generated
-2
@@ -786,7 +786,6 @@ dependencies = [
|
||||
"codewhale-config",
|
||||
"codewhale-core",
|
||||
"codewhale-hooks",
|
||||
"codewhale-mcp",
|
||||
"codewhale-paths",
|
||||
"codewhale-protocol",
|
||||
"codewhale-release",
|
||||
@@ -912,7 +911,6 @@ dependencies = [
|
||||
"chrono",
|
||||
"codewhale-config",
|
||||
"codewhale-hooks",
|
||||
"codewhale-mcp",
|
||||
"codewhale-protocol",
|
||||
"codewhale-state",
|
||||
"regex",
|
||||
|
||||
@@ -19,7 +19,6 @@ codewhale-agent = { path = "../agent", version = "0.10.1" }
|
||||
codewhale-config = { path = "../config", version = "0.10.1" }
|
||||
codewhale-core = { path = "../core", version = "0.10.1" }
|
||||
codewhale-hooks = { path = "../hooks", version = "0.10.1" }
|
||||
codewhale-mcp = { path = "../mcp", version = "0.10.1" }
|
||||
codewhale-paths = { path = "../paths", version = "0.10.1" }
|
||||
codewhale-protocol = { path = "../protocol", version = "0.10.1" }
|
||||
codewhale-release = { path = "../release", version = "0.10.1" }
|
||||
|
||||
@@ -17,7 +17,6 @@ use codewhale_agent::ModelRegistry;
|
||||
use codewhale_config::ConfigStore;
|
||||
use codewhale_core::Runtime;
|
||||
use codewhale_hooks::{HookDispatcher, JsonlHookSink, StdoutHookSink, UnixSocketHookSink};
|
||||
use codewhale_mcp::McpManager;
|
||||
use codewhale_protocol::{
|
||||
AppRequest, AppResponse, EventFrame, PromptRequest, PromptResponse, ResponseChannel,
|
||||
ThreadGoalClearParams, ThreadGoalGetParams, ThreadGoalSetParams, ThreadRequest, ThreadResponse,
|
||||
@@ -373,7 +372,7 @@ async fn shutdown_signal() {
|
||||
/// `RuntimeBridge::stream_turn_events` forwards only `item.delta` and the
|
||||
/// turn's completion, and there is no decision route, so approval-gated work
|
||||
/// belongs on the Runtime API (`/v1/threads/*`, `POST /v1/approvals/{id}`).
|
||||
const ADVERTISED_ROUTES: &[&str] = &["/thread", "/app", "/prompt", "/jobs", "/mcp/startup"];
|
||||
const ADVERTISED_ROUTES: &[&str] = &["/thread", "/app", "/prompt", "/jobs"];
|
||||
|
||||
fn app_router(state: AppState, cors_origins: &[String]) -> Router {
|
||||
let protected_routes = Router::new()
|
||||
@@ -391,7 +390,6 @@ fn app_router(state: AppState, cors_origins: &[String]) -> Router {
|
||||
)),
|
||||
)
|
||||
.route("/jobs", get(jobs_handler))
|
||||
.route("/mcp/startup", post(mcp_startup_handler))
|
||||
.route(
|
||||
"/v1/chat/completions",
|
||||
post(chat_completions::chat_completions_handler),
|
||||
@@ -740,15 +738,6 @@ async fn jobs_handler(State(state): State<AppState>) -> Json<AppResponse> {
|
||||
Json(runtime.app_status())
|
||||
}
|
||||
|
||||
async fn mcp_startup_handler(State(state): State<AppState>) -> Json<Value> {
|
||||
let runtime = state.runtime.read().await;
|
||||
let summary = runtime.mcp_startup().await;
|
||||
Json(json!({
|
||||
"ok": true,
|
||||
"summary": summary
|
||||
}))
|
||||
}
|
||||
|
||||
async fn app_handler(
|
||||
State(state): State<AppState>,
|
||||
Json(req): Json<AppRequest>,
|
||||
@@ -817,12 +806,7 @@ fn build_state_with_transport(
|
||||
hooks.add_sink(Arc::new(UnixSocketHookSink::new(socket_path.clone())));
|
||||
}
|
||||
|
||||
let runtime = Runtime::new(
|
||||
config.clone(),
|
||||
state_store,
|
||||
Arc::new(McpManager::default()),
|
||||
hooks,
|
||||
);
|
||||
let runtime = Runtime::new(config.clone(), state_store, hooks);
|
||||
|
||||
Ok(AppState {
|
||||
config_path,
|
||||
@@ -2658,7 +2642,7 @@ async fn process_app_request(
|
||||
data: json!({
|
||||
"routes": ADVERTISED_ROUTES,
|
||||
"config": ["get", "set", "unset", "list", "reload"],
|
||||
"events": ["response_start", "response_delta", "response_end", "tool_call_start", "tool_call_result", "mcp_startup_update", "mcp_startup_complete"],
|
||||
"events": ["response_start", "response_delta", "response_end", "tool_call_start", "tool_call_result"],
|
||||
"transport": "stdio+http",
|
||||
"config_path": state.config_path.as_ref().map(|p| p.display().to_string()),
|
||||
}),
|
||||
@@ -3095,6 +3079,40 @@ mod tests {
|
||||
assert!(!advertised.iter().any(|route| route == "/tool"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn mcp_startup_route_cannot_start_a_parallel_pool() {
|
||||
let (app, tmp) = app_with_config(Some("test-token"));
|
||||
let response = app
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
.method(Method::POST)
|
||||
.uri("/mcp/startup")
|
||||
.header(header::AUTHORIZATION, "Bearer test-token")
|
||||
.body(Body::empty())
|
||||
.expect("request"),
|
||||
)
|
||||
.await
|
||||
.expect("response");
|
||||
assert_eq!(response.status(), StatusCode::NOT_FOUND);
|
||||
|
||||
let state = build_state(Some(tmp.path().join("config.toml")), None).expect("state");
|
||||
let caps = process_app_request(&state, AppRequest::Capabilities, AppTransport::Http).await;
|
||||
assert!(
|
||||
!caps.data["routes"]
|
||||
.as_array()
|
||||
.expect("routes")
|
||||
.iter()
|
||||
.any(|route| route == "/mcp/startup")
|
||||
);
|
||||
assert!(
|
||||
!caps.data["events"]
|
||||
.as_array()
|
||||
.expect("events")
|
||||
.iter()
|
||||
.any(|event| event == "mcp_startup_update" || event == "mcp_startup_complete")
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cors_does_not_allow_arbitrary_origins() {
|
||||
let (app, _tmp) = app_with_config(Some("test-token"));
|
||||
@@ -3953,7 +3971,7 @@ mod tests {
|
||||
let state = build_state(Some(config_path.clone()), None).expect("state");
|
||||
*state.runtime_bridge.lock().await = Some(sentinel_bridge());
|
||||
|
||||
// A long reader (e.g. `/mcp/startup`) holds the runtime, so the set
|
||||
// A concurrent reader holds the runtime, so the set
|
||||
// saves to disk and then waits to propagate.
|
||||
let runtime_reader = state.runtime.read().await;
|
||||
let request = process_app_request(
|
||||
|
||||
@@ -17,7 +17,6 @@ serde.workspace = true
|
||||
thiserror.workspace = true
|
||||
codewhale-config = { path = "../config", version = "0.10.1" }
|
||||
codewhale-hooks = { path = "../hooks", version = "0.10.1" }
|
||||
codewhale-mcp = { path = "../mcp", version = "0.10.1" }
|
||||
codewhale-protocol = { path = "../protocol", version = "0.10.1" }
|
||||
codewhale-state = { path = "../state", version = "0.10.1" }
|
||||
regex = "1.11"
|
||||
|
||||
+3
-66
@@ -16,14 +16,10 @@ pub use context_reference::{
|
||||
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::Arc;
|
||||
|
||||
use anyhow::{Result, anyhow};
|
||||
use codewhale_config::{ConfigToml, ProviderKind};
|
||||
use codewhale_hooks::{HookDispatcher, HookEvent};
|
||||
use codewhale_mcp::{
|
||||
McpManager, McpStartupCompleteEvent, McpStartupStatus as McpManagerStartupStatus,
|
||||
};
|
||||
use codewhale_hooks::HookDispatcher;
|
||||
use codewhale_protocol::{
|
||||
AppResponse, EventFrame, ResponseChannel, Status, Thread, ThreadForkParams, ThreadGoal,
|
||||
ThreadGoalClearParams, ThreadGoalGetParams, ThreadGoalProgressParams, ThreadGoalSetParams,
|
||||
@@ -922,7 +918,7 @@ impl ThreadManager {
|
||||
}
|
||||
}
|
||||
|
||||
/// Top-level headless runtime combining config, threads, MCP, and hooks.
|
||||
/// Headless bookkeeping combining config, threads, jobs, and hooks.
|
||||
///
|
||||
/// It does not execute tools and holds no approval policy: the Engine behind
|
||||
/// the app-server's runtime bridge is the only tool and approval authority.
|
||||
@@ -931,8 +927,6 @@ pub struct Runtime {
|
||||
pub config: ConfigToml,
|
||||
/// Manages conversation thread lifecycle.
|
||||
pub thread_manager: ThreadManager,
|
||||
/// Manager for MCP server connections.
|
||||
pub mcp_manager: Arc<McpManager>,
|
||||
/// Dispatcher for lifecycle hooks.
|
||||
pub hooks: HookDispatcher,
|
||||
/// Manager for background job lifecycle.
|
||||
@@ -941,12 +935,7 @@ pub struct Runtime {
|
||||
|
||||
impl Runtime {
|
||||
/// Constructs a new `Runtime`, loading existing jobs from the state store.
|
||||
pub fn new(
|
||||
config: ConfigToml,
|
||||
state: StateStore,
|
||||
mcp_manager: Arc<McpManager>,
|
||||
hooks: HookDispatcher,
|
||||
) -> Self {
|
||||
pub fn new(config: ConfigToml, state: StateStore, hooks: HookDispatcher) -> Self {
|
||||
let mut jobs = JobManager::default();
|
||||
if let Err(e) = jobs.load_from_store(&state) {
|
||||
tracing::warn!("Failed to load job store, starting with empty job list: {e}");
|
||||
@@ -954,7 +943,6 @@ impl Runtime {
|
||||
Self {
|
||||
config,
|
||||
thread_manager: ThreadManager::new(state),
|
||||
mcp_manager,
|
||||
hooks,
|
||||
jobs,
|
||||
}
|
||||
@@ -1308,55 +1296,6 @@ impl Runtime {
|
||||
}
|
||||
}
|
||||
|
||||
/// Starts all configured MCP servers and emits startup events via hooks.
|
||||
pub async fn mcp_startup(&self) -> McpStartupCompleteEvent {
|
||||
let mut updates = Vec::new();
|
||||
let summary = self.mcp_manager.start_all(|update| {
|
||||
updates.push(update);
|
||||
});
|
||||
for update in updates {
|
||||
let status = match update.status {
|
||||
McpManagerStartupStatus::Starting => codewhale_protocol::McpStartupStatus::Starting,
|
||||
McpManagerStartupStatus::Ready => codewhale_protocol::McpStartupStatus::Ready,
|
||||
McpManagerStartupStatus::Failed { error } => {
|
||||
codewhale_protocol::McpStartupStatus::Failed { error }
|
||||
}
|
||||
McpManagerStartupStatus::Cancelled => {
|
||||
codewhale_protocol::McpStartupStatus::Cancelled
|
||||
}
|
||||
};
|
||||
self.hooks
|
||||
.emit(HookEvent::GenericEventFrame {
|
||||
frame: Box::new(EventFrame::McpStartupUpdate {
|
||||
update: codewhale_protocol::McpStartupUpdateEvent {
|
||||
server_name: update.server_name,
|
||||
status,
|
||||
},
|
||||
}),
|
||||
})
|
||||
.await;
|
||||
}
|
||||
self.hooks
|
||||
.emit(HookEvent::GenericEventFrame {
|
||||
frame: Box::new(EventFrame::McpStartupComplete {
|
||||
summary: codewhale_protocol::McpStartupCompleteEvent {
|
||||
ready: summary.ready.clone(),
|
||||
failed: summary
|
||||
.failed
|
||||
.iter()
|
||||
.map(|f| codewhale_protocol::McpStartupFailure {
|
||||
server_name: f.server_name.clone(),
|
||||
error: f.error.clone(),
|
||||
})
|
||||
.collect(),
|
||||
cancelled: summary.cancelled.clone(),
|
||||
},
|
||||
}),
|
||||
})
|
||||
.await;
|
||||
summary
|
||||
}
|
||||
|
||||
/// Returns the current application status including all jobs and their history.
|
||||
pub fn app_status(&self) -> AppResponse {
|
||||
let jobs = self.jobs.list();
|
||||
@@ -2591,7 +2530,6 @@ mod tests {
|
||||
let mut runtime = Runtime::new(
|
||||
ConfigToml::default(),
|
||||
temp_core_state("archive-unknown"),
|
||||
Arc::new(McpManager::default()),
|
||||
HookDispatcher::default(),
|
||||
);
|
||||
for request in [
|
||||
@@ -2641,7 +2579,6 @@ mod tests {
|
||||
let mut runtime = Runtime::new(
|
||||
ConfigToml::default(),
|
||||
temp_core_state("message-refused"),
|
||||
Arc::new(McpManager::default()),
|
||||
HookDispatcher::default(),
|
||||
);
|
||||
let spawned = runtime
|
||||
|
||||
@@ -1535,6 +1535,10 @@
|
||||
"SessionsSortSize": "mida",
|
||||
"SessionsSearchPrompt": "Cerca: escriu per filtrar, Enter per aplicar",
|
||||
"SessionsDeleteFailed": "L'eliminació ha fallat: {error}",
|
||||
"SessionsDeleteOpenHere": "No es pot eliminar la sessió oberta aquí. Canvia primer a una altra sessió.",
|
||||
"SessionsDeleteOpenElsewhere": "No es pot eliminar: aquesta sessió està oberta en una altra finestra de Codewhale.",
|
||||
"SessionSaveFailed": "No s'ha pogut desar la sessió {id} ({error}). És possible que la feina recent no estigui desada.",
|
||||
"SessionSaveFailedAtExit": "codewhale: l'últim desament de la sessió {id} ha fallat ({error}); és possible que la feina recent no s'hagi desat.",
|
||||
"SessionsDeleted": "Sessió {id} eliminada",
|
||||
"SessionsNoSelection": "Cap sessió seleccionada",
|
||||
"SessionsTitleLength": "El títol ha de tenir entre 1 i 100 caràcters",
|
||||
|
||||
@@ -1535,6 +1535,10 @@
|
||||
"SessionsSortSize": "Größe",
|
||||
"SessionsSearchPrompt": "Suche: tippen zum Filtern, Enter zum Anwenden",
|
||||
"SessionsDeleteFailed": "Löschen fehlgeschlagen: {error}",
|
||||
"SessionsDeleteOpenHere": "Die hier geöffnete Sitzung kann nicht gelöscht werden. Wechsle zuerst zu einer anderen Sitzung.",
|
||||
"SessionsDeleteOpenElsewhere": "Löschen nicht möglich: Diese Sitzung ist in einem anderen Codewhale-Fenster geöffnet.",
|
||||
"SessionSaveFailed": "Sitzung {id} konnte nicht gespeichert werden ({error}). Neuere Arbeit ist möglicherweise nicht gespeichert.",
|
||||
"SessionSaveFailedAtExit": "codewhale: Das letzte Speichern der Sitzung {id} ist fehlgeschlagen ({error}); neuere Arbeit wurde möglicherweise nicht gespeichert.",
|
||||
"SessionsDeleted": "Sitzung {id} gelöscht",
|
||||
"SessionsNoSelection": "Keine Sitzung ausgewählt",
|
||||
"SessionsTitleLength": "Titel muss 1–100 Zeichen haben",
|
||||
|
||||
@@ -1558,6 +1558,10 @@
|
||||
"SessionsSortSize": "size",
|
||||
"SessionsSearchPrompt": "Search: type, Enter to apply",
|
||||
"SessionsDeleteFailed": "Could not delete: {error}",
|
||||
"SessionsDeleteOpenHere": "Can't delete the session open here. Switch to another session first.",
|
||||
"SessionsDeleteOpenElsewhere": "Can't delete: this session is open in another Codewhale window.",
|
||||
"SessionSaveFailed": "Session {id} could not be saved ({error}). Recent work may not be saved.",
|
||||
"SessionSaveFailedAtExit": "codewhale: the latest save of session {id} failed ({error}); recent work may not have been saved.",
|
||||
"SessionsDeleted": "Deleted {id}",
|
||||
"SessionsNoSelection": "No session selected",
|
||||
"SessionsTitleLength": "Title must be 1–100 characters",
|
||||
|
||||
@@ -1558,6 +1558,10 @@
|
||||
"SessionsSortSize": "tamaño",
|
||||
"SessionsSearchPrompt": "Buscar: escribe para filtrar, Enter para aplicar",
|
||||
"SessionsDeleteFailed": "Error al eliminar: {error}",
|
||||
"SessionsDeleteOpenHere": "No se puede eliminar la sesión abierta aquí. Cambia primero a otra sesión.",
|
||||
"SessionsDeleteOpenElsewhere": "No se puede eliminar: esta sesión está abierta en otra ventana de Codewhale.",
|
||||
"SessionSaveFailed": "No se pudo guardar la sesión {id} ({error}). Es posible que el trabajo reciente no esté guardado.",
|
||||
"SessionSaveFailedAtExit": "codewhale: el último guardado de la sesión {id} falló ({error}); es posible que el trabajo reciente no se haya guardado.",
|
||||
"SessionsDeleted": "Sesión {id} eliminada",
|
||||
"SessionsNoSelection": "Ninguna sesión seleccionada",
|
||||
"SessionsTitleLength": "El título debe tener entre 1 y 100 caracteres",
|
||||
|
||||
@@ -1535,6 +1535,10 @@
|
||||
"SessionsSortSize": "taille",
|
||||
"SessionsSearchPrompt": "Recherche : tapez pour filtrer, Enter pour appliquer",
|
||||
"SessionsDeleteFailed": "Échec de la suppression : {error}",
|
||||
"SessionsDeleteOpenHere": "Impossible de supprimer la session ouverte ici. Passez d'abord à une autre session.",
|
||||
"SessionsDeleteOpenElsewhere": "Suppression impossible : cette session est ouverte dans une autre fenêtre Codewhale.",
|
||||
"SessionSaveFailed": "Impossible d'enregistrer la session {id} ({error}). Le travail récent n'est peut-être pas enregistré.",
|
||||
"SessionSaveFailedAtExit": "codewhale : le dernier enregistrement de la session {id} a échoué ({error}) ; le travail récent n'a peut-être pas été enregistré.",
|
||||
"SessionsDeleted": "Session {id} supprimée",
|
||||
"SessionsNoSelection": "Aucune session sélectionnée",
|
||||
"SessionsTitleLength": "Le titre doit faire 1 à 100 caractères",
|
||||
|
||||
@@ -1535,6 +1535,10 @@
|
||||
"SessionsSortSize": "आकार",
|
||||
"SessionsSearchPrompt": "खोज: फ़िल्टर के लिए टाइप करें, लागू के लिए Enter",
|
||||
"SessionsDeleteFailed": "हटाना विफल: {error}",
|
||||
"SessionsDeleteOpenHere": "यहाँ खुला सत्र हटाया नहीं जा सकता। पहले किसी दूसरे सत्र पर जाएँ।",
|
||||
"SessionsDeleteOpenElsewhere": "हटाया नहीं जा सकता: यह सत्र किसी दूसरी Codewhale विंडो में खुला है।",
|
||||
"SessionSaveFailed": "सत्र {id} सहेजा नहीं जा सका ({error})। हाल का काम शायद सहेजा नहीं गया है।",
|
||||
"SessionSaveFailedAtExit": "codewhale: सत्र {id} को पिछली बार सहेजना विफल रहा ({error}); हाल का काम शायद सहेजा नहीं गया।",
|
||||
"SessionsDeleted": "सत्र {id} हटाया गया",
|
||||
"SessionsNoSelection": "कोई सत्र चुना नहीं गया",
|
||||
"SessionsTitleLength": "शीर्षक 1–100 अक्षरों का होना चाहिए",
|
||||
|
||||
@@ -1535,6 +1535,10 @@
|
||||
"SessionsSortSize": "ukuran",
|
||||
"SessionsSearchPrompt": "Cari: ketik untuk memfilter, Enter untuk menerapkan",
|
||||
"SessionsDeleteFailed": "Hapus gagal: {error}",
|
||||
"SessionsDeleteOpenHere": "Sesi yang terbuka di sini tidak dapat dihapus. Beralih ke sesi lain terlebih dahulu.",
|
||||
"SessionsDeleteOpenElsewhere": "Tidak dapat dihapus: sesi ini terbuka di jendela Codewhale lain.",
|
||||
"SessionSaveFailed": "Sesi {id} tidak dapat disimpan ({error}). Pekerjaan terbaru mungkin belum tersimpan.",
|
||||
"SessionSaveFailedAtExit": "codewhale: penyimpanan terakhir sesi {id} gagal ({error}); pekerjaan terbaru mungkin belum tersimpan.",
|
||||
"SessionsDeleted": "Sesi {id} dihapus",
|
||||
"SessionsNoSelection": "Tidak ada sesi terpilih",
|
||||
"SessionsTitleLength": "Judul harus 1–100 karakter",
|
||||
|
||||
@@ -1558,6 +1558,10 @@
|
||||
"SessionsSortSize": "サイズ",
|
||||
"SessionsSearchPrompt": "検索: 入力して絞り込み、Enter で適用",
|
||||
"SessionsDeleteFailed": "削除に失敗しました: {error}",
|
||||
"SessionsDeleteOpenHere": "ここで開いているセッションは削除できません。先に別のセッションに切り替えてください。",
|
||||
"SessionsDeleteOpenElsewhere": "削除できません: このセッションは別の Codewhale ウィンドウで開かれています。",
|
||||
"SessionSaveFailed": "セッション {id} を保存できませんでした ({error})。最近の作業が保存されていない可能性があります。",
|
||||
"SessionSaveFailedAtExit": "codewhale: セッション {id} の最後の保存に失敗しました ({error})。最近の作業が保存されていない可能性があります。",
|
||||
"SessionsDeleted": "セッション {id} を削除しました",
|
||||
"SessionsNoSelection": "セッションが選択されていません",
|
||||
"SessionsTitleLength": "タイトルは 1〜100 文字にしてください",
|
||||
|
||||
@@ -1558,6 +1558,10 @@
|
||||
"SessionsSortSize": "크기",
|
||||
"SessionsSearchPrompt": "검색: 입력하여 필터, Enter로 적용",
|
||||
"SessionsDeleteFailed": "삭제 실패: {error}",
|
||||
"SessionsDeleteOpenHere": "여기에서 열려 있는 세션은 삭제할 수 없습니다. 먼저 다른 세션으로 전환하세요.",
|
||||
"SessionsDeleteOpenElsewhere": "삭제할 수 없음: 이 세션은 다른 Codewhale 창에서 열려 있습니다.",
|
||||
"SessionSaveFailed": "세션 {id}을(를) 저장하지 못했습니다 ({error}). 최근 작업이 저장되지 않았을 수 있습니다.",
|
||||
"SessionSaveFailedAtExit": "codewhale: 세션 {id}의 마지막 저장에 실패했습니다 ({error}). 최근 작업이 저장되지 않았을 수 있습니다.",
|
||||
"SessionsDeleted": "세션 {id} 삭제됨",
|
||||
"SessionsNoSelection": "선택된 세션 없음",
|
||||
"SessionsTitleLength": "제목은 1–100자여야 합니다",
|
||||
|
||||
@@ -1558,6 +1558,10 @@
|
||||
"SessionsSortSize": "tamanho",
|
||||
"SessionsSearchPrompt": "Buscar: digite para filtrar, Enter para aplicar",
|
||||
"SessionsDeleteFailed": "Falha ao excluir: {error}",
|
||||
"SessionsDeleteOpenHere": "Não é possível excluir a sessão aberta aqui. Mude para outra sessão primeiro.",
|
||||
"SessionsDeleteOpenElsewhere": "Não é possível excluir: esta sessão está aberta em outra janela do Codewhale.",
|
||||
"SessionSaveFailed": "Não foi possível salvar a sessão {id} ({error}). O trabalho recente pode não estar salvo.",
|
||||
"SessionSaveFailedAtExit": "codewhale: o último salvamento da sessão {id} falhou ({error}); o trabalho recente pode não ter sido salvo.",
|
||||
"SessionsDeleted": "Sessão {id} excluída",
|
||||
"SessionsNoSelection": "Nenhuma sessão selecionada",
|
||||
"SessionsTitleLength": "O título deve ter de 1 a 100 caracteres",
|
||||
|
||||
@@ -1535,6 +1535,10 @@
|
||||
"SessionsSortSize": "размер",
|
||||
"SessionsSearchPrompt": "Поиск: введите для фильтра, Enter — применить",
|
||||
"SessionsDeleteFailed": "Ошибка удаления: {error}",
|
||||
"SessionsDeleteOpenHere": "Нельзя удалить сессию, открытую здесь. Сначала переключитесь на другую сессию.",
|
||||
"SessionsDeleteOpenElsewhere": "Удаление невозможно: эта сессия открыта в другом окне Codewhale.",
|
||||
"SessionSaveFailed": "Не удалось сохранить сессию {id} ({error}). Последние изменения могут быть не сохранены.",
|
||||
"SessionSaveFailedAtExit": "codewhale: последнее сохранение сессии {id} не удалось ({error}); последние изменения могли не сохраниться.",
|
||||
"SessionsDeleted": "Сессия {id} удалена",
|
||||
"SessionsNoSelection": "Сессия не выбрана",
|
||||
"SessionsTitleLength": "Заголовок должен быть от 1 до 100 символов",
|
||||
|
||||
@@ -1535,6 +1535,10 @@
|
||||
"SessionsSortSize": "розмір",
|
||||
"SessionsSearchPrompt": "Пошук: вводьте для фільтрації, Enter — застосувати",
|
||||
"SessionsDeleteFailed": "Видалення не вдалося: {error}",
|
||||
"SessionsDeleteOpenHere": "Не можна видалити сесію, відкриту тут. Спочатку перейдіть до іншої сесії.",
|
||||
"SessionsDeleteOpenElsewhere": "Видалення неможливе: ця сесія відкрита в іншому вікні Codewhale.",
|
||||
"SessionSaveFailed": "Не вдалося зберегти сесію {id} ({error}). Останні зміни можуть бути не збережені.",
|
||||
"SessionSaveFailedAtExit": "codewhale: останнє збереження сесії {id} не вдалося ({error}); останні зміни могли не зберегтися.",
|
||||
"SessionsDeleted": "Сесію {id} видалено",
|
||||
"SessionsNoSelection": "Сесію не вибрано",
|
||||
"SessionsTitleLength": "Назва має містити 1–100 символів",
|
||||
|
||||
@@ -1558,6 +1558,10 @@
|
||||
"SessionsSortSize": "kích thước",
|
||||
"SessionsSearchPrompt": "Tìm: gõ để lọc, Enter để áp dụng",
|
||||
"SessionsDeleteFailed": "Xóa thất bại: {error}",
|
||||
"SessionsDeleteOpenHere": "Không thể xóa phiên đang mở ở đây. Hãy chuyển sang phiên khác trước.",
|
||||
"SessionsDeleteOpenElsewhere": "Không thể xóa: phiên này đang mở trong một cửa sổ Codewhale khác.",
|
||||
"SessionSaveFailed": "Không thể lưu phiên {id} ({error}). Công việc gần đây có thể chưa được lưu.",
|
||||
"SessionSaveFailedAtExit": "codewhale: lần lưu gần nhất của phiên {id} thất bại ({error}); công việc gần đây có thể chưa được lưu.",
|
||||
"SessionsDeleted": "Đã xóa phiên {id}",
|
||||
"SessionsNoSelection": "Chưa chọn phiên",
|
||||
"SessionsTitleLength": "Tiêu đề phải từ 1–100 ký tự",
|
||||
|
||||
@@ -1558,6 +1558,10 @@
|
||||
"SessionsSortSize": "大小",
|
||||
"SessionsSearchPrompt": "搜索:输入筛选,Enter 应用",
|
||||
"SessionsDeleteFailed": "删除失败:{error}",
|
||||
"SessionsDeleteOpenHere": "无法删除此处打开的会话。请先切换到其他会话。",
|
||||
"SessionsDeleteOpenElsewhere": "无法删除:此会话已在另一个 Codewhale 窗口中打开。",
|
||||
"SessionSaveFailed": "无法保存会话 {id}({error})。最近的工作可能尚未保存。",
|
||||
"SessionSaveFailedAtExit": "codewhale: 会话 {id} 的最近一次保存失败({error});最近的工作可能尚未保存。",
|
||||
"SessionsDeleted": "已删除会话 {id}",
|
||||
"SessionsNoSelection": "未选择会话",
|
||||
"SessionsTitleLength": "标题长度须为 1–100 个字符",
|
||||
@@ -2434,7 +2438,5 @@
|
||||
"ModelPickerPinReorderFailed": "无法调整固定顺序:{error}",
|
||||
"ModelPickerReadinessRefreshed": "已刷新模型就绪状态 · 已重建目录行",
|
||||
"ModelPickerOpenToRefresh": "打开 /model 以刷新就绪状态和目录",
|
||||
"ModelPickerPinnedChip": "已固定",
|
||||
"TranscriptThought": "思考完毕",
|
||||
"TranscriptThoughtFor": "思考用时 {duration}"
|
||||
"ModelPickerPinnedChip": "已固定"
|
||||
}
|
||||
@@ -1556,6 +1556,10 @@
|
||||
"SessionsSortSize": "大小",
|
||||
"SessionsSearchPrompt": "搜尋:輸入篩選,Enter 套用",
|
||||
"SessionsDeleteFailed": "刪除失敗:{error}",
|
||||
"SessionsDeleteOpenHere": "無法刪除此處開啟的工作階段。請先切換到其他工作階段。",
|
||||
"SessionsDeleteOpenElsewhere": "無法刪除:此工作階段已在另一個 Codewhale 視窗中開啟。",
|
||||
"SessionSaveFailed": "無法儲存工作階段 {id}({error})。最近的工作可能尚未儲存。",
|
||||
"SessionSaveFailedAtExit": "codewhale: 工作階段 {id} 的最近一次儲存失敗({error});最近的工作可能尚未儲存。",
|
||||
"SessionsDeleted": "已刪除工作階段 {id}",
|
||||
"SessionsNoSelection": "未選擇工作階段",
|
||||
"SessionsTitleLength": "標題長度須為 1–100 個字元",
|
||||
|
||||
@@ -1940,6 +1940,10 @@ pub enum MessageId {
|
||||
SessionsSortSize,
|
||||
SessionsSearchPrompt,
|
||||
SessionsDeleteFailed,
|
||||
SessionsDeleteOpenHere,
|
||||
SessionsDeleteOpenElsewhere,
|
||||
SessionSaveFailed,
|
||||
SessionSaveFailedAtExit,
|
||||
SessionsDeleted,
|
||||
SessionsNoSelection,
|
||||
SessionsTitleLength,
|
||||
@@ -4415,6 +4419,10 @@ pub const ALL_MESSAGE_IDS: &[MessageId] = &[
|
||||
MessageId::SessionsSortSize,
|
||||
MessageId::SessionsSearchPrompt,
|
||||
MessageId::SessionsDeleteFailed,
|
||||
MessageId::SessionsDeleteOpenHere,
|
||||
MessageId::SessionsDeleteOpenElsewhere,
|
||||
MessageId::SessionSaveFailed,
|
||||
MessageId::SessionSaveFailedAtExit,
|
||||
MessageId::SessionsDeleted,
|
||||
MessageId::SessionsNoSelection,
|
||||
MessageId::SessionsTitleLength,
|
||||
|
||||
@@ -1,10 +1,22 @@
|
||||
//! Compiled-regex cache for patterns a user or model supplies (tool search,
|
||||
//! purge). Those patterns are untrusted input, so compilation is bounded:
|
||||
//! the pattern length is capped, the compiled program and lazy DFA have
|
||||
//! explicit size limits, compilation runs outside the cache lock, and a
|
||||
//! rejected pattern is remembered so repeating it costs a lookup.
|
||||
|
||||
use std::num::NonZeroUsize;
|
||||
use std::sync::{Mutex, OnceLock};
|
||||
|
||||
use lru::LruCache;
|
||||
use regex::Regex;
|
||||
use regex::{Regex, RegexBuilder};
|
||||
|
||||
const DEFAULT_USER_REGEX_CACHE_CAPACITY: usize = 64;
|
||||
/// Longest pattern accepted, in bytes. Search and purge patterns are short.
|
||||
pub const MAX_USER_REGEX_PATTERN_BYTES: usize = 4 * 1024;
|
||||
/// Compiled program limit (the `regex` default is 10 MiB).
|
||||
const USER_REGEX_SIZE_LIMIT: usize = 1024 * 1024;
|
||||
/// Lazy DFA cache limit per matcher.
|
||||
const USER_REGEX_DFA_SIZE_LIMIT: usize = 1024 * 1024;
|
||||
|
||||
static USER_REGEX_CACHE: OnceLock<UserRegexCache> = OnceLock::new();
|
||||
|
||||
@@ -16,8 +28,23 @@ fn user_regex_cache() -> &'static UserRegexCache {
|
||||
USER_REGEX_CACHE.get_or_init(UserRegexCache::new)
|
||||
}
|
||||
|
||||
fn compile_bounded(pattern: &str) -> Result<Regex, regex::Error> {
|
||||
if pattern.len() > MAX_USER_REGEX_PATTERN_BYTES {
|
||||
return Err(regex::Error::Syntax(format!(
|
||||
"pattern is {} bytes; the limit is {MAX_USER_REGEX_PATTERN_BYTES}",
|
||||
pattern.len()
|
||||
)));
|
||||
}
|
||||
RegexBuilder::new(pattern)
|
||||
.size_limit(USER_REGEX_SIZE_LIMIT)
|
||||
.dfa_size_limit(USER_REGEX_DFA_SIZE_LIMIT)
|
||||
.build()
|
||||
}
|
||||
|
||||
struct UserRegexCache {
|
||||
inner: Mutex<LruCache<String, Regex>>,
|
||||
/// Compiled patterns and rejected ones alike, so a repeated bad pattern
|
||||
/// is not recompiled.
|
||||
inner: Mutex<LruCache<String, Result<Regex, regex::Error>>>,
|
||||
}
|
||||
|
||||
impl UserRegexCache {
|
||||
@@ -34,16 +61,22 @@ impl UserRegexCache {
|
||||
}
|
||||
|
||||
fn compile(&self, pattern: &str) -> Result<Regex, regex::Error> {
|
||||
let Ok(mut cache) = self.inner.lock() else {
|
||||
return Regex::new(pattern);
|
||||
};
|
||||
if let Some(regex) = cache.get(pattern) {
|
||||
return Ok(regex.clone());
|
||||
// An over-long pattern is refused before it is hashed or stored.
|
||||
if pattern.len() > MAX_USER_REGEX_PATTERN_BYTES {
|
||||
return compile_bounded(pattern);
|
||||
}
|
||||
|
||||
let regex = Regex::new(pattern)?;
|
||||
cache.put(pattern.to_string(), regex.clone());
|
||||
Ok(regex)
|
||||
if let Ok(mut cache) = self.inner.lock()
|
||||
&& let Some(cached) = cache.get(pattern)
|
||||
{
|
||||
return cached.clone();
|
||||
}
|
||||
// Compile without the lock: one slow pattern must not stall every
|
||||
// other caller. Two racing misses both compile; the result is equal.
|
||||
let compiled = compile_bounded(pattern);
|
||||
if let Ok(mut cache) = self.inner.lock() {
|
||||
cache.put(pattern.to_string(), compiled.clone());
|
||||
}
|
||||
compiled
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -88,11 +121,31 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn invalid_pattern_is_not_cached() {
|
||||
fn invalid_pattern_is_remembered_as_invalid() {
|
||||
let cache = UserRegexCache::with_capacity(NonZeroUsize::new(2).unwrap());
|
||||
|
||||
assert!(cache.compile("[").is_err());
|
||||
assert!(cache.contains("["));
|
||||
assert!(cache.compile("[").is_err());
|
||||
|
||||
assert_eq!(cache.len(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn oversized_patterns_are_refused_without_being_cached() {
|
||||
let cache = UserRegexCache::with_capacity(NonZeroUsize::new(2).unwrap());
|
||||
|
||||
let long = "a".repeat(MAX_USER_REGEX_PATTERN_BYTES + 1);
|
||||
assert!(cache.compile(&long).is_err());
|
||||
assert_eq!(cache.len(), 0);
|
||||
|
||||
// Within the length cap, a pattern whose compiled form exceeds the
|
||||
// program limit is refused rather than built.
|
||||
let explosive = r"(?:\w{100}){100}";
|
||||
assert!(explosive.len() <= MAX_USER_REGEX_PATTERN_BYTES);
|
||||
assert!(matches!(
|
||||
cache.compile(explosive),
|
||||
Err(regex::Error::CompiledTooBig(_))
|
||||
));
|
||||
}
|
||||
}
|
||||
@@ -23,10 +23,20 @@
|
||||
//! supposed to be invisible. If a future feature ever needs per-engine
|
||||
//! retry surfaces, swap this for an `Arc<RwLock<...>>` carried on the
|
||||
//! `EngineHandle`; the public API stays the same.
|
||||
//!
|
||||
//! The `Retry-After` pause is process-wide too, but keyed by provider scope:
|
||||
//! every request to the rate-limited route waits it out, and requests to any
|
||||
//! other route (another provider, a local runtime) do not.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::sync::{Mutex, OnceLock};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
/// Rate-limit pause deadlines, one per provider scope (see
|
||||
/// [`note_rate_limit`]). A 429 from one provider must not stall requests to
|
||||
/// another provider, a local runtime, or a sub-agent on a different route.
|
||||
type RateLimitPauses = HashMap<String, Instant>;
|
||||
|
||||
/// One in-flight retry attempt. `deadline` is the wall-clock time the
|
||||
/// next request will fire — the UI subtracts `Instant::now()` from it
|
||||
/// to render a live countdown.
|
||||
@@ -97,10 +107,10 @@ fn with_state<R>(f: impl FnOnce(&mut RetryState) -> R) -> R {
|
||||
}
|
||||
|
||||
#[cfg(not(any(test, all(feature = "test-thread-scoped-state", debug_assertions))))]
|
||||
fn with_rate_limit<R>(f: impl FnOnce(&mut Option<Instant>) -> R) -> R {
|
||||
static STATE: OnceLock<Mutex<Option<Instant>>> = OnceLock::new();
|
||||
fn with_rate_limit<R>(f: impl FnOnce(&mut RateLimitPauses) -> R) -> R {
|
||||
static STATE: OnceLock<Mutex<RateLimitPauses>> = OnceLock::new();
|
||||
let mut state = STATE
|
||||
.get_or_init(|| Mutex::new(None))
|
||||
.get_or_init(|| Mutex::new(HashMap::new()))
|
||||
.lock()
|
||||
.unwrap_or_else(|error| error.into_inner());
|
||||
f(&mut state)
|
||||
@@ -140,13 +150,11 @@ fn with_state<R>(f: impl FnOnce(&mut RetryState) -> R) -> R {
|
||||
}
|
||||
|
||||
#[cfg(any(test, all(feature = "test-thread-scoped-state", debug_assertions)))]
|
||||
fn with_rate_limit<R>(f: impl FnOnce(&mut Option<Instant>) -> R) -> R {
|
||||
#[allow(clippy::type_complexity)]
|
||||
static STATE: OnceLock<
|
||||
Mutex<std::collections::HashMap<std::thread::ThreadId, Option<Instant>>>,
|
||||
> = OnceLock::new();
|
||||
fn with_rate_limit<R>(f: impl FnOnce(&mut RateLimitPauses) -> R) -> R {
|
||||
static STATE: OnceLock<Mutex<HashMap<std::thread::ThreadId, RateLimitPauses>>> =
|
||||
OnceLock::new();
|
||||
let mut by_thread = STATE
|
||||
.get_or_init(|| Mutex::new(std::collections::HashMap::new()))
|
||||
.get_or_init(|| Mutex::new(HashMap::new()))
|
||||
.lock()
|
||||
.unwrap_or_else(|error| error.into_inner());
|
||||
f(by_thread.entry(std::thread::current().id()).or_default())
|
||||
@@ -160,26 +168,32 @@ pub fn snapshot() -> RetryState {
|
||||
with_state(|state| state.clone())
|
||||
}
|
||||
|
||||
/// Extend the provider-wide rate-limit pause window. This is separate from
|
||||
/// the footer banner so one successful concurrent request cannot clear another
|
||||
/// request's active `Retry-After` window.
|
||||
pub fn note_rate_limit(delay: Duration) {
|
||||
let deadline = Instant::now() + delay;
|
||||
with_rate_limit(|current| {
|
||||
if current.is_none_or(|existing| existing < deadline) {
|
||||
*current = Some(deadline);
|
||||
/// Extend the rate-limit pause window for one provider `scope` (the caller's
|
||||
/// route identity and host). This is separate from the footer banner so one
|
||||
/// successful concurrent request cannot clear another request's active
|
||||
/// `Retry-After` window, and it is keyed so a 429 from one provider never
|
||||
/// pauses requests to another. Expired scopes are dropped here, so the map
|
||||
/// holds at most the providers currently rate limited.
|
||||
pub fn note_rate_limit(scope: &str, delay: Duration) {
|
||||
let now = Instant::now();
|
||||
let deadline = now + delay;
|
||||
with_rate_limit(|pauses| {
|
||||
pauses.retain(|_, existing| *existing > now);
|
||||
let current = pauses.entry(scope.to_string()).or_insert(deadline);
|
||||
if *current < deadline {
|
||||
*current = deadline;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/// Remaining provider-wide rate-limit pause, if any.
|
||||
/// Remaining rate-limit pause for `scope`, if any.
|
||||
#[must_use]
|
||||
pub fn rate_limit_remaining() -> Option<Duration> {
|
||||
pub fn rate_limit_remaining(scope: &str) -> Option<Duration> {
|
||||
let now = Instant::now();
|
||||
with_rate_limit(|current| match *current {
|
||||
with_rate_limit(|pauses| match pauses.get(scope).copied() {
|
||||
Some(deadline) if deadline > now => Some(deadline.duration_since(now)),
|
||||
Some(_) => {
|
||||
*current = None;
|
||||
pauses.remove(scope);
|
||||
None
|
||||
}
|
||||
None => None,
|
||||
@@ -220,9 +234,10 @@ pub fn clear() {
|
||||
with_state(|state| *state = RetryState::Idle);
|
||||
}
|
||||
|
||||
/// Drop every scope's rate-limit pause.
|
||||
#[cfg(any(test, feature = "test-support"))]
|
||||
pub fn clear_rate_limit() {
|
||||
with_rate_limit(|current| *current = None);
|
||||
with_rate_limit(HashMap::clear);
|
||||
}
|
||||
|
||||
/// Test helper: serialize tests that touch the global state so cargo's
|
||||
@@ -300,13 +315,29 @@ mod tests {
|
||||
#[test]
|
||||
fn rate_limit_deadline_survives_banner_clear() {
|
||||
let _g = setup();
|
||||
note_rate_limit(Duration::from_secs(5));
|
||||
note_rate_limit("provider-a", Duration::from_secs(5));
|
||||
start(1, Duration::from_secs(5), "rate limited");
|
||||
succeeded();
|
||||
assert!(
|
||||
rate_limit_remaining().is_some(),
|
||||
"provider-wide rate limit pause must not be cleared by an unrelated success"
|
||||
rate_limit_remaining("provider-a").is_some(),
|
||||
"provider rate limit pause must not be cleared by an unrelated success"
|
||||
);
|
||||
clear_rate_limit();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn one_providers_rate_limit_does_not_pause_another() {
|
||||
let _g = setup();
|
||||
note_rate_limit("[email protected]", Duration::from_secs(30));
|
||||
assert!(rate_limit_remaining("[email protected]").is_some());
|
||||
assert_eq!(rate_limit_remaining("[email protected]:11434"), None);
|
||||
|
||||
// A shorter Retry-After never shortens an existing window.
|
||||
note_rate_limit("[email protected]", Duration::from_secs(1));
|
||||
assert!(
|
||||
rate_limit_remaining("[email protected]").unwrap() > Duration::from_secs(20)
|
||||
);
|
||||
clear_rate_limit();
|
||||
assert_eq!(rate_limit_remaining("[email protected]"), None);
|
||||
}
|
||||
}
|
||||
@@ -901,6 +901,11 @@ async function callTool(params) {
|
||||
try { teardown = await destroyDockerComputer(entry); }
|
||||
catch (err) { teardown = { destroyed: false, cleanup_error: err.message ?? String(err) }; }
|
||||
}
|
||||
if (teardown?.reason === "other_session") {
|
||||
// Another live (or crashed) MCP session owns this desktop. Leave its
|
||||
// container and registry entry alone; the owner removes both at exit.
|
||||
return { content: [{ type: "text", text: JSON.stringify(fail(entry, "computer_owned_elsewhere", `computer "${entry.id}" was spawned by another session and is still its disposable desktop — it is removed when that session ends. If that session is gone, remove the container with docker rm -f ${entry.container}.`)) }], isError: true };
|
||||
}
|
||||
const res = registry.remove(args.computer);
|
||||
if (activeComputerId === args.computer) activeComputerId = "local";
|
||||
res.active = activeComputerId;
|
||||
@@ -1107,7 +1112,12 @@ async function callTool(params) {
|
||||
const timeoutMs = backendMethod.startsWith("recording") || backendMethod === "get_app_state" ? 60_000 : 30_000;
|
||||
const invoke = async (tool, a) => {
|
||||
const r = await remoteCall({ tool, args: a }, { timeoutMs });
|
||||
if (!r.ok) throw new ServerError(r.error?.code ?? "remote_error", r.error?.message ?? "remote agent failed");
|
||||
if (!r.ok) {
|
||||
const error = new ServerError(r.error?.code ?? "remote_error", r.error?.message ?? "remote agent failed");
|
||||
// The helper's backend reported that input may already have landed.
|
||||
if (r.error?.request_dispatched === true) error.requestDispatched = true;
|
||||
throw error;
|
||||
}
|
||||
return r.data;
|
||||
};
|
||||
data = name === "type" ? await invokeType(invoke, wireArgs) : await invoke(backendMethod, wireArgs);
|
||||
@@ -1246,7 +1256,9 @@ async function callTool(params) {
|
||||
// A failed open_application cleared the backend's input binding before it
|
||||
// attempted anything — the tracked bound app must not claim otherwise.
|
||||
if (name === "open_application") boundApps.delete(computer.id);
|
||||
let outcomeUnknown = !!err.requestDispatched;
|
||||
// A local backend that timed out or was cancelled after posting input
|
||||
// (inputMayHaveBeenSent) is as unknown as a transport that lost the reply.
|
||||
let outcomeUnknown = !!(err.requestDispatched || err.inputMayHaveBeenSent);
|
||||
if (dispatched && !outcomeUnknown) {
|
||||
// A transport/backend can fail after delivering input. Reconcile its
|
||||
// captured route on failure too, without replacing the original error
|
||||
|
||||
@@ -30,7 +30,7 @@ Never retry a refusal unchanged — re-observe, re-target, or change route.
|
||||
| `menu_item_not_found` | exact title not present (menus expose items only while open) | check the exact title; an ellipsis is part of it |
|
||||
| `menu_item_disabled` | item present but the app refuses it right now (often a missing key window) | use the window's own control element instead |
|
||||
| `app_not_found` | selector missed — `open_application` names/bundle ids that resolve nowhere and dead pids report it too | `list_apps` (or `all:true`) for exact names/pids |
|
||||
| `ambiguous_application` | `kill_app` name matched several running apps | pass `pid` to choose one |
|
||||
| `ambiguous_application` | an app name or bundle id matched several running apps (observation, input or `kill_app`) | pass `pid` to choose one |
|
||||
| `protected_application` | the target is the Computer Use helper or its host | name the intended app instead; these cannot be terminated through the plugin |
|
||||
| `browser_not_running` | browser action before `browser {action:"start"}` (or the browser went away) | start it; a closed CDP connection clears the session state |
|
||||
| `browser_not_installed` | no Chromium-family browser found | install one, or set `CODEWHALE_CU_BROWSER_APP` to the app path |
|
||||
@@ -56,6 +56,7 @@ Never retry a refusal unchanged — re-observe, re-target, or change route.
|
||||
| `spawn_failed` | provisioning failed or the desktop did not become ready | read the message; the failed container is removed automatically — fix the cause and spawn again |
|
||||
| `invalid_container` | a docker registry entry lacks a valid container name | register it through `computer spawn`, never by hand |
|
||||
| `cleanup_failed` | `docker rm` failed while tearing down a spawned computer | the registry entry is still removed; check `docker ps` for the labeled container and remove it manually |
|
||||
| `computer_owned_elsewhere` | `computer remove` named a desktop another session spawned | leave it; its session removes it at exit (the message names the container if that session is gone) |
|
||||
| `script_error` | osascript exited non-zero; stderr is in the message | read the error, check the app's scripting dictionary (`sdef`), fix the script |
|
||||
| `script_timeout` | the script — or a consent dialog — was still open at the deadline | narrow the script; a consent prompt is the person's choice, report it |
|
||||
| `script_cancelled` | the script's own dialog was cancelled (-128) | the user declined in-app; stop or ask |
|
||||
@@ -75,5 +76,7 @@ computer; a typing pause does not authorize foreground control.
|
||||
not task success. Verify the effect with a fresh observation.
|
||||
- `front_lease` / `front_restored` describe focus accounting for window-record
|
||||
deliveries in explicitly authorized foreground mode. `front_restored:false` is a person-visible event: say it out loud.
|
||||
- `input_may_have_been_sent` on an error means the press left before the
|
||||
failure: observe the target before doing anything else.
|
||||
- `outcome_unknown: true` (with `request_dispatched: true`) on an error means
|
||||
the action was already handed to the helper, remote agent or browser when it
|
||||
timed out, was cancelled or lost its reply: observe the target before doing
|
||||
anything else, and never retry the action blindly.
|
||||
@@ -226,7 +226,7 @@ export async function handle(req, { computerId = "local", sessionId = "direct",
|
||||
return { ok: true, platform: process.platform, tool, data };
|
||||
}));
|
||||
} catch (err) {
|
||||
return { ok: false, platform: process.platform, tool, error: { code: err?.code ?? "tool_error", message: String(err?.message ?? err) } };
|
||||
return { ok: false, platform: process.platform, tool, error: { code: err?.code ?? "tool_error", message: String(err?.message ?? err), ...(err?.inputMayHaveBeenSent || err?.requestDispatched ? { request_dispatched: true } : {}) } };
|
||||
} finally {
|
||||
signal?.removeEventListener("abort", abort);
|
||||
session.requests.delete(controller);
|
||||
|
||||
@@ -74,27 +74,37 @@ export function writeRegistration(reg) {
|
||||
fs.writeFileSync(registrationPath(), JSON.stringify({ ...reg, registeredAt: new Date().toISOString() }, null, 2) + "\n");
|
||||
}
|
||||
|
||||
/** Send one request to the app and await its single-line reply. */
|
||||
/**
|
||||
* Send one request to the app and await its single-line reply.
|
||||
*
|
||||
* Once the request line is written the helper may already have acted on it,
|
||||
* so every failure after that point (timeout, cancel, a dropped connection, a
|
||||
* malformed reply) carries `requestDispatched: true`. The MCP server turns
|
||||
* that into `outcome_unknown` instead of a plain cancel that invites a blind
|
||||
* retry of an input action that may have landed.
|
||||
*/
|
||||
function requestConnection(request, { timeoutMs = 30_000, signal = currentSignal(), keepOpen = false } = {}) {
|
||||
throwIfAborted(signal);
|
||||
return new Promise((resolve, reject) => {
|
||||
const sock = net.connect(socketPath());
|
||||
let buf = "";
|
||||
let settled = false;
|
||||
let written = false;
|
||||
const failure = (message, code) => Object.assign(new ExecError(message), { code }, written ? { requestDispatched: true } : {});
|
||||
const done = (fn, v) => { if (settled) return; settled = true; clearTimeout(timer); signal?.removeEventListener("abort", abort); if (!keepOpen || fn === reject) sock.destroy(); fn(v); };
|
||||
const abort = () => done(reject, Object.assign(new ExecError("computer request cancelled"), { code: "cancelled" }));
|
||||
const timer = setTimeout(() => done(reject, Object.assign(new ExecError(`${APP_NAME}: request timed out after ${timeoutMs}ms`), { code: "app_timeout" })), timeoutMs);
|
||||
const abort = () => done(reject, failure("computer request cancelled", "cancelled"));
|
||||
const timer = setTimeout(() => done(reject, failure(`${APP_NAME}: request timed out after ${timeoutMs}ms`, "app_timeout")), timeoutMs);
|
||||
signal?.addEventListener("abort", abort, { once: true });
|
||||
sock.on("error", (err) => done(reject, Object.assign(new ExecError(`${APP_NAME} is not reachable at ${socketPath()}: ${err.code ?? err.message}`), { code: "app_unavailable" })));
|
||||
sock.on("connect", () => sock.write(JSON.stringify(request) + "\n"));
|
||||
sock.on("error", (err) => done(reject, failure(`${APP_NAME} is not reachable at ${socketPath()}: ${err.code ?? err.message}`, "app_unavailable")));
|
||||
sock.on("connect", () => { written = true; sock.write(JSON.stringify(request) + "\n"); });
|
||||
sock.on("data", (d) => {
|
||||
buf += d.toString("utf8");
|
||||
const nl = buf.indexOf("\n");
|
||||
if (nl === -1) return;
|
||||
try { const reply = JSON.parse(buf.slice(0, nl)); done(resolve, keepOpen ? { reply, socket: sock } : reply); }
|
||||
catch { done(reject, Object.assign(new ExecError(`${APP_NAME}: malformed reply`), { code: "app_bad_reply" })); }
|
||||
catch { done(reject, failure(`${APP_NAME}: malformed reply`, "app_bad_reply")); }
|
||||
});
|
||||
sock.on("close", () => done(reject, Object.assign(new ExecError(`${APP_NAME}: connection closed before a reply`), { code: "app_unavailable" })));
|
||||
sock.on("close", () => done(reject, failure(`${APP_NAME}: connection closed before a reply`, "app_unavailable")));
|
||||
});
|
||||
}
|
||||
|
||||
@@ -128,6 +138,11 @@ export function openAppSession(sessionId) {
|
||||
// Library clients need not keep Node alive solely for an idle lease.
|
||||
socket.unref();
|
||||
return lease;
|
||||
}, (error) => {
|
||||
// Opening a lease sends no input, so its failure is never
|
||||
// outcome-unknown for the action that was waiting on it.
|
||||
if (error && typeof error === "object") delete error.requestDispatched;
|
||||
throw error;
|
||||
});
|
||||
// A refused or unreachable open is retried on the next request, not cached.
|
||||
pending.catch(() => { if (sessionLeases.get(sessionId) === pending) sessionLeases.delete(sessionId); });
|
||||
|
||||
@@ -856,13 +856,23 @@ static NSRunningApplication *resolve(NSDictionary *ref) {
|
||||
} else return nil;
|
||||
}
|
||||
NSString *bundle=ref[@"bundle_id"], *name=ref[@"name"];
|
||||
// A name or bundle shared by several running processes (two instances of
|
||||
// one browser, a helper and its app) must not resolve to whichever happens
|
||||
// to be listed first: that sends observation and input to the wrong window.
|
||||
// Refuse like kill_app does; the observed pid disambiguates.
|
||||
NSMutableArray<NSRunningApplication *> *hits=[NSMutableArray array];
|
||||
for(NSRunningApplication *a in NSWorkspace.sharedWorkspace.runningApplications) {
|
||||
if(ref[@"pid"] && a.processIdentifier!=[ref[@"pid"] intValue]) continue;
|
||||
if(bundle && !matchesName(a.bundleIdentifier,bundle)) continue;
|
||||
if(name && !matchesName(a.localizedName,name)) continue;
|
||||
return a;
|
||||
[hits addObject:a];
|
||||
}
|
||||
return nil;
|
||||
if(hits.count>1) {
|
||||
NSMutableArray *desc=[NSMutableArray array];
|
||||
for(NSRunningApplication *a in hits) [desc addObject:[NSString stringWithFormat:@"%@ (pid %d)",a.localizedName?:@"?",a.processIdentifier]];
|
||||
@throw [NSException exceptionWithName:@"app" reason:[NSString stringWithFormat:@"several running applications match (%@); pass pid to choose one",[desc componentsJoinedByString:@", "]] userInfo:nil];
|
||||
}
|
||||
return hits.firstObject;
|
||||
}
|
||||
static CGEventRef textEvent(NSString *text, BOOL down) {
|
||||
UniChar *chars=calloc(text.length,sizeof(UniChar)); [text getCharacters:chars range:NSMakeRange(0,text.length)];
|
||||
|
||||
@@ -105,12 +105,22 @@ export function create({ exec }) {
|
||||
return { action_sent: true, strategy: "event", backend: "uitest" };
|
||||
}
|
||||
|
||||
/**
|
||||
* Re-find an observed element in a fresh dump. uitest indexes are
|
||||
* positional and shift whenever a sibling appears, so the index alone can
|
||||
* name a different control; the element is addressed by its observed tree
|
||||
* path and must still carry the observed role and label, or nothing is sent.
|
||||
*/
|
||||
async function centerOf(target) {
|
||||
rejectAppSelectors(target);
|
||||
const stale = (why) => Object.assign(new ExecError(`element_stale — ${why}; re-run get_app_state (uitest indexes change with the UI)`), { code: "element_stale" });
|
||||
if (!Array.isArray(target?.path)) throw stale("the element target carries no observed tree path");
|
||||
const tree = await dumpLayout();
|
||||
const els = flatten(tree);
|
||||
const el = els[target.index];
|
||||
if (!el || !el.bounds) throw new ExecError("element_stale — re-run get_app_state; uitest indexes change with the UI");
|
||||
const key = JSON.stringify(target.path);
|
||||
const el = flatten(tree).find((candidate) => JSON.stringify(candidate.path) === key);
|
||||
if (!el || !el.bounds) throw stale("the observed element is gone");
|
||||
if (target.role !== undefined && el.role !== target.role) throw stale(`role changed (${target.role} → ${el.role})`);
|
||||
if (target.label !== undefined && el.label !== target.label) throw stale(`label changed (${target.label} → ${el.label})`);
|
||||
return el.bounds;
|
||||
}
|
||||
|
||||
|
||||
@@ -191,6 +191,12 @@ function defaultConnect(url, { timeoutMs = 8_000 } = {}) {
|
||||
});
|
||||
}
|
||||
|
||||
/** Per-command CDP reply deadline; teardown uses its own shorter bound. */
|
||||
const commandTimeoutMs = () => Number(process.env.CODEWHALE_CU_BROWSER_COMMAND_TIMEOUT_MS) || 30_000;
|
||||
// Teardown ignores the (possibly already cancelled) request signal and gives
|
||||
// an unanswered peer at most three seconds per command.
|
||||
const teardown = () => ({ timeoutMs: Math.min(3_000, commandTimeoutMs()), signal: null });
|
||||
|
||||
/** id-matched JSON-RPC over the WebSocket, plus CDP event fan-out. */
|
||||
function makeChannel(ws) {
|
||||
let nextId = 0;
|
||||
@@ -199,7 +205,7 @@ function makeChannel(ws) {
|
||||
let closed = false;
|
||||
const failAll = (reason) => {
|
||||
closed = true;
|
||||
for (const [, entry] of pending) entry.reject(Object.assign(new ExecError(reason), { code: "browser_not_running" }));
|
||||
for (const entry of [...pending.values()]) entry.reject(Object.assign(new ExecError(reason), { code: "browser_not_running" }));
|
||||
pending.clear();
|
||||
};
|
||||
ws.addEventListener("message", (event) => {
|
||||
@@ -218,11 +224,29 @@ function makeChannel(ws) {
|
||||
ws.addEventListener("error", () => {});
|
||||
return {
|
||||
get alive() { return !closed; },
|
||||
send(method, params = {}, sessionId) {
|
||||
/**
|
||||
* One CDP command. Bounded: a peer that keeps the socket open but never
|
||||
* answers must not hang the request (or a stop/teardown awaiting it)
|
||||
* forever. After the command is written its effect is unknown, so a
|
||||
* timeout or cancel carries requestDispatched; a late reply is dropped.
|
||||
*/
|
||||
send(method, params = {}, sessionId, { timeoutMs = commandTimeoutMs(), signal = currentSignal() } = {}) {
|
||||
if (closed) return Promise.reject(Object.assign(new ExecError("the browser is no longer reachable (CDP connection closed)"), { code: "browser_not_running" }));
|
||||
if (signal?.aborted) return Promise.reject(Object.assign(new ExecError("computer request cancelled"), { code: "cancelled" }));
|
||||
return new Promise((resolve, reject) => {
|
||||
const id = ++nextId;
|
||||
pending.set(id, { resolve, reject });
|
||||
let timer = null;
|
||||
const finish = () => { clearTimeout(timer); signal?.removeEventListener("abort", onAbort); pending.delete(id); };
|
||||
const onAbort = () => { finish(); reject(Object.assign(new ExecError("computer request cancelled"), { code: "cancelled", requestDispatched: true })); };
|
||||
pending.set(id, {
|
||||
resolve: (value) => { finish(); resolve(value); },
|
||||
reject: (error) => { finish(); reject(error); },
|
||||
});
|
||||
timer = setTimeout(() => {
|
||||
finish();
|
||||
reject(Object.assign(new ExecError(`CDP ${method} got no reply within ${timeoutMs}ms`), { code: "timeout", requestDispatched: true }));
|
||||
}, timeoutMs);
|
||||
signal?.addEventListener("abort", onAbort, { once: true });
|
||||
ws.send(JSON.stringify({ id, method, params, ...(sessionId ? { sessionId } : {}) }));
|
||||
});
|
||||
},
|
||||
@@ -280,8 +304,8 @@ export function createBrowser({
|
||||
const { targetInfo: info } = await state.channel.send("Target.getTargetInfo", { targetId });
|
||||
return { url: info?.url ?? "", title: info?.title ?? "" };
|
||||
}
|
||||
async function listTabs() {
|
||||
const { targetInfos } = await state.channel.send("Target.getTargets", {});
|
||||
async function listTabs(options) {
|
||||
const { targetInfos } = await state.channel.send("Target.getTargets", {}, undefined, options);
|
||||
return (targetInfos ?? []).filter((t) => t.type === "page");
|
||||
}
|
||||
async function ensureDom() {
|
||||
@@ -570,17 +594,17 @@ export function createBrowser({
|
||||
if (!state.channel) return { running: false, note: "no browser session for this computer session" };
|
||||
if (state.attached) {
|
||||
// The person's browser: never close a tab or the browser, only detach.
|
||||
if (state.sessionId) await state.channel.send("Target.detachFromTarget", { sessionId: state.sessionId }).catch(() => {});
|
||||
if (state.sessionId) await state.channel.send("Target.detachFromTarget", { sessionId: state.sessionId }, undefined, teardown()).catch(() => {});
|
||||
state.channel.close();
|
||||
state.channel = null; state.targetId = null; state.sessionId = null; state.pageEnabled = false; state.domEnabled = false; state.attached = false;
|
||||
return { running: false, detached: true, browser_closed: false, note: "detached from the shared browser; its window and tabs stay as they are" };
|
||||
}
|
||||
try { await state.channel.send("Target.closeTarget", { targetId: state.targetId }); } catch { /* the tab may already be gone */ }
|
||||
try { await state.channel.send("Target.closeTarget", { targetId: state.targetId }, undefined, teardown()); } catch { /* the tab may already be gone */ }
|
||||
let remaining = null;
|
||||
try { remaining = (await listTabs()).length; } catch { remaining = null; }
|
||||
try { remaining = (await listTabs(teardown())).length; } catch { remaining = null; }
|
||||
let browserClosed = false;
|
||||
if (remaining === 0) {
|
||||
try { await state.channel.send("Browser.close"); browserClosed = true; } catch {}
|
||||
try { await state.channel.send("Browser.close", {}, undefined, teardown()); browserClosed = true; } catch {}
|
||||
await sleep(200);
|
||||
}
|
||||
state.channel.close();
|
||||
|
||||
@@ -116,19 +116,23 @@ export async function spawnDockerComputer({ id, image = DEFAULT_IMAGE } = {}) {
|
||||
}
|
||||
|
||||
/**
|
||||
* Destroy a spawned container — but only one this plugin created. A docker
|
||||
* computer whose container lacks our spawn label is left running and reported
|
||||
* not_spawned; removing its registry entry is still the caller's choice.
|
||||
* Destroy a spawned container — but only one this MCP process created. A
|
||||
* docker computer whose container lacks our spawn label is left running and
|
||||
* reported not_spawned; one spawned by another session (the registry is
|
||||
* shared between MCP processes) is left running and reported other_session,
|
||||
* so one task cannot tear down a desktop another task is driving.
|
||||
*/
|
||||
export async function destroyDockerComputer(computer) {
|
||||
export async function destroyDockerComputer(computer, command = docker) {
|
||||
const container = computer?.container;
|
||||
if (!container || !CONTAINER_RE.test(container)) {
|
||||
throw new SpawnError("invalid_container", "docker computer has no valid container name");
|
||||
}
|
||||
const insp = await docker(["container", "inspect", "--format", `{{index .Config.Labels "${SPAWN_LABEL}"}}`, container], { timeoutMs: 10_000, signal: null });
|
||||
const insp = await command(["container", "inspect", "--format", `{{index .Config.Labels "${SPAWN_LABEL}"}}|{{index .Config.Labels "${SESSION_LABEL}"}}`, container], { timeoutMs: 10_000, signal: null });
|
||||
if (insp.code !== 0) return { destroyed: false, reason: "container_gone" };
|
||||
if (insp.stdout.trim() !== "1") return { destroyed: false, reason: "not_spawned" };
|
||||
const r = await docker(["rm", "-f", container], { timeoutMs: 20_000, signal: null });
|
||||
const [spawned, session] = insp.stdout.trim().split("|");
|
||||
if (spawned !== "1") return { destroyed: false, reason: "not_spawned" };
|
||||
if (session !== SESSION_ID) return { destroyed: false, reason: "other_session" };
|
||||
const r = await command(["rm", "-f", container], { timeoutMs: 20_000, signal: null });
|
||||
if (r.code !== 0) throw new SpawnError("cleanup_failed", `docker rm -f ${container} failed: ${trim(r.stderr)}`, r);
|
||||
return { destroyed: true };
|
||||
}
|
||||
|
||||
@@ -218,8 +218,10 @@ export function sshExec(computer, binding) {
|
||||
const r = await run("ssh", [...base, "node", remoteAgent, b64({ args: request.args ?? {}, tool: request.tool, nonce: crypto.randomBytes(6).toString("hex") })], {
|
||||
timeoutMs: opts.timeoutMs ?? 25_000,
|
||||
});
|
||||
if (r.aborted) throw Object.assign(new ExecError("computer request cancelled", r), { code: "cancelled" });
|
||||
if (r.timedOut) throw new ExecError(`ssh ${userHost}: timed out`, r);
|
||||
// A spawned one-shot agent may have acted before it was cancelled or
|
||||
// timed out; say so rather than inviting a blind retry.
|
||||
if (r.aborted) throw Object.assign(new ExecError("computer request cancelled", r), { code: "cancelled" }, r.spawned ? { requestDispatched: true } : {});
|
||||
if (r.timedOut) throw Object.assign(new ExecError(`ssh ${userHost}: timed out`, r), { code: "remote_timeout", requestDispatched: true });
|
||||
if (r.code !== 0) throw new ExecError(`ssh ${userHost} exited ${r.code}: ${r.stderr.trim().slice(0, 400)}`, r);
|
||||
// The agent prints exactly one JSON line; anything before it is MOTD noise.
|
||||
const line = r.stdout.trim().split("\n").filter((l) => l.startsWith("{")).pop();
|
||||
@@ -254,8 +256,8 @@ export function dockerExec(computer, binding) {
|
||||
const r = await run("docker", ["exec", container, "/bin/sh", remoteAgent, b64({ args: request.args ?? {}, tool: request.tool, nonce: crypto.randomBytes(6).toString("hex") })], {
|
||||
timeoutMs: opts.timeoutMs ?? 25_000,
|
||||
});
|
||||
if (r.aborted) throw Object.assign(new ExecError("computer request cancelled", r), { code: "cancelled" });
|
||||
if (r.timedOut) throw new ExecError(`docker exec ${container}: timed out`, r);
|
||||
if (r.aborted) throw Object.assign(new ExecError("computer request cancelled", r), { code: "cancelled" }, r.spawned ? { requestDispatched: true } : {});
|
||||
if (r.timedOut) throw Object.assign(new ExecError(`docker exec ${container}: timed out`, r), { code: "remote_timeout", requestDispatched: true });
|
||||
if (r.code !== 0) throw new ExecError(`docker exec ${container} exited ${r.code}: ${r.stderr.trim().slice(0, 400)}`, r);
|
||||
const line = r.stdout.trim().split("\n").filter((l) => l.startsWith("{")).pop();
|
||||
const reply = line ? JSON.parse(line) : null;
|
||||
|
||||
@@ -71,6 +71,17 @@ test('real handler/backend/native resolver never redirects an explicit app refer
|
||||
assert.equal(reply.ok,false,`${tool} must reject ${JSON.stringify(app_ref)}`);
|
||||
assert.match(reply.error.message,/application not found/);
|
||||
}
|
||||
// A shared name or bundle must never pick whichever process is listed first.
|
||||
for(const app_ref of [{name:'Twin'},{bundle_id:'test.twin'},{name:'twin',bundle_id:'test.twin'}]) {
|
||||
const reply=await call(tool,{app_ref});
|
||||
assert.equal(reply.ok,false,`${tool} must refuse ambiguous ${JSON.stringify(app_ref)}`);
|
||||
assert.equal(reply.error.code,'ambiguous_application',JSON.stringify(reply));
|
||||
assert.match(reply.error.message,/Twin \(pid 200\), Twin \(pid 201\)/);
|
||||
}
|
||||
}
|
||||
for(const pid of [200,201]) {
|
||||
const reply=await call('list_windows',{app_ref:{pid,name:'Twin'}});
|
||||
assert.equal(reply.ok,true,JSON.stringify(reply));assert.equal(reply.data.pid,pid);
|
||||
}
|
||||
} finally { delete process.env.CU_TARGETING_NATIVE; }
|
||||
});
|
||||
@@ -47,7 +47,7 @@ function harmonyFixtureExec(t) {
|
||||
return fakeJpeg(168, 120);
|
||||
},
|
||||
};
|
||||
return { exec, calls };
|
||||
return { exec, calls, layout };
|
||||
}
|
||||
|
||||
test("harmony: parseBounds handles uitest bounds strings", () => {
|
||||
@@ -102,6 +102,24 @@ test("harmony: click routes through uitest uiInput with validated args", async (
|
||||
assert.deepEqual(ui.args, ["uitest", "uiInput", "click", "124", "45"]);
|
||||
});
|
||||
|
||||
test("harmony: element actions re-find the observed element and refuse a reordered tree", async () => {
|
||||
const { exec, calls, layout } = harmonyFixtureExec();
|
||||
const mod = await import("../src/backends/harmonyos.mjs");
|
||||
const b = mod.create({ exec });
|
||||
const ok = (await b.get_app_state({})).elements.find((e) => e.label === "OK");
|
||||
const target = { index: ok.index, path: ok.path, role: ok.role, label: ok.label };
|
||||
await b.perform_action({ target, action: "click" });
|
||||
assert.deepEqual(calls.filter((c) => c.args?.[1] === "uiInput").at(-1).args, ["uitest", "uiInput", "click", "200", "230"]);
|
||||
// A new sibling ahead of OK shifts every uitest index: the stored index now
|
||||
// names "Delete". Nothing may be clicked.
|
||||
layout.children.unshift({ attributes: { type: "Button", text: "Delete", bounds: "[0,300][100,360]" }, children: [] });
|
||||
const before = calls.filter((c) => c.args?.[1] === "uiInput").length;
|
||||
await assert.rejects(b.perform_action({ target, action: "click" }), (error) => error.code === "element_stale" && /label changed/.test(error.message));
|
||||
await assert.rejects(b.set_value({ target, value: "x" }), (error) => error.code === "element_stale");
|
||||
await assert.rejects(b.perform_action({ target: { index: ok.index }, action: "click" }), (error) => error.code === "element_stale", "a bare index has no identity to verify");
|
||||
assert.equal(calls.filter((c) => c.args?.[1] === "uiInput").length, before, "no input after the tree changed");
|
||||
});
|
||||
|
||||
test("harmony: clipboard and select_text fail closed with named reasons", async () => {
|
||||
const { exec } = harmonyFixtureExec();
|
||||
const mod = await import("../src/backends/harmonyos.mjs");
|
||||
|
||||
@@ -271,3 +271,19 @@ test('findBrowserApp uses actual Windows vendor folders and executable names', (
|
||||
assert.equal(findBrowserApp('win32', { [root]: 'C:\\fixture' }, candidate => candidate === expected), expected);
|
||||
}
|
||||
});
|
||||
|
||||
test("an unanswered CDP command times out as outcome-unknown and stop still finishes", async (t) => {
|
||||
const prior = process.env.CODEWHALE_CU_BROWSER_COMMAND_TIMEOUT_MS;
|
||||
process.env.CODEWHALE_CU_BROWSER_COMMAND_TIMEOUT_MS = "60";
|
||||
t.after(() => { if (prior === undefined) delete process.env.CODEWHALE_CU_BROWSER_COMMAND_TIMEOUT_MS; else process.env.CODEWHALE_CU_BROWSER_COMMAND_TIMEOUT_MS = prior; });
|
||||
// An open peer that never answers these commands (a wedged renderer).
|
||||
const script = { ...defaultScript(pageState()), "Input.insertText": undefined, "Target.closeTarget": undefined, "Browser.close": undefined };
|
||||
const { browser, sockets } = harness(t, { script });
|
||||
await browser.start({});
|
||||
await assert.rejects(browser.type({ text: "hello" }), (error) => error.code === "timeout" && error.requestDispatched === true && /Input\.insertText/.test(error.message));
|
||||
const stopped = await Promise.race([browser.stop(), new Promise((resolve) => setTimeout(() => resolve("hung"), 2_000))]);
|
||||
assert.notEqual(stopped, "hung", "stop must not wait forever on an unanswered peer");
|
||||
assert.equal(stopped.closed, true);
|
||||
assert.equal(sockets[0].ws.sentOf("Target.closeTarget").length, 1);
|
||||
assert.equal((await browser.status()).running, false);
|
||||
});
|
||||
@@ -7,7 +7,7 @@ import path from "node:path";
|
||||
import net from "node:net";
|
||||
import { run, runOk, runInputLease, ExecError, have, trim, withSignal } from "../src/exec.mjs";
|
||||
import { safeRemotePath, b64, localExec, hdcExec, executorFor } from "../src/transport.mjs";
|
||||
import { ensureApp, writeRegistration } from "../src/app-socket.mjs";
|
||||
import { appRequest, appSessionRequest, ensureApp, writeRegistration } from "../src/app-socket.mjs";
|
||||
|
||||
test("a missing registered bundle gives a repair path without falling back to host input",async t=>{
|
||||
const directory=fs.mkdtempSync(path.join(os.tmpdir(),"cu-missing-app-"));
|
||||
@@ -208,3 +208,30 @@ test("hdc pullFile rejects traversal and shell metacharacters before execution",
|
||||
await assert.rejects(ex.pullFile(remote, "/unused-fixture-output"), /refusing unsafe remote path/);
|
||||
}
|
||||
});
|
||||
|
||||
test("a helper request that was written reports an unknown outcome when it times out or is cancelled", {skip:process.platform==="win32"}, async t => {
|
||||
const dir=fs.mkdtempSync(path.join(os.tmpdir(),"cu-silent-helper-"));
|
||||
const saved=process.env.CODEWHALE_CU_APP_SOCKET;
|
||||
process.env.CODEWHALE_CU_APP_SOCKET=path.join(dir,"app.sock");
|
||||
const received=[];
|
||||
// Accepts and reads the request, then never answers (a wedged helper).
|
||||
const server=net.createServer(socket=>socket.on("data",data=>received.push(String(data))));
|
||||
await new Promise(resolve=>server.listen(process.env.CODEWHALE_CU_APP_SOCKET,resolve));
|
||||
t.after(async()=>{
|
||||
server.close();
|
||||
if(saved===undefined) delete process.env.CODEWHALE_CU_APP_SOCKET; else process.env.CODEWHALE_CU_APP_SOCKET=saved;
|
||||
fs.rmSync(dir,{recursive:true,force:true});
|
||||
});
|
||||
await assert.rejects(appRequest({tool:"left_click",args:{}},{timeoutMs:80,signal:null}),error=>error.code==="app_timeout"&&error.requestDispatched===true);
|
||||
const controller=new AbortController();
|
||||
const pending=appRequest({tool:"left_click",args:{}},{timeoutMs:5_000,signal:controller.signal});
|
||||
while(received.length<2) await new Promise(resolve=>setTimeout(resolve,5));
|
||||
controller.abort();
|
||||
await assert.rejects(pending,error=>error.code==="cancelled"&&error.requestDispatched===true);
|
||||
assert.equal(received.length,2,"each request reached the helper before its failure");
|
||||
// Opening the session lease sends no input: its timeout is not outcome-unknown.
|
||||
await assert.rejects(appSessionRequest({tool:"left_click",args:{},sessionId:"silent-lease"},{timeoutMs:80,signal:null}),error=>error.code==="app_timeout"&&error.requestDispatched===undefined);
|
||||
// Nothing written: an unreachable helper is a plain failure, safe to retry.
|
||||
process.env.CODEWHALE_CU_APP_SOCKET=path.join(dir,"absent.sock");
|
||||
await assert.rejects(appRequest({tool:"left_click",args:{}},{timeoutMs:1_000,signal:null}),error=>error.code==="app_unavailable"&&error.requestDispatched===undefined);
|
||||
});
|
||||
@@ -20,7 +20,12 @@
|
||||
CUFixtureApplication *target=[CUFixtureApplication new], *front=[CUFixtureApplication new];
|
||||
target.processIdentifier=123; target.localizedName=@"Fixture"; target.bundleIdentifier=@"test.fixture";
|
||||
front.processIdentifier=999; front.localizedName=@"Other"; front.bundleIdentifier=@"test.other";
|
||||
return @[front,target];
|
||||
// Two running instances sharing one name and bundle (a second browser
|
||||
// profile, an `open -n` copy): only a pid may choose between them.
|
||||
CUFixtureApplication *twinA=[CUFixtureApplication new], *twinB=[CUFixtureApplication new];
|
||||
twinA.processIdentifier=200; twinA.localizedName=@"Twin"; twinA.bundleIdentifier=@"test.twin";
|
||||
twinB.processIdentifier=201; twinB.localizedName=@"Twin"; twinB.bundleIdentifier=@"test.twin";
|
||||
return @[front,target,twinA,twinB];
|
||||
}
|
||||
- (NSRunningApplication *)frontmostApplication { return (NSRunningApplication *)self.runningApplications[0]; }
|
||||
@end
|
||||
@@ -50,7 +55,9 @@ int main(int argc, const char *argv[]) { @autoreleasepool {
|
||||
return 0;
|
||||
}
|
||||
if(![@[@"list_windows",@"get_app_state",@"resolve_element",@"window_info"] containsObject:request[@"tool"]]) return 2;
|
||||
NSRunningApplication *app=resolve(request[@"args"][@"app_ref"]);
|
||||
NSRunningApplication *app=nil;
|
||||
@try { app=resolve(request[@"args"][@"app_ref"]); }
|
||||
@catch(NSException *error) { fprintf(stderr,"%s\n",error.reason.UTF8String); return 1; }
|
||||
if(!app) { fputs("application not found\n",stderr); return 1; }
|
||||
NSDictionary *receipt=@{@"found":@YES,@"pid":@(app.processIdentifier),@"name":app.localizedName,@"bundle_id":app.bundleIdentifier,@"windows":@[],@"elements":@[]};
|
||||
NSData *json=[NSJSONSerialization dataWithJSONObject:receipt options:0 error:nil];
|
||||
|
||||
@@ -231,3 +231,17 @@ test("Docker desktop entrypoint survives repeated orderly restarts", { ...NEED_D
|
||||
assert.equal(observed, true, `desktop unavailable after restart ${cycle}: ${last}`);
|
||||
}
|
||||
});
|
||||
|
||||
test("destroyDockerComputer never removes a desktop another session spawned", async () => {
|
||||
const issued = [];
|
||||
const labels = (value) => async (args) => {
|
||||
issued.push(args);
|
||||
return args[0] === "container" ? { code: 0, stdout: `${value}\n`, stderr: "" } : { code: 0, stdout: "", stderr: "" };
|
||||
};
|
||||
const computer = { container: "cu-spawn-other-ab12cd" };
|
||||
assert.deepEqual(await spawnMod.destroyDockerComputer(computer, labels("1|another-session")), { destroyed: false, reason: "other_session" });
|
||||
assert.deepEqual(await spawnMod.destroyDockerComputer(computer, labels("|")), { destroyed: false, reason: "not_spawned" });
|
||||
assert.equal(issued.filter((args) => args[0] === "rm").length, 0, "no docker rm for a container this session does not own");
|
||||
assert.deepEqual(await spawnMod.destroyDockerComputer(computer, labels(`1|${SESSION_ID}`)), { destroyed: true });
|
||||
assert.deepEqual(issued.at(-1), ["rm", "-f", "cu-spawn-other-ab12cd"]);
|
||||
});
|
||||
@@ -77,6 +77,11 @@ const MAX_ACP_SESSIONS: usize = 64;
|
||||
/// broken client cannot strand the stdio server forever after cancellation.
|
||||
const ACP_PERMISSION_CANCEL_GRACE: Duration = Duration::from_secs(2);
|
||||
|
||||
/// How long a cancelled tool gets to observe its token and wind down (kill a
|
||||
/// child, flush a write) before the turn stops waiting and drops it. A tool
|
||||
/// that ignores its token must not hold the stdio server forever.
|
||||
const ACP_TOOL_CANCEL_GRACE: Duration = Duration::from_secs(5);
|
||||
|
||||
/// Agent-originated JSON-RPC request ids have their own namespace. Strings
|
||||
/// avoid the client-specific numeric response-id compatibility shim used for
|
||||
/// replies to client-originated requests.
|
||||
@@ -1226,11 +1231,22 @@ where
|
||||
write_jsonrpc_result(writer, msg_id, json!(null)).await?;
|
||||
}
|
||||
cancel_token.cancel();
|
||||
// Give the tool a chance to observe the token and
|
||||
// wind down (e.g. kill a running child process)
|
||||
// before we drop it.
|
||||
// Give the tool a bounded chance to observe the
|
||||
// token and wind down (e.g. kill a running child
|
||||
// process) before we drop it.
|
||||
cancelled = true;
|
||||
break (&mut exec_fut).await;
|
||||
break match tokio::time::timeout(
|
||||
ACP_TOOL_CANCEL_GRACE,
|
||||
&mut exec_fut,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(result) => result,
|
||||
Err(_) => Err(ToolError::cancelled(format!(
|
||||
"the tool did not stop within {}s of cancellation and was abandoned",
|
||||
ACP_TOOL_CANCEL_GRACE.as_secs()
|
||||
))),
|
||||
};
|
||||
}
|
||||
if let Some(msg_id) = msg_id {
|
||||
let msg_id = response_id_policy.response_id(msg_id);
|
||||
|
||||
+32
-12
@@ -3765,17 +3765,21 @@ impl CodewhaleClient {
|
||||
return self.send_with_isolated_retry(build, disclosure).await;
|
||||
}
|
||||
let retry_cfg: LlmRetryConfig = self.retry.clone().into();
|
||||
let pause_scope = self.rate_limit_scope();
|
||||
let callback_scope = pause_scope.clone();
|
||||
let request_result = with_retry(
|
||||
&retry_cfg,
|
||||
|| {
|
||||
let request = build();
|
||||
let pause_scope = pause_scope.as_str();
|
||||
async move {
|
||||
// Sleep in bounded slices rather than the full remaining
|
||||
// window: the pause is process-global, so a concurrent
|
||||
// `clear_rate_limit()` (or a shortened deadline) must
|
||||
// release requests that are already waiting instead of
|
||||
// stranding them for the whole original window.
|
||||
while let Some(delay) = crate::retry_status::rate_limit_remaining() {
|
||||
// window: the pause is shared by every request to this
|
||||
// route, so a concurrent `clear_rate_limit()` (or a
|
||||
// shortened deadline) must release requests that are
|
||||
// already waiting instead of stranding them for the whole
|
||||
// original window.
|
||||
while let Some(delay) = crate::retry_status::rate_limit_remaining(pause_scope) {
|
||||
tokio::time::sleep(delay.min(RATE_LIMIT_PAUSE_RECHECK_INTERVAL)).await;
|
||||
}
|
||||
self.wait_for_rate_limit().await;
|
||||
@@ -3793,7 +3797,7 @@ impl CodewhaleClient {
|
||||
Err(self.http_error_with_route_context(status.as_u16(), &body, retry_after))
|
||||
}
|
||||
},
|
||||
Some(Box::new(|err, attempt, delay| {
|
||||
Some(Box::new(move |err, attempt, delay| {
|
||||
let (reason_label, human_reason) = retry_reason_label_and_human(err);
|
||||
logging::warn(format!(
|
||||
"HTTP retry reason={} attempt={} delay={:.2}s",
|
||||
@@ -3802,7 +3806,7 @@ impl CodewhaleClient {
|
||||
delay.as_secs_f64(),
|
||||
));
|
||||
if matches!(err, LlmError::RateLimited { .. }) {
|
||||
crate::retry_status::note_rate_limit(delay);
|
||||
crate::retry_status::note_rate_limit(&callback_scope, delay);
|
||||
}
|
||||
crate::retry_status::start(attempt + 1, delay, human_reason);
|
||||
})),
|
||||
@@ -3818,6 +3822,7 @@ impl CodewhaleClient {
|
||||
Err(err) => {
|
||||
if let LlmError::RateLimited { retry_after, .. } = &err.last_error {
|
||||
crate::retry_status::note_rate_limit(
|
||||
&pause_scope,
|
||||
retry_after
|
||||
.unwrap_or_else(|| retry_cfg.delay_for_attempt(retry_cfg.max_retries)),
|
||||
);
|
||||
@@ -3838,6 +3843,21 @@ impl CodewhaleClient {
|
||||
}
|
||||
}
|
||||
|
||||
/// Key for this route's shared `Retry-After` pause: the configured route
|
||||
/// identity plus the host it reaches. A 429 from one provider pauses only
|
||||
/// requests that would hit the same limit, never another provider, a
|
||||
/// local runtime, or a sub-agent on a different route.
|
||||
pub(crate) fn rate_limit_scope(&self) -> String {
|
||||
let route = if self.provider_identity.is_empty() {
|
||||
self.api_provider.as_str()
|
||||
} else {
|
||||
self.provider_identity.as_str()
|
||||
};
|
||||
let host = crate::llm_client::base_url_authority(&self.base_url)
|
||||
.unwrap_or_else(|| redact_url_for_display(&self.base_url));
|
||||
format!("{route}@{host}")
|
||||
}
|
||||
|
||||
/// The same bounded transport retry policy without process-global retry
|
||||
/// banners, provider-wide pause cells, or shared connection-health writes.
|
||||
/// Used only by the Auto classifier during read-only request inspection.
|
||||
@@ -6596,7 +6616,6 @@ mod tests {
|
||||
crate::retry_status::clear();
|
||||
crate::retry_status::clear_rate_limit();
|
||||
crate::retry_status::start(7, Duration::from_secs(60), "foreground sentinel");
|
||||
crate::retry_status::note_rate_limit(Duration::from_secs(60));
|
||||
let server = MockServer::start().await;
|
||||
Mock::given(method("POST"))
|
||||
.respond_with(ResponseTemplate::new(200).set_body_json(json!({
|
||||
@@ -6615,6 +6634,7 @@ mod tests {
|
||||
.await;
|
||||
|
||||
let client = deepseek_request_boundary_client("https://api.deepseek.com/v1", server.uri());
|
||||
crate::retry_status::note_rate_limit(&client.rate_limit_scope(), Duration::from_secs(60));
|
||||
let request = MessageRequest {
|
||||
model: "deepseek-v4-pro".to_string(),
|
||||
messages: vec![Message {
|
||||
@@ -6681,7 +6701,7 @@ mod tests {
|
||||
state => panic!("isolated success mutated retry state: {state:?}"),
|
||||
}
|
||||
assert!(
|
||||
crate::retry_status::rate_limit_remaining().is_some(),
|
||||
crate::retry_status::rate_limit_remaining(&client.rate_limit_scope()).is_some(),
|
||||
"isolated success must not clear the foreground provider pause"
|
||||
);
|
||||
crate::retry_status::clear();
|
||||
@@ -6697,7 +6717,6 @@ mod tests {
|
||||
crate::retry_status::clear();
|
||||
crate::retry_status::clear_rate_limit();
|
||||
crate::retry_status::start(9, Duration::from_secs(60), "foreground sentinel 429");
|
||||
crate::retry_status::note_rate_limit(Duration::from_secs(60));
|
||||
|
||||
let server = MockServer::start().await;
|
||||
Mock::given(method("POST"))
|
||||
@@ -6713,6 +6732,7 @@ mod tests {
|
||||
deepseek_request_boundary_client("https://api.deepseek.com/v1", server.uri());
|
||||
client.retry.enabled = false;
|
||||
client.retry.max_retries = 0;
|
||||
crate::retry_status::note_rate_limit(&client.rate_limit_scope(), Duration::from_secs(60));
|
||||
let request = MessageRequest {
|
||||
model: "deepseek-v4-pro".to_string(),
|
||||
messages: vec![Message {
|
||||
@@ -6751,8 +6771,8 @@ mod tests {
|
||||
}
|
||||
state => panic!("isolated 429 mutated retry state: {state:?}"),
|
||||
}
|
||||
let remaining =
|
||||
crate::retry_status::rate_limit_remaining().expect("foreground provider pause remains");
|
||||
let remaining = crate::retry_status::rate_limit_remaining(&client.rate_limit_scope())
|
||||
.expect("foreground provider pause remains");
|
||||
assert!(
|
||||
remaining < Duration::from_secs(70),
|
||||
"classifier Retry-After must not extend the global pause: {remaining:?}"
|
||||
|
||||
@@ -34,10 +34,10 @@ use crate::logging;
|
||||
use crate::tools::schema_sanitize;
|
||||
use codewhale_models::{ContentBlock, MessageRequest, MessageResponse, StreamEvent, Usage};
|
||||
|
||||
use super::CodewhaleClient;
|
||||
use super::prepared::WireDialect;
|
||||
use super::role_placement::{RolePlacement, role_placement};
|
||||
use super::wire::{extract_sse_data_value, next_sse_line};
|
||||
use super::{CodewhaleClient, ERROR_BODY_MAX_BYTES, bounded_error_text};
|
||||
use super::wire::{extract_sse_data_value, next_sse_line, push_sse_event_data};
|
||||
|
||||
/// Maximum `cache_control` breakpoints Anthropic accepts per request.
|
||||
const MAX_CACHE_BREAKPOINTS: usize = 4;
|
||||
@@ -213,78 +213,64 @@ impl CodewhaleClient {
|
||||
body
|
||||
}
|
||||
|
||||
/// Non-streaming send through the shared typed retry path
|
||||
/// (`send_with_retry`): 429/5xx/transport failures retry with backoff and
|
||||
/// honor `Retry-After`, and a final failure stays a downcastable
|
||||
/// `LlmError` so the engine can classify auth, rate-limit, context and
|
||||
/// invalid-request failures like every other wire.
|
||||
async fn send_anthropic_request(&self, url: &str, body: &Value) -> Result<reqwest::Response> {
|
||||
let url = self.messages_transport_url(url);
|
||||
self.wait_for_rate_limit().await;
|
||||
let response = self
|
||||
.http_client
|
||||
.post(&url)
|
||||
.header("Accept", "text/event-stream")
|
||||
.json(body)
|
||||
.send()
|
||||
.await
|
||||
.context("Anthropic Messages API request failed")?;
|
||||
self.check_anthropic_response(response).await
|
||||
}
|
||||
|
||||
/// Shared status/error-envelope handling for streaming and
|
||||
/// non-streaming Messages responses.
|
||||
async fn check_anthropic_response(
|
||||
&self,
|
||||
response: reqwest::Response,
|
||||
) -> Result<reqwest::Response> {
|
||||
let status = response.status();
|
||||
if !status.is_success() {
|
||||
let raw = bounded_error_text(response, ERROR_BODY_MAX_BYTES).await;
|
||||
let (error_type, message) = parse_anthropic_error_envelope(&raw);
|
||||
self.mark_request_failure(&format!("anthropic status={status}"))
|
||||
.await;
|
||||
anyhow::bail!("Anthropic API error (HTTP {status} {error_type}): {message}");
|
||||
}
|
||||
self.mark_request_success().await;
|
||||
Ok(response)
|
||||
let request_body =
|
||||
serde_json::to_vec(body).context("Failed to serialize Anthropic Messages request")?;
|
||||
self.send_with_retry(|| {
|
||||
self.http_client
|
||||
.post(&url)
|
||||
.header(reqwest::header::CONTENT_TYPE, "application/json")
|
||||
.header("Accept", "text/event-stream")
|
||||
.body(request_body.clone())
|
||||
})
|
||||
.await
|
||||
.context("Anthropic Messages API request failed")
|
||||
}
|
||||
|
||||
/// Open the streaming Messages request through the shared stream-entry
|
||||
/// transport policy: bounded header wait, dual-client selection, and at
|
||||
/// most one HTTP/1.1 fallback retry on a classified H2 header stall.
|
||||
/// Wire-specific request construction (headers, endpoint, body) stays
|
||||
/// here at the adapter edge.
|
||||
/// Inside each open attempt the provider retry loop (`send_with_retry`)
|
||||
/// handles rate limits and transient upstream failures before any stream
|
||||
/// body exists, as the Chat and Responses adapters do. Wire-specific
|
||||
/// request construction (headers, endpoint, body) stays here at the
|
||||
/// adapter edge.
|
||||
async fn open_anthropic_stream_response(
|
||||
&self,
|
||||
url: &str,
|
||||
body: &Value,
|
||||
) -> Result<reqwest::Response> {
|
||||
let url = self.messages_transport_url(url);
|
||||
let request_body =
|
||||
serde_json::to_vec(body).context("Failed to serialize Anthropic Messages request")?;
|
||||
let open_req = self.stream_open_request();
|
||||
let opened = super::stream_entry::open_sse_response(&open_req, |policy| {
|
||||
super::stream_entry::open_sse_response(&open_req, |policy| {
|
||||
let url = url.clone();
|
||||
let request_body = request_body.clone();
|
||||
async move {
|
||||
self.wait_for_rate_limit().await;
|
||||
let client = super::stream_entry::client_for_policy(
|
||||
&self.http_client,
|
||||
self.http1_fallback_client(),
|
||||
policy,
|
||||
);
|
||||
client
|
||||
.post(&url)
|
||||
.header("Accept", "text/event-stream")
|
||||
.json(body)
|
||||
.send()
|
||||
.await
|
||||
.context("Anthropic Messages API request failed")
|
||||
self.send_with_retry(|| {
|
||||
client
|
||||
.post(&url)
|
||||
.header(reqwest::header::CONTENT_TYPE, "application/json")
|
||||
.header("Accept", "text/event-stream")
|
||||
.body(request_body.clone())
|
||||
})
|
||||
.await
|
||||
.context("Anthropic Messages API request failed")
|
||||
}
|
||||
})
|
||||
.await;
|
||||
let response = match opened {
|
||||
Ok(response) => response,
|
||||
Err(err) => {
|
||||
self.mark_request_failure(&format!("anthropic stream open: {err}"))
|
||||
.await;
|
||||
return Err(err);
|
||||
}
|
||||
};
|
||||
self.check_anthropic_response(response).await
|
||||
.await
|
||||
}
|
||||
|
||||
/// Handle a streaming Messages API request.
|
||||
@@ -312,6 +298,9 @@ impl CodewhaleClient {
|
||||
// multi-byte UTF-8 char (CJK/emoji) split across HTTP/2 DATA is
|
||||
// never corrupted to U+FFFD. Genuine invalid bytes fail closed.
|
||||
let mut buffer: Vec<u8> = Vec::new();
|
||||
// `data:` fields of the event being assembled, dispatched at the
|
||||
// blank line that ends it (or at stream end).
|
||||
let mut event_data = String::new();
|
||||
let stream_start = std::time::Instant::now();
|
||||
let mut last_chunk_at = std::time::Instant::now();
|
||||
let mut bytes_received: usize = 0;
|
||||
@@ -350,22 +339,36 @@ impl CodewhaleClient {
|
||||
}
|
||||
|
||||
loop {
|
||||
let line = match next_sse_line(&mut buffer, ended) {
|
||||
Ok(Some(line)) => line,
|
||||
let data = match next_sse_line(&mut buffer, ended) {
|
||||
// A blank line ends the event.
|
||||
Ok(Some(line)) if line.is_empty() => std::mem::take(&mut event_data),
|
||||
Ok(Some(line)) => {
|
||||
// `event:` lines are redundant (the data payload
|
||||
// carries `type`) and comment/heartbeat lines are
|
||||
// ignorable.
|
||||
if let Some(value) = extract_sse_data_value(&line)
|
||||
&& let Err(err) = push_sse_event_data(&mut event_data, value)
|
||||
{
|
||||
yield Err(anyhow::anyhow!("{err}"));
|
||||
return;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
// The final event may arrive without its blank line.
|
||||
Ok(None) if ended && !event_data.is_empty() => {
|
||||
std::mem::take(&mut event_data)
|
||||
}
|
||||
Ok(None) => break,
|
||||
Err(err) => {
|
||||
yield Err(anyhow::anyhow!("{err}"));
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
// `event:` lines are redundant (the data payload carries
|
||||
// `type`) and comment/heartbeat lines are ignorable.
|
||||
let Some(data) = extract_sse_data_value(&line) else {
|
||||
if data.is_empty() {
|
||||
continue;
|
||||
};
|
||||
}
|
||||
|
||||
match convert_anthropic_sse_data(data) {
|
||||
match convert_anthropic_sse_data(&data) {
|
||||
Some(Ok(StreamEvent::Error { error })) => {
|
||||
let (error_type, message) = anthropic_error_fields(&error);
|
||||
yield Err(anyhow::anyhow!(
|
||||
@@ -391,6 +394,12 @@ impl CodewhaleClient {
|
||||
break;
|
||||
}
|
||||
}
|
||||
// Only `message_stop` (returned above) or a provider error proves
|
||||
// the response is whole. A bare HTTP EOF is truncation, and ending
|
||||
// the stream quietly would hand the turn a partial answer as done.
|
||||
yield Err(anyhow::anyhow!(
|
||||
"Anthropic Messages stream closed before message_stop"
|
||||
));
|
||||
};
|
||||
|
||||
Ok(Box::pin(stream))
|
||||
@@ -994,14 +1003,6 @@ fn parse_anthropic_usage(usage: &Value) -> Usage {
|
||||
/// Extract `error.type` / `error.message` from an Anthropic error envelope
|
||||
/// (`{"type":"error","error":{"type":...,"message":...}}`), falling back to
|
||||
/// the raw body so nothing is swallowed.
|
||||
fn parse_anthropic_error_envelope(raw: &str) -> (String, String) {
|
||||
let Ok(value) = serde_json::from_str::<Value>(raw) else {
|
||||
return ("unknown".to_string(), raw.to_string());
|
||||
};
|
||||
let error = value.get("error").unwrap_or(&value);
|
||||
anthropic_error_fields(error)
|
||||
}
|
||||
|
||||
fn anthropic_error_fields(error: &Value) -> (String, String) {
|
||||
let error_type = error
|
||||
.get("type")
|
||||
@@ -2027,19 +2028,6 @@ mod tests {
|
||||
assert_eq!(usage.prompt_cache_write_tokens, Some(2045));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn error_envelope_parses_type_and_message() {
|
||||
let (error_type, message) = parse_anthropic_error_envelope(
|
||||
r#"{"type":"error","error":{"type":"rate_limit_error","message":"Too many requests"},"request_id":"req_1"}"#,
|
||||
);
|
||||
assert_eq!(error_type, "rate_limit_error");
|
||||
assert_eq!(message, "Too many requests");
|
||||
|
||||
let (error_type, message) = parse_anthropic_error_envelope("upstream blew up");
|
||||
assert_eq!(error_type, "unknown");
|
||||
assert_eq!(message, "upstream blew up");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn data_url_image_becomes_a_base64_source_not_a_url_source() {
|
||||
// Anthropic rejects a `data:` URL under `{"type":"url"}`. This is the
|
||||
@@ -2248,6 +2236,73 @@ mod tests {
|
||||
assert!(saw_stop, "message_stop should arrive through the seam");
|
||||
}
|
||||
|
||||
async fn collect_anthropic_stream(body: &'static str) -> Vec<Result<StreamEvent>> {
|
||||
use futures_util::StreamExt;
|
||||
use wiremock::matchers::{method, path};
|
||||
use wiremock::{Mock, MockServer, ResponseTemplate};
|
||||
|
||||
let server = MockServer::start().await;
|
||||
Mock::given(method("POST"))
|
||||
.and(path("/v1/messages"))
|
||||
.respond_with(
|
||||
ResponseTemplate::new(200)
|
||||
.insert_header("Content-Type", "text/event-stream")
|
||||
.set_body_string(body),
|
||||
)
|
||||
.mount(&server)
|
||||
.await;
|
||||
let client = deepseek_test_client(&server.uri());
|
||||
let stream = client
|
||||
.handle_anthropic_stream(
|
||||
&client
|
||||
.prepare_outbound_request(request_with("deepseek-v4", None, None, None), true)
|
||||
.expect("anthropic request prepares"),
|
||||
)
|
||||
.await
|
||||
.expect("stream opens");
|
||||
tokio::time::timeout(std::time::Duration::from_secs(5), stream.collect())
|
||||
.await
|
||||
.expect("stream ends")
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn anthropic_stream_joins_multiline_data_fields_into_one_event() {
|
||||
use codewhale_models::Delta;
|
||||
// One JSON payload split across two `data:` fields of a single event.
|
||||
let events = collect_anthropic_stream(concat!(
|
||||
"event: content_block_delta\n",
|
||||
"data: {\"type\":\"content_block_delta\",\"index\":0,\n",
|
||||
"data: \"delta\":{\"type\":\"text_delta\",\"text\":\"joined\"}}\n\n",
|
||||
"data: {\"type\":\"message_stop\"}\n\n",
|
||||
))
|
||||
.await;
|
||||
assert!(
|
||||
events.iter().any(|event| matches!(
|
||||
event,
|
||||
Ok(StreamEvent::ContentBlockDelta {
|
||||
delta: Delta::TextDelta { text },
|
||||
..
|
||||
}) if text == "joined"
|
||||
)),
|
||||
"the split event was lost: {events:?}"
|
||||
);
|
||||
assert!(matches!(events.last(), Some(Ok(StreamEvent::MessageStop))));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn anthropic_stream_eof_without_message_stop_is_an_error() {
|
||||
let events = collect_anthropic_stream(
|
||||
"data: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\"partial\"}}\n\n",
|
||||
)
|
||||
.await;
|
||||
let last = events.last().expect("the delta and a terminal item");
|
||||
assert!(
|
||||
last.as_ref()
|
||||
.is_err_and(|error| error.to_string().contains("closed before message_stop")),
|
||||
"a truncated stream must not end quietly: {events:?}"
|
||||
);
|
||||
}
|
||||
|
||||
/// Fault injection (#6184): a provider that answers the headers and then
|
||||
/// sends nothing fails the stream at the first-byte bound with a
|
||||
/// distinct error and a `crashes/` stall record, instead of holding the
|
||||
@@ -2344,10 +2399,63 @@ mod tests {
|
||||
Ok(_) => panic!("auth errors must fail fast"),
|
||||
Err(err) => err,
|
||||
};
|
||||
let text = err.to_string();
|
||||
// Typed through the shared classifier, provider message kept.
|
||||
assert!(
|
||||
text.contains("HTTP 401") && text.contains("authentication_error"),
|
||||
"error envelope should be preserved: {text}"
|
||||
matches!(
|
||||
err.chain()
|
||||
.find_map(|cause| cause.downcast_ref::<crate::llm_client::LlmError>()),
|
||||
Some(crate::llm_client::LlmError::AuthenticationError(_))
|
||||
),
|
||||
"a 401 must stay a typed authentication error: {err:#}"
|
||||
);
|
||||
assert!(format!("{err:#}").contains("bad key"), "{err:#}");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn anthropic_rate_limit_is_retried_honoring_retry_after() {
|
||||
use wiremock::matchers::{method, path};
|
||||
use wiremock::{Mock, MockServer, ResponseTemplate};
|
||||
|
||||
let server = MockServer::start().await;
|
||||
Mock::given(method("POST"))
|
||||
.and(path("/v1/messages"))
|
||||
.respond_with(
|
||||
ResponseTemplate::new(429)
|
||||
.insert_header("retry-after", "0")
|
||||
.set_body_string(
|
||||
"{\"type\":\"error\",\"error\":{\"type\":\"rate_limit_error\",\"message\":\"slow down\"}}",
|
||||
),
|
||||
)
|
||||
.up_to_n_times(1)
|
||||
.expect(1)
|
||||
.mount(&server)
|
||||
.await;
|
||||
Mock::given(method("POST"))
|
||||
.and(path("/v1/messages"))
|
||||
.respond_with(
|
||||
ResponseTemplate::new(200)
|
||||
.insert_header("Content-Type", "text/event-stream")
|
||||
.set_body_string("data: {\"type\":\"message_stop\"}\n\n"),
|
||||
)
|
||||
.expect(1)
|
||||
.mount(&server)
|
||||
.await;
|
||||
|
||||
let mut client = deepseek_test_client(&server.uri());
|
||||
client.retry.enabled = true;
|
||||
client.retry.max_retries = 2;
|
||||
client.retry.initial_delay = 0.0;
|
||||
client.retry.max_delay = 0.0;
|
||||
let stream = client
|
||||
.handle_anthropic_stream(
|
||||
&client
|
||||
.prepare_outbound_request(request_with("deepseek-v4", None, None, None), true)
|
||||
.expect("anthropic request prepares"),
|
||||
)
|
||||
.await;
|
||||
assert!(
|
||||
stream.is_ok(),
|
||||
"a 429 before the stream body must be retried, not surfaced"
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@@ -33,7 +33,7 @@ use codewhale_models::{
|
||||
|
||||
use super::prepared::WireDialect;
|
||||
use super::role_placement::{RolePlacement, role_placement};
|
||||
use super::wire::{extract_sse_data_value, flush_sse_line, take_sse_line};
|
||||
use super::wire::{extract_sse_data_value, flush_sse_line, push_sse_event_data, take_sse_line};
|
||||
use super::{
|
||||
CodewhaleClient, ERROR_BODY_MAX_BYTES, SSE_BACKPRESSURE_HIGH_WATERMARK,
|
||||
SSE_BACKPRESSURE_SLEEP_MS, SSE_MAX_LINES_PER_CHUNK, acquire_stream_buffer,
|
||||
@@ -1603,15 +1603,13 @@ impl CodewhaleClient {
|
||||
continue;
|
||||
}
|
||||
|
||||
if let Some(data) = extract_sse_data_value(&line) {
|
||||
// The SSE spec joins multiple `data:` fields within one
|
||||
// event with '\n'; concatenating with no separator would
|
||||
// yield `{…}{…}` and fail JSON parsing, silently dropping
|
||||
// the frame.
|
||||
if !line_buf.is_empty() {
|
||||
line_buf.push('\n');
|
||||
}
|
||||
line_buf.push_str(data);
|
||||
if let Some(data) = extract_sse_data_value(&line)
|
||||
&& let Err(err) = push_sse_event_data(&mut line_buf, data)
|
||||
{
|
||||
decode_failed = true;
|
||||
stream_failed = true;
|
||||
yield Err(anyhow::anyhow!("{err}"));
|
||||
break 'stream;
|
||||
}
|
||||
// Ignore other SSE fields (event:, id:, retry:)
|
||||
|
||||
@@ -1639,11 +1637,12 @@ impl CodewhaleClient {
|
||||
if !saw_done && !decode_failed {
|
||||
match flush_sse_line(&mut byte_buf) {
|
||||
Ok(Some(line)) => {
|
||||
if let Some(data) = extract_sse_data_value(&line) {
|
||||
if !line_buf.is_empty() {
|
||||
line_buf.push('\n');
|
||||
}
|
||||
line_buf.push_str(data);
|
||||
if let Some(data) = extract_sse_data_value(&line)
|
||||
&& let Err(err) = push_sse_event_data(&mut line_buf, data)
|
||||
{
|
||||
decode_failed = true;
|
||||
stream_failed = true;
|
||||
yield Err(anyhow::anyhow!("{err}"));
|
||||
}
|
||||
}
|
||||
Ok(None) => {}
|
||||
|
||||
@@ -21,7 +21,7 @@ use codewhale_models::{
|
||||
|
||||
use super::prepared::WireDialect;
|
||||
use super::role_placement::{RolePlacement, role_placement};
|
||||
use super::wire::{extract_sse_data_value, next_sse_line};
|
||||
use super::wire::{extract_sse_data_value, next_sse_line, push_sse_event_data};
|
||||
use super::{
|
||||
CodewhaleClient, ERROR_BODY_MAX_BYTES, bounded_error_text, from_api_tool_name,
|
||||
system_to_instructions, to_api_tool_name,
|
||||
@@ -255,6 +255,9 @@ impl CodewhaleClient {
|
||||
// multi-byte UTF-8 char split across HTTP/2 DATA is never
|
||||
// corrupted to U+FFFD. Genuine invalid bytes fail closed.
|
||||
let mut buffer: Vec<u8> = Vec::new();
|
||||
// `data:` fields of the event being assembled, dispatched at the
|
||||
// blank line that ends it (or at stream end).
|
||||
let mut event_data = String::new();
|
||||
let mut done = false;
|
||||
let mut ended = false;
|
||||
let mut content_block_counter: u32 = 0;
|
||||
@@ -297,25 +300,37 @@ impl CodewhaleClient {
|
||||
|
||||
// Process complete SSE lines, and the unterminated tail at stream end.
|
||||
loop {
|
||||
let line = match next_sse_line(&mut buffer, ended) {
|
||||
Ok(Some(line)) => line,
|
||||
let data = match next_sse_line(&mut buffer, ended) {
|
||||
// A blank line ends the event.
|
||||
Ok(Some(line)) if line.is_empty() => std::mem::take(&mut event_data),
|
||||
Ok(Some(line)) => {
|
||||
if line.starts_with(':') {
|
||||
// SSE comment keep-alive: the provider is alive (#6184).
|
||||
yield Ok(StreamEvent::Ping);
|
||||
} else if let Some(value) = extract_sse_data_value(&line)
|
||||
&& let Err(err) = push_sse_event_data(&mut event_data, value)
|
||||
{
|
||||
yield Err(anyhow::anyhow!("{err}"));
|
||||
return;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
// The final event may arrive without its blank line.
|
||||
Ok(None) if ended && !event_data.is_empty() => {
|
||||
std::mem::take(&mut event_data)
|
||||
}
|
||||
Ok(None) => break,
|
||||
Err(err) => {
|
||||
yield Err(anyhow::anyhow!("{err}"));
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
if line.is_empty() {
|
||||
continue;
|
||||
}
|
||||
if line.starts_with(':') {
|
||||
// SSE comment keep-alive: the provider is alive (#6184).
|
||||
yield Ok(StreamEvent::Ping);
|
||||
if data.is_empty() {
|
||||
continue;
|
||||
}
|
||||
|
||||
if let Some(data) = extract_sse_data_value(&line) {
|
||||
{
|
||||
let data = data.as_str();
|
||||
if data == "[DONE]" {
|
||||
done = true;
|
||||
break;
|
||||
@@ -561,7 +576,15 @@ impl CodewhaleClient {
|
||||
}
|
||||
}
|
||||
|
||||
// Emit MessageStop.
|
||||
// Only `[DONE]` or `response.completed`/`response.incomplete`
|
||||
// proves the response is whole. A bare HTTP EOF is truncation, and
|
||||
// a MessageStop here would hand the turn a partial answer as done.
|
||||
if !done {
|
||||
yield Err(anyhow::anyhow!(
|
||||
"Responses stream closed before response.completed or [DONE]"
|
||||
));
|
||||
return;
|
||||
}
|
||||
yield Ok(StreamEvent::MessageStop);
|
||||
};
|
||||
|
||||
|
||||
@@ -234,6 +234,82 @@ async fn responses_stream_retries_upstream_499_before_streaming() {
|
||||
assert_eq!(attempts.load(Ordering::SeqCst), 2);
|
||||
}
|
||||
|
||||
async fn collect_responses_stream(sse_body: &'static str) -> Vec<Result<StreamEvent>> {
|
||||
let server = MockServer::start().await;
|
||||
Mock::given(method("POST"))
|
||||
.and(path(CODEX_RESPONSES_PATH))
|
||||
.respond_with(
|
||||
ResponseTemplate::new(200)
|
||||
.insert_header("Content-Type", "text/event-stream")
|
||||
.set_body_string(sse_body),
|
||||
)
|
||||
.mount(&server)
|
||||
.await;
|
||||
let client = {
|
||||
let _env_lock = crate::test_support::lock_test_env();
|
||||
let _codex_token =
|
||||
crate::test_support::EnvVarGuard::set("OPENAI_CODEX_ACCESS_TOKEN", "test-token");
|
||||
let _legacy_codex_token = crate::test_support::EnvVarGuard::remove("CODEX_ACCESS_TOKEN");
|
||||
CodewhaleClient::new(&test_codex_config(&server)).unwrap()
|
||||
};
|
||||
let stream = client
|
||||
.handle_responses_stream(
|
||||
&client
|
||||
.prepare_outbound_request(minimal_responses_request(), true)
|
||||
.expect("responses request prepares"),
|
||||
)
|
||||
.await
|
||||
.expect("Responses stream opens");
|
||||
tokio::time::timeout(std::time::Duration::from_secs(5), stream.collect())
|
||||
.await
|
||||
.expect("stream ends")
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn responses_stream_eof_without_a_terminal_event_is_an_error() {
|
||||
let events = collect_responses_stream(concat!(
|
||||
"data: {\"type\":\"response.output_item.added\",\"item\":{\"type\":\"message\",\"id\":\"m\"}}\n\n",
|
||||
"data: {\"type\":\"response.output_text.delta\",\"delta\":\"partial\"}\n\n",
|
||||
))
|
||||
.await;
|
||||
assert!(
|
||||
!events
|
||||
.iter()
|
||||
.any(|event| matches!(event, Ok(StreamEvent::MessageStop))),
|
||||
"a truncated stream must not report MessageStop: {events:?}"
|
||||
);
|
||||
assert!(
|
||||
events
|
||||
.last()
|
||||
.is_some_and(|event| event.as_ref().is_err_and(|error| error
|
||||
.to_string()
|
||||
.contains("closed before response.completed"))),
|
||||
"{events:?}"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn responses_stream_joins_multiline_data_fields_into_one_event() {
|
||||
let events = collect_responses_stream(concat!(
|
||||
"data: {\"type\":\"response.output_item.added\",\"item\":{\"type\":\"message\",\"id\":\"m\"}}\n\n",
|
||||
"data: {\"type\":\"response.output_text.delta\",\n",
|
||||
"data: \"delta\":\"joined\"}\n\n",
|
||||
"data: [DONE]\n\n",
|
||||
))
|
||||
.await;
|
||||
assert!(
|
||||
events.iter().any(|event| matches!(
|
||||
event,
|
||||
Ok(StreamEvent::ContentBlockDelta {
|
||||
delta: Delta::TextDelta { text },
|
||||
..
|
||||
}) if text == "joined"
|
||||
)),
|
||||
"the split event was lost: {events:?}"
|
||||
);
|
||||
assert!(matches!(events.last(), Some(Ok(StreamEvent::MessageStop))));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn responses_stream_finishes_on_semantic_terminal_event_without_done_marker() {
|
||||
let server = MockServer::start().await;
|
||||
|
||||
@@ -98,6 +98,10 @@ pub(super) fn extract_sse_data_value(line: &str) -> Option<&str> {
|
||||
/// Hard ceiling for one pending SSE line, matching the MCP frame limit.
|
||||
/// This does not bound the size of a multi-line event or the whole stream.
|
||||
const MAX_SSE_LINE_BYTES: usize = 8 * 1024 * 1024;
|
||||
/// Hard ceiling for one assembled SSE event (its `data:` fields joined), so a
|
||||
/// peer that never sends the blank line ending an event cannot grow memory
|
||||
/// without bound. Same size as the single-line ceiling.
|
||||
const MAX_SSE_EVENT_BYTES: usize = MAX_SSE_LINE_BYTES;
|
||||
|
||||
/// Invalid or oversized SSE line (or unterminated flush).
|
||||
///
|
||||
@@ -109,6 +113,7 @@ const MAX_SSE_LINE_BYTES: usize = 8 * 1024 * 1024;
|
||||
pub(super) enum SseLineError {
|
||||
InvalidUtf8 { valid_up_to: usize },
|
||||
TooLong,
|
||||
EventTooLong,
|
||||
}
|
||||
|
||||
impl std::fmt::Display for SseLineError {
|
||||
@@ -121,6 +126,10 @@ impl std::fmt::Display for SseLineError {
|
||||
f,
|
||||
"SSE line exceeded {MAX_SSE_LINE_BYTES} bytes (8 MiB) — aborting stream"
|
||||
),
|
||||
Self::EventTooLong => write!(
|
||||
f,
|
||||
"SSE event exceeded {MAX_SSE_EVENT_BYTES} bytes (8 MiB) — aborting stream"
|
||||
),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -184,6 +193,27 @@ pub(super) fn flush_sse_line(buffer: &mut Vec<u8>) -> Result<Option<String>, Sse
|
||||
decoded.map(|line| (!line.is_empty()).then_some(line))
|
||||
}
|
||||
|
||||
/// Append one `data:` field to the event being assembled. The SSE spec joins
|
||||
/// the `data:` fields of one event with '\n' and dispatches the event at the
|
||||
/// blank line that ends it; a provider may split one JSON payload across
|
||||
/// several fields, so parsing each field alone would drop the event. Fails
|
||||
/// closed (clearing the event) past [`MAX_SSE_EVENT_BYTES`].
|
||||
pub(super) fn push_sse_event_data(event: &mut String, data: &str) -> Result<(), SseLineError> {
|
||||
let needed = event
|
||||
.len()
|
||||
.saturating_add(usize::from(!event.is_empty()))
|
||||
.saturating_add(data.len());
|
||||
if needed > MAX_SSE_EVENT_BYTES {
|
||||
event.clear();
|
||||
return Err(SseLineError::EventTooLong);
|
||||
}
|
||||
if !event.is_empty() {
|
||||
event.push('\n');
|
||||
}
|
||||
event.push_str(data);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Next decoded SSE line. When `at_end` is false, wait for `\n`. When `at_end`
|
||||
/// is true, also flush an unterminated tail (stream closed).
|
||||
pub(super) fn next_sse_line(
|
||||
|
||||
@@ -21,7 +21,6 @@
|
||||
//! provider-accepted active observation.
|
||||
|
||||
use std::fs;
|
||||
use std::io::Write;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::process::Command;
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
@@ -288,25 +287,13 @@ impl CloudJobStore {
|
||||
}
|
||||
|
||||
/// Persist a job atomically. Never writes credentials.
|
||||
///
|
||||
/// The record is written through a unique temporary file (never a fixed
|
||||
/// `.json.tmp` two writers would share) and renamed into place while the
|
||||
/// job's lock is held, so it cannot interleave with a
|
||||
/// [`Self::save_unless_canceled`] or a cancel.
|
||||
pub fn save(&self, job: &CloudJob) -> Result<()> {
|
||||
fs::create_dir_all(&self.root).context("failed to create cloud-jobs directory")?;
|
||||
let path = self.job_path(&job.id)?;
|
||||
let tmp = path.with_extension("json.tmp");
|
||||
let body = serde_json::to_vec_pretty(job).context("failed to encode cloud job")?;
|
||||
{
|
||||
let mut file =
|
||||
fs::File::create(&tmp).context("failed to start a private cloud job write")?;
|
||||
file.write_all(&body)
|
||||
.context("failed to write the cloud job record")?;
|
||||
file.sync_all().ok();
|
||||
}
|
||||
#[cfg(unix)]
|
||||
{
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
let _ = fs::set_permissions(&tmp, fs::Permissions::from_mode(0o600));
|
||||
}
|
||||
fs::rename(&tmp, &path).context("failed to commit the cloud job record")?;
|
||||
Ok(())
|
||||
self.with_job_lock(&job.id, || self.write_record(job))
|
||||
}
|
||||
|
||||
/// Cancel-authoritative save: refuse to overwrite a `canceled` record.
|
||||
@@ -319,19 +306,53 @@ impl CloudJobStore {
|
||||
/// persisted record is already `canceled`, `Ok(true)` after a normal
|
||||
/// save.
|
||||
///
|
||||
/// This is load-check-save: the store is file-backed with no
|
||||
/// cross-process lock, so the check cannot remove the load→save window
|
||||
/// entirely — it narrows the clobber window from a whole phase (seconds
|
||||
/// to minutes) to the span of one save, which is the single-writer
|
||||
/// discipline this store assumes elsewhere.
|
||||
/// The load, check and save run under the job's advisory file lock, and
|
||||
/// [`cancel_job`] flips the record under the same lock, so the check and
|
||||
/// the write are one step with respect to a cancel in another process.
|
||||
pub fn save_unless_canceled(&self, job: &CloudJob) -> Result<bool> {
|
||||
if let Ok(current) = self.load(&job.id)
|
||||
&& current.status == CloudJobStatus::Canceled
|
||||
{
|
||||
return Ok(false);
|
||||
}
|
||||
self.save(job)?;
|
||||
Ok(true)
|
||||
self.with_job_lock(&job.id, || {
|
||||
match self.load(&job.id) {
|
||||
Ok(current) if current.status == CloudJobStatus::Canceled => return Ok(false),
|
||||
Ok(_) => {}
|
||||
Err(error)
|
||||
if error
|
||||
.downcast_ref::<std::io::Error>()
|
||||
.is_some_and(|error| error.kind() == std::io::ErrorKind::NotFound) => {}
|
||||
Err(error) => return Err(error.context("cannot verify the persisted job status")),
|
||||
}
|
||||
self.write_record(job)?;
|
||||
Ok(true)
|
||||
})
|
||||
}
|
||||
|
||||
/// Run `f` holding the job's exclusive advisory lock
|
||||
/// (`<root>/<id>.json.lock`). The lock is released when `f` returns.
|
||||
///
|
||||
/// Limitation: the lock is advisory, so it serializes Codewhale writers
|
||||
/// only; it is not reentrant, so `f` must not call a locking method.
|
||||
fn with_job_lock<T>(&self, id: &str, f: impl FnOnce() -> Result<T>) -> Result<T> {
|
||||
let path = self.job_path(id)?;
|
||||
fs::create_dir_all(&self.root).context("failed to create cloud-jobs directory")?;
|
||||
let lock_file = fs::OpenOptions::new()
|
||||
.create(true)
|
||||
.truncate(false)
|
||||
.read(true)
|
||||
.write(true)
|
||||
.open(path.with_extension("json.lock"))
|
||||
.context("failed to open the cloud job lock")?;
|
||||
let mut lock = fd_lock::RwLock::new(lock_file);
|
||||
let _guard = lock
|
||||
.write()
|
||||
.context("failed to lock the cloud job record")?;
|
||||
f()
|
||||
}
|
||||
|
||||
/// Write the record through a unique private temporary file. Callers
|
||||
/// hold the job lock.
|
||||
fn write_record(&self, job: &CloudJob) -> Result<()> {
|
||||
let path = self.job_path(&job.id)?;
|
||||
let body = serde_json::to_vec_pretty(job).context("failed to encode cloud job")?;
|
||||
crate::utils::write_atomic(&path, &body).context("failed to commit the cloud job record")
|
||||
}
|
||||
|
||||
/// Load one job by id.
|
||||
@@ -789,23 +810,32 @@ pub fn cancel_job(
|
||||
id: &str,
|
||||
launcher: &dyn DaytonaLauncher,
|
||||
) -> Result<CloudJob> {
|
||||
let mut job = store.load(id)?;
|
||||
if matches!(
|
||||
job.status,
|
||||
CloudJobStatus::Canceled | CloudJobStatus::Failed | CloudJobStatus::Refused
|
||||
) {
|
||||
return Ok(job);
|
||||
}
|
||||
let had_sandbox = job.sandbox_id.is_some();
|
||||
let sandbox_may_exist = had_sandbox || job.sandbox_pending;
|
||||
job.status = CloudJobStatus::Canceled;
|
||||
job.finished_unix = Some(unix_now());
|
||||
job.note = if sandbox_may_exist {
|
||||
"Canceled locally; the cloud agent sandbox is being torn down.".to_string()
|
||||
} else {
|
||||
"Canceled locally before a sandbox was created.".to_string()
|
||||
// The flip is one locked read-modify-write, so a runner phase save cannot
|
||||
// land between this load and this write (and lose either the cancel or
|
||||
// the sandbox id the runner just recorded).
|
||||
let flipped = store.with_job_lock(id, || {
|
||||
let mut job = store.load(id)?;
|
||||
if matches!(
|
||||
job.status,
|
||||
CloudJobStatus::Canceled | CloudJobStatus::Failed | CloudJobStatus::Refused
|
||||
) {
|
||||
return Ok((job, false));
|
||||
}
|
||||
let sandbox_may_exist = job.sandbox_id.is_some() || job.sandbox_pending;
|
||||
job.status = CloudJobStatus::Canceled;
|
||||
job.finished_unix = Some(unix_now());
|
||||
job.note = if sandbox_may_exist {
|
||||
"Canceled locally; the cloud agent sandbox is being torn down.".to_string()
|
||||
} else {
|
||||
"Canceled locally before a sandbox was created.".to_string()
|
||||
};
|
||||
store.write_record(&job)?;
|
||||
Ok((job, true))
|
||||
})?;
|
||||
let mut job = match flipped {
|
||||
(job, false) => return Ok(job),
|
||||
(job, true) => job,
|
||||
};
|
||||
store.save(&job)?;
|
||||
if let Some(sandbox_id) = job.sandbox_id.clone() {
|
||||
let receipt = SandboxReceipt {
|
||||
sandbox_id,
|
||||
@@ -3181,6 +3211,75 @@ mod tests {
|
||||
assert_eq!(persisted.finished_unix, Some(10_000_060));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_phase_save_preserves_an_unreadable_job_record() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let store = CloudJobStore::from_path(temp.path().join("jobs"));
|
||||
let job = stored_job(CloudJobStatus::Running, 10_000_000);
|
||||
// A genuinely absent record can be created by the first phase save.
|
||||
assert!(store.save_unless_canceled(&job).unwrap());
|
||||
let path = store.job_path(&job.id).unwrap();
|
||||
let damaged = b"{\"status\":\"canceled\", interrupted write";
|
||||
fs::write(&path, damaged).unwrap();
|
||||
let error = store.save_unless_canceled(&job).unwrap_err();
|
||||
assert!(format!("{error:#}").contains("cannot verify the persisted job status"));
|
||||
assert_eq!(fs::read(&path).unwrap(), damaged);
|
||||
|
||||
// An I/O failure is uncertainty too, rather than permission to replace it.
|
||||
fs::remove_file(&path).unwrap();
|
||||
fs::create_dir(&path).unwrap();
|
||||
assert!(store.save_unless_canceled(&job).is_err());
|
||||
assert!(path.is_dir());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_phase_save_waits_for_a_cancel_holding_the_job_lock() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let store = CloudJobStore::from_path(temp.path().join("jobs"));
|
||||
let job = stored_job(CloudJobStatus::Running, 10_000_000);
|
||||
store.save(&job).unwrap();
|
||||
let mut runner_copy = job.clone();
|
||||
runner_copy.status = CloudJobStatus::OpeningPr;
|
||||
|
||||
// A cancel in progress holds the lock between its load and its write;
|
||||
// the runner's check-and-save must not slip into that window.
|
||||
let outcome = store
|
||||
.with_job_lock(&job.id, || {
|
||||
let runner_store = store.clone();
|
||||
let runner = std::thread::spawn(move || {
|
||||
runner_store.save_unless_canceled(&runner_copy).unwrap()
|
||||
});
|
||||
std::thread::sleep(std::time::Duration::from_millis(200));
|
||||
assert!(
|
||||
!runner.is_finished(),
|
||||
"the phase save ran inside the cancel's read-modify-write"
|
||||
);
|
||||
let mut canceled = store.load(&job.id)?;
|
||||
canceled.status = CloudJobStatus::Canceled;
|
||||
store.write_record(&canceled)?;
|
||||
Ok(runner)
|
||||
})
|
||||
.unwrap()
|
||||
.join()
|
||||
.unwrap();
|
||||
assert!(
|
||||
!outcome,
|
||||
"the phase save must see the cancel and stand down"
|
||||
);
|
||||
assert_eq!(
|
||||
store.load(&job.id).unwrap().status,
|
||||
CloudJobStatus::Canceled
|
||||
);
|
||||
assert!(
|
||||
!temp
|
||||
.path()
|
||||
.join("jobs")
|
||||
.join(format!("{}.json.tmp", job.id))
|
||||
.exists(),
|
||||
"records are written through unique temporaries"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sweep_fails_stale_active_jobs_and_tears_down_their_sandboxes() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
|
||||
@@ -2348,32 +2348,39 @@ pub fn set_config_value(app: &mut App, key: &str, value: &str, persist: bool) ->
|
||||
}
|
||||
let next_path = PathBuf::from(expand_tilde(value));
|
||||
let path_changed = next_path != app.mcp_config_path;
|
||||
app.mcp_config_path = next_path;
|
||||
if path_changed {
|
||||
app.mcp_reload_required = true;
|
||||
}
|
||||
let reload_note = if path_changed {
|
||||
"; run /mcp reload to rebuild the live tool pool"
|
||||
} else {
|
||||
""
|
||||
};
|
||||
let message = if persist {
|
||||
// Persist before touching live state (C01-08): a failed save
|
||||
// must leave the session exactly as it was, not report failure
|
||||
// over a path and reload flag that already moved.
|
||||
let saved_to = if persist {
|
||||
match persist_root_string_key(app.config_path.as_deref(), "mcp_config_path", value)
|
||||
{
|
||||
Ok(path) => format!(
|
||||
"mcp_config_path = {} (saved to {}){}",
|
||||
app.mcp_config_path.display(),
|
||||
path.display(),
|
||||
reload_note
|
||||
),
|
||||
Ok(path) => Some(path),
|
||||
Err(err) => return CommandResult::error(format!("Failed to save: {err}")),
|
||||
}
|
||||
} else {
|
||||
format!(
|
||||
None
|
||||
};
|
||||
app.mcp_config_path = next_path;
|
||||
if path_changed {
|
||||
app.mcp_reload_required = true;
|
||||
}
|
||||
let message = match saved_to {
|
||||
Some(path) => format!(
|
||||
"mcp_config_path = {} (saved to {}){}",
|
||||
app.mcp_config_path.display(),
|
||||
path.display(),
|
||||
reload_note
|
||||
),
|
||||
None => format!(
|
||||
"mcp_config_path = {} (session only){}",
|
||||
app.mcp_config_path.display(),
|
||||
reload_note
|
||||
)
|
||||
),
|
||||
};
|
||||
return CommandResult::message(message);
|
||||
}
|
||||
@@ -2525,7 +2532,6 @@ pub fn set_config_value(app: &mut App, key: &str, value: &str, persist: bool) ->
|
||||
} else {
|
||||
raw
|
||||
};
|
||||
app.stream_chunk_timeout_secs = resolved;
|
||||
let value_label = stream_chunk_timeout_value_label(raw, resolved);
|
||||
if persist {
|
||||
match persist_table_value_key(
|
||||
@@ -2535,6 +2541,9 @@ pub fn set_config_value(app: &mut App, key: &str, value: &str, persist: bool) ->
|
||||
(raw as i64).into(), // validated above: at most 3600
|
||||
) {
|
||||
Ok(path) => {
|
||||
// Live state moves only with the engine action and
|
||||
// only after the save landed (C01-08).
|
||||
app.stream_chunk_timeout_secs = resolved;
|
||||
return CommandResult::with_message_and_action(
|
||||
format!(
|
||||
"stream_chunk_timeout_secs = {value_label} (saved to {}; affects subsequent turns in this session)",
|
||||
@@ -2546,6 +2555,7 @@ pub fn set_config_value(app: &mut App, key: &str, value: &str, persist: bool) ->
|
||||
Err(err) => return CommandResult::error(format!("Failed to save: {err}")),
|
||||
}
|
||||
}
|
||||
app.stream_chunk_timeout_secs = resolved;
|
||||
return CommandResult::with_message_and_action(
|
||||
format!(
|
||||
"stream_chunk_timeout_secs = {value_label} (session only; affects subsequent turns in this session)"
|
||||
@@ -2731,14 +2741,8 @@ pub fn set_config_value(app: &mut App, key: &str, value: &str, persist: bool) ->
|
||||
Ok(value) => value,
|
||||
Err(err) => return CommandResult::error(err),
|
||||
};
|
||||
match field {
|
||||
"keep_header" => app.mini_window.keep_header = value,
|
||||
"keep_input" => app.mini_window.keep_input = value,
|
||||
"keep_todo" => app.mini_window.keep_todo = value,
|
||||
"keep_sidebar" => app.mini_window.keep_sidebar = value,
|
||||
"keep_footer" => app.mini_window.keep_footer = value,
|
||||
_ => unreachable!("mini_window field matched above"),
|
||||
}
|
||||
// Persist first: a failed save leaves the live layout as it was
|
||||
// (C01-08).
|
||||
if persist
|
||||
&& let Err(err) = crate::config_persistence::persist_mini_window_bool_key(
|
||||
app.config_path.as_deref(),
|
||||
@@ -2748,6 +2752,14 @@ pub fn set_config_value(app: &mut App, key: &str, value: &str, persist: bool) ->
|
||||
{
|
||||
return CommandResult::error(format!("Failed to persist: {err}"));
|
||||
}
|
||||
match field {
|
||||
"keep_header" => app.mini_window.keep_header = value,
|
||||
"keep_input" => app.mini_window.keep_input = value,
|
||||
"keep_todo" => app.mini_window.keep_todo = value,
|
||||
"keep_sidebar" => app.mini_window.keep_sidebar = value,
|
||||
"keep_footer" => app.mini_window.keep_footer = value,
|
||||
_ => unreachable!("mini_window field matched above"),
|
||||
}
|
||||
app.needs_redraw = true;
|
||||
}
|
||||
"composer_density" | "composer" => {
|
||||
@@ -2798,15 +2810,17 @@ pub fn set_config_value(app: &mut App, key: &str, value: &str, persist: bool) ->
|
||||
app.needs_redraw = true;
|
||||
}
|
||||
"workspace_follow_symlinks" | "follow_symlinks" => {
|
||||
// Persist first: a failed save leaves the live value as it was
|
||||
// (C01-08).
|
||||
if persist && let Err(e) = persist_single_setting(&key, value) {
|
||||
return CommandResult::error(format!("Failed to save: {e}"));
|
||||
}
|
||||
app.workspace_follow_symlinks = settings.workspace_follow_symlinks;
|
||||
app.composer.mention_completion_cache = None;
|
||||
app.composer.mention_discovery.invalidate();
|
||||
app.needs_redraw = true;
|
||||
// Engine tools use EngineConfig which is fixed at startup
|
||||
return CommandResult::message(if persist {
|
||||
if let Err(e) = persist_single_setting(&key, value) {
|
||||
return CommandResult::error(format!("Failed to save: {e}"));
|
||||
}
|
||||
format!(
|
||||
"workspace_follow_symlinks = {} (saved; restart required for engine tools)",
|
||||
settings.workspace_follow_symlinks
|
||||
@@ -2904,7 +2918,18 @@ pub fn set_config_value(app: &mut App, key: &str, value: &str, persist: bool) ->
|
||||
|
||||
let message = if persist {
|
||||
if let Err(e) = persist_single_setting(&key, value) {
|
||||
return CommandResult::error(format!("Failed to save: {e}"));
|
||||
// C01-08: the projection above already changed live state (and
|
||||
// some arms validate only while projecting, so saving first could
|
||||
// persist a value that never applied). Report exactly that partial
|
||||
// effect, and keep the action so the engine matches the UI instead
|
||||
// of silently diverging from it.
|
||||
return CommandResult {
|
||||
message: Some(format!(
|
||||
"Error: {key} = {display_value} applies to this session only; saving failed: {e}"
|
||||
)),
|
||||
action,
|
||||
is_error: true,
|
||||
};
|
||||
}
|
||||
format!("{key} = {display_value} (saved)")
|
||||
} else {
|
||||
@@ -5384,6 +5409,35 @@ context_window = 262144
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn failed_save_leaves_live_config_state_untouched() {
|
||||
// C01-08: a `--save` that cannot write must not report failure over
|
||||
// live state that already moved.
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let _guard = EnvGuard::new(temp.path());
|
||||
let blocker = temp.path().join("not-a-directory");
|
||||
fs::write(&blocker, "a file where the config directory should be").unwrap();
|
||||
let mut app = create_test_app();
|
||||
app.config_path = Some(blocker.join("config.toml"));
|
||||
let mcp_before = app.mcp_config_path.clone();
|
||||
app.mcp_reload_required = false;
|
||||
let timeout_before = app.stream_chunk_timeout_secs;
|
||||
|
||||
let result = set_config_value(&mut app, "mcp_config_path", "/elsewhere/mcp.json", true);
|
||||
assert!(result.is_error, "{:?}", result.message);
|
||||
assert_eq!(app.mcp_config_path, mcp_before);
|
||||
assert!(!app.mcp_reload_required);
|
||||
|
||||
let next_timeout = if timeout_before == 120 { "121" } else { "120" };
|
||||
let result = set_config_value(&mut app, "stream_chunk_timeout_secs", next_timeout, true);
|
||||
assert!(result.is_error, "{:?}", result.message);
|
||||
assert!(
|
||||
result.action.is_none(),
|
||||
"no engine update for an unsaved value"
|
||||
);
|
||||
assert_eq!(app.stream_chunk_timeout_secs, timeout_before);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn config_command_stream_chunk_timeout_save_overrides_canonical_value() {
|
||||
let nanos = SystemTime::now()
|
||||
|
||||
@@ -127,9 +127,19 @@ pub async fn capture_and_transcribe(
|
||||
if !is_available() {
|
||||
return Err(tr(locale, MessageId::VoiceErrNoRecorder).to_string());
|
||||
}
|
||||
let api_key = config
|
||||
.active_route_api_key()
|
||||
.map_err(|_| tr(locale, MessageId::VoiceErrNoAuth).to_string())?;
|
||||
// C01-10: the credential check follows the ASR backend that will run,
|
||||
// not the chat provider. Local Whisper and Groq need no provider key, so
|
||||
// resolve it lazily — the same contract as `crate::voice::dictate_once`
|
||||
// — and only refuse before recording when provider ASR is the backend.
|
||||
let provider_key = || {
|
||||
config
|
||||
.active_route_api_key()
|
||||
.map_err(|_| tr(locale, MessageId::VoiceErrNoAuth).to_string())
|
||||
};
|
||||
let (asr_kind, _asr_model) = resolve_asr_choice(config);
|
||||
if asr_kind == "provider" {
|
||||
provider_key()?;
|
||||
}
|
||||
let base_url = config.active_route_base_url();
|
||||
let openrouter_vendor = config
|
||||
.openrouter_vendor()
|
||||
@@ -143,7 +153,6 @@ pub async fn capture_and_transcribe(
|
||||
// Streaming interim: poll every 700ms and show partial transcript like Grok Build's
|
||||
// VoiceEvent::Interim → VoiceState::Recording{interim}. We re-transcribe the
|
||||
// growing buffer (local-whisper is cheap; Groq is ~300ms; provider falls back).
|
||||
let (asr_kind, _asr_model) = resolve_asr_choice(config);
|
||||
let interim_enabled = true; // always show partials — feels alive like Spark
|
||||
|
||||
// Spawn recorder on blocking thread with a shared buffer for interim polling.
|
||||
@@ -241,13 +250,30 @@ pub async fn capture_and_transcribe(
|
||||
let text = match asr_kind.as_str() {
|
||||
"local-whisper" => match transcribe_local_whisper(&samples).await {
|
||||
Ok(v) => Ok(v),
|
||||
Err(_) => transcribe(&api_key, &base_url, &samples, openrouter_vendor.as_deref()).await,
|
||||
Err(_) => {
|
||||
transcribe(
|
||||
&provider_key()?,
|
||||
&base_url,
|
||||
&samples,
|
||||
openrouter_vendor.as_deref(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
},
|
||||
"groq" => match transcribe_groq(&samples).await {
|
||||
Ok(v) => Ok(v),
|
||||
Err(_) => transcribe(&api_key, &base_url, &samples, openrouter_vendor.as_deref()).await,
|
||||
Err(_) => {
|
||||
transcribe(
|
||||
&provider_key()?,
|
||||
&base_url,
|
||||
&samples,
|
||||
openrouter_vendor.as_deref(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
},
|
||||
_ => {
|
||||
let api_key = provider_key()?;
|
||||
if app.voice_control_enabled {
|
||||
process_voice_control(
|
||||
&api_key,
|
||||
|
||||
@@ -350,8 +350,9 @@ pub enum ComputerMeterError {
|
||||
/// Interval ends before it starts.
|
||||
#[error("A Computer meter interval cannot end before it starts.")]
|
||||
IntervalReversed,
|
||||
/// Admission has expired before the interval started.
|
||||
#[error("The Computer admission expired before this interval.")]
|
||||
/// Admission expired before the interval ended. An interval must lie
|
||||
/// wholly inside the admission; the tail after expiry was never authorized.
|
||||
#[error("The Computer admission expired before this interval ended.")]
|
||||
AdmissionExpired,
|
||||
/// Meter or catalog revision is not the pinned v3 revision.
|
||||
#[error("{message}")]
|
||||
@@ -479,7 +480,10 @@ pub fn issue_computer_meter_receipt(
|
||||
return Err(ComputerMeterError::IntervalReversed);
|
||||
}
|
||||
let expires = parse_timestamp(&admission.expires_at, "expiresAt")?;
|
||||
if started >= expires {
|
||||
// `ended_at` is exclusive, so an interval ending exactly at expiry is
|
||||
// wholly authorized; one that runs past it is refused rather than clipped,
|
||||
// because a clipped receipt would no longer restate the provider interval.
|
||||
if started >= expires || ended > expires {
|
||||
return Err(ComputerMeterError::AdmissionExpired);
|
||||
}
|
||||
let accepted_seconds = elapsed_whole_seconds(started, ended);
|
||||
@@ -539,21 +543,21 @@ pub fn issue_computer_meter_receipt(
|
||||
Ok(receipt)
|
||||
}
|
||||
|
||||
/// Exact replay of an existing receipt. Identity and digest must match.
|
||||
/// Exact replay of an existing receipt. Every field must match (only the
|
||||
/// `replay_of` back-reference may differ), and the incoming digest and id must
|
||||
/// be the ones its own fields produce: a caller-supplied digest is never
|
||||
/// trusted to stand for fields it does not cover.
|
||||
pub fn assert_computer_meter_receipt_replay(
|
||||
existing: &ComputerMeterReceipt,
|
||||
incoming: &ComputerMeterReceipt,
|
||||
) -> Result<(), ComputerMeterError> {
|
||||
if existing.receipt_id == incoming.receipt_id
|
||||
&& existing.binding_digest == incoming.binding_digest
|
||||
&& existing.admission_id == incoming.admission_id
|
||||
&& existing.account_id == incoming.account_id
|
||||
&& existing.provider_event_ref == incoming.provider_event_ref
|
||||
&& existing.accepted_seconds == incoming.accepted_seconds
|
||||
&& existing.standard_equivalent_seconds == incoming.standard_equivalent_seconds
|
||||
&& existing.profile_id == incoming.profile_id
|
||||
&& existing.multiplier == incoming.multiplier
|
||||
{
|
||||
let self_consistent = receipt_binding_digest(incoming) == incoming.binding_digest
|
||||
&& receipt_id_for(incoming) == incoming.receipt_id;
|
||||
let same_terms = ComputerMeterReceipt {
|
||||
replay_of: existing.replay_of.clone(),
|
||||
..incoming.clone()
|
||||
} == *existing;
|
||||
if self_consistent && same_terms {
|
||||
return Ok(());
|
||||
}
|
||||
Err(ComputerMeterError::ReplayConflict)
|
||||
|
||||
@@ -190,6 +190,61 @@ fn exact_replay_is_idempotent_and_a_mutated_binding_conflicts() {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replay_rejects_fields_the_supplied_digest_does_not_vouch_for() {
|
||||
let case = fixtures()
|
||||
.into_iter()
|
||||
.find(|case| case.id == "accepted-standard-8")
|
||||
.unwrap();
|
||||
let first = issue_case(&case).unwrap();
|
||||
|
||||
// Same id and digest, different interval: the digest no longer matches
|
||||
// the fields it claims to bind.
|
||||
let mut shifted = first.clone();
|
||||
shifted.started_at = "2026-08-31T13:00:00.000Z".to_string();
|
||||
shifted.ended_at = "2026-08-31T13:10:00.000Z".to_string();
|
||||
assert_eq!(
|
||||
assert_computer_meter_receipt_replay(&first, &shifted)
|
||||
.unwrap_err()
|
||||
.code(),
|
||||
"computer_meter_receipt_replay_conflict"
|
||||
);
|
||||
|
||||
// A field outside the digest must still match exactly.
|
||||
let mut restated = first.clone();
|
||||
restated.provider_allocation.accepted = false;
|
||||
assert_eq!(
|
||||
assert_computer_meter_receipt_replay(&first, &restated)
|
||||
.unwrap_err()
|
||||
.code(),
|
||||
"computer_meter_receipt_replay_conflict"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_interval_that_runs_past_admission_expiry_is_refused() {
|
||||
let case = fixtures()
|
||||
.into_iter()
|
||||
.find(|case| case.id == "accepted-standard-8")
|
||||
.unwrap();
|
||||
let mut admission_request = case.admission.clone().unwrap();
|
||||
let observation = case.observation.clone().unwrap();
|
||||
// Starts before expiry, ends after it.
|
||||
admission_request.expires_at = "2026-08-31T12:05:00.000Z".to_string();
|
||||
let admission = bind_computer_admission(admission_request.clone()).unwrap();
|
||||
assert_eq!(
|
||||
issue_computer_meter_receipt(&admission, observation.clone())
|
||||
.unwrap_err()
|
||||
.code(),
|
||||
"computer_admission_expired"
|
||||
);
|
||||
|
||||
// Ending exactly at expiry is inside the admission (end is exclusive).
|
||||
admission_request.expires_at = observation.ended_at.clone();
|
||||
let admission = bind_computer_admission(admission_request).unwrap();
|
||||
assert!(issue_computer_meter_receipt(&admission, observation).is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn admission_binds_profile_multiplier_account_and_expiry_before_dispatch() {
|
||||
let admission = bind_computer_admission(ComputerAdmissionRequest {
|
||||
|
||||
+115
-39
@@ -912,11 +912,17 @@ pub struct Engine {
|
||||
/// the channel closing (task exited with the pool) disarms it and the
|
||||
/// next pool ensure respawns.
|
||||
mcp_supervisor_rx: Option<mpsc::Receiver<McpSupervisorUpdate>>,
|
||||
/// Abort the supervisor's reconnect handshakes when its pool is retired.
|
||||
mcp_supervisor_task: Option<tokio::task::AbortHandle>,
|
||||
mcp_boot_done: Option<tokio::sync::watch::Receiver<bool>>,
|
||||
/// Generation owned by the currently installed boot receiver. Terminal
|
||||
/// cleanup is conditional on this exact value so an older pass can never
|
||||
/// clear a newer receiver.
|
||||
mcp_boot_generation: Option<u64>,
|
||||
/// Abort handle for the boot pass that owns `mcp_boot_generation`. A
|
||||
/// session or workspace boundary drops the pool that pass is dialing
|
||||
/// into, so it aborts the pass instead of letting it run on (C02-08).
|
||||
mcp_boot_task: Option<tokio::task::AbortHandle>,
|
||||
/// Monotonic generation for engine-authored MCP session snapshots. Boot
|
||||
/// task updates retain their spawn generation so later passes can reject
|
||||
/// only genuinely stale work.
|
||||
@@ -2060,8 +2066,10 @@ impl Engine {
|
||||
mcp_boot_in_flight: false,
|
||||
mcp_boot_rx: None,
|
||||
mcp_supervisor_rx: None,
|
||||
mcp_supervisor_task: None,
|
||||
mcp_boot_done: None,
|
||||
mcp_boot_generation: None,
|
||||
mcp_boot_task: None,
|
||||
mcp_event_generation: 0,
|
||||
plugin_registry,
|
||||
extension_host,
|
||||
@@ -2789,7 +2797,12 @@ impl Engine {
|
||||
// Runtime-added MCP servers are conversation capabilities even when
|
||||
// both conversations use the same workspace. Configured servers can
|
||||
// reconnect lazily after the new session is installed.
|
||||
self.mcp_pool = None;
|
||||
self.drop_mcp_pool();
|
||||
// C02-17: cumulative usage belongs to the conversation that spent
|
||||
// it. The sync carries no prior total for the installed session, so
|
||||
// it starts from zero instead of inheriting the previous session's
|
||||
// tokens into its snapshot and saved `total_tokens`.
|
||||
self.session.total_usage = Default::default();
|
||||
self.session.id = next_session_id;
|
||||
Some(previous_session_id)
|
||||
}
|
||||
@@ -3645,7 +3658,7 @@ impl Engine {
|
||||
self.config.plugin_registry = Some(Arc::clone(&self.plugin_registry));
|
||||
// A pool may contain plugin servers and authority
|
||||
// receipts from the previous workspace snapshot.
|
||||
self.mcp_pool = None;
|
||||
self.drop_mcp_pool();
|
||||
if let Some(attachment) = &self.extension_host {
|
||||
attachment.set_plugins(Arc::clone(&self.plugin_registry));
|
||||
attachment.sync_in_background();
|
||||
@@ -3841,39 +3854,65 @@ impl Engine {
|
||||
continue;
|
||||
}
|
||||
};
|
||||
// C02-02: stage the cut. The removed exchange is kept
|
||||
// until the replacement turn actually starts; a send
|
||||
// that never starts puts it back (below).
|
||||
let removed_exchange = self
|
||||
.session
|
||||
.messages
|
||||
.get(idx..)
|
||||
.map(<[_]>::to_vec)
|
||||
.unwrap_or_default();
|
||||
self.session.messages.truncate_to(idx);
|
||||
self.session.bump_messages_revision();
|
||||
// Now dispatch the new message as a normal send,
|
||||
// reusing the engine's stored mode/model config.
|
||||
let mode = self.current_mode;
|
||||
self.handle_send_message(TurnSpec {
|
||||
content: new_message.clone(),
|
||||
mode,
|
||||
route: Box::new(route),
|
||||
compaction: Box::new(self.config.compaction.clone()),
|
||||
initial_routed_usage: Box::new(
|
||||
crate::cost_status::RuntimeUsageBatch::default(),
|
||||
),
|
||||
goal_objective: self.config.goal_objective.clone(),
|
||||
goal_token_budget: self.config.goal_token_budget,
|
||||
goal_status: self.config.goal_status,
|
||||
reasoning_effort: self.session.reasoning_effort.clone(),
|
||||
reasoning_effort_auto: self.session.reasoning_effort_auto,
|
||||
auto_model: self.session.auto_model,
|
||||
allow_shell: self.session.allow_shell,
|
||||
trust_mode: self.session.trust_mode,
|
||||
auto_approve: self.session.auto_approve,
|
||||
approval_mode: self.session.approval_mode,
|
||||
translation_enabled: self.config.translation_enabled,
|
||||
allowed_tools: self.config.allowed_tools.clone(),
|
||||
dynamic_tools: Vec::new(),
|
||||
hook_executor: self.config.hook_executor.clone(),
|
||||
verbosity: self.config.verbosity.clone(),
|
||||
provenance: UserInputProvenance::ExternalUser,
|
||||
images: Vec::new(),
|
||||
max_output_tokens: None,
|
||||
})
|
||||
.await;
|
||||
let outcome = self
|
||||
.handle_send_message(TurnSpec {
|
||||
content: new_message.clone(),
|
||||
mode,
|
||||
route: Box::new(route),
|
||||
compaction: Box::new(self.config.compaction.clone()),
|
||||
initial_routed_usage: Box::new(
|
||||
crate::cost_status::RuntimeUsageBatch::default(),
|
||||
),
|
||||
goal_objective: self.config.goal_objective.clone(),
|
||||
goal_token_budget: self.config.goal_token_budget,
|
||||
goal_status: self.config.goal_status,
|
||||
reasoning_effort: self.session.reasoning_effort.clone(),
|
||||
reasoning_effort_auto: self.session.reasoning_effort_auto,
|
||||
auto_model: self.session.auto_model,
|
||||
allow_shell: self.session.allow_shell,
|
||||
trust_mode: self.session.trust_mode,
|
||||
auto_approve: self.session.auto_approve,
|
||||
approval_mode: self.session.approval_mode,
|
||||
translation_enabled: self.config.translation_enabled,
|
||||
allowed_tools: self.config.allowed_tools.clone(),
|
||||
dynamic_tools: Vec::new(),
|
||||
hook_executor: self.config.hook_executor.clone(),
|
||||
verbosity: self.config.verbosity.clone(),
|
||||
provenance: UserInputProvenance::ExternalUser,
|
||||
images: Vec::new(),
|
||||
max_output_tokens: None,
|
||||
})
|
||||
.await;
|
||||
if matches!(outcome, SendMessageOutcome::NotStarted { .. }) {
|
||||
// Anything the failed send appended after the cut
|
||||
// (a drained shell-completion notice) stays, after
|
||||
// the restored exchange.
|
||||
let appended = self
|
||||
.session
|
||||
.messages
|
||||
.get(idx..)
|
||||
.map(<[_]>::to_vec)
|
||||
.unwrap_or_default();
|
||||
self.session.messages.truncate_to(idx);
|
||||
self.session.messages.push_batch(removed_exchange);
|
||||
self.session.messages.push_batch(appended);
|
||||
self.session.bump_messages_revision();
|
||||
self.emit_session_updated().await;
|
||||
}
|
||||
}
|
||||
Op::SetAdvisorEnabled { enabled } => {
|
||||
self.config.advisor_config.enabled = enabled;
|
||||
@@ -6711,29 +6750,42 @@ impl Engine {
|
||||
self.ensure_mcp_supervisor();
|
||||
return Ok(pool);
|
||||
}
|
||||
// C02-18: misconfiguration fails loud. A missing file is an ordinary
|
||||
// empty config (`load_config` returns the default); an unreadable or
|
||||
// malformed one still yields an empty, source-aware pool so a fixed
|
||||
// file and `/mcp reload` recover in-process — but the person is told
|
||||
// that the configured servers are gone, not left to find no tools.
|
||||
let mut load_failure = None;
|
||||
let mut pool = McpPool::from_config_path_with_workspace_and_plugins(
|
||||
&self.session.mcp_config_path,
|
||||
&self.session.workspace,
|
||||
Arc::clone(&self.plugin_registry),
|
||||
)
|
||||
.unwrap_or_else(|e| {
|
||||
tracing::debug!(
|
||||
"MCP config unavailable: {}",
|
||||
crate::mcp::format_mcp_error_for_display(&e)
|
||||
);
|
||||
let reason = crate::mcp::format_mcp_error_for_display(&e);
|
||||
tracing::warn!("MCP config unavailable: {reason}");
|
||||
load_failure = Some(reason);
|
||||
McpPool::empty_with_workspace_config_sources(
|
||||
&self.session.mcp_config_path,
|
||||
&self.session.workspace,
|
||||
Arc::clone(&self.plugin_registry),
|
||||
)
|
||||
.unwrap_or_else(|fallback_error| {
|
||||
tracing::debug!(
|
||||
tracing::warn!(
|
||||
"MCP reload source setup failed: {}",
|
||||
crate::mcp::format_mcp_error_for_display(&fallback_error)
|
||||
);
|
||||
McpPool::new(McpConfig::default())
|
||||
})
|
||||
});
|
||||
if let Some(reason) = load_failure {
|
||||
let _ = self
|
||||
.tx_event
|
||||
.send(Event::status(format!(
|
||||
"MCP config could not be loaded, so none of its servers are available: {reason}. Fix it, then run /mcp reload."
|
||||
)))
|
||||
.await;
|
||||
}
|
||||
pool = pool.with_disallowed_tools(self.config.disallowed_tools.clone().unwrap_or_default());
|
||||
if let Some(decider) = self.config.network_policy.as_ref() {
|
||||
pool = pool.with_network_policy(decider.clone());
|
||||
@@ -6752,8 +6804,8 @@ impl Engine {
|
||||
}
|
||||
|
||||
/// Start the connection supervisor once per pool. The task holds only a
|
||||
/// Weak: pool replacement lets the old task exit, its channel closes, the
|
||||
/// run loop disarms, and the next ensure respawns against the new pool.
|
||||
/// Weak between sweeps, but an in-flight handshake holds the pool. Keep
|
||||
/// its abort handle so a boundary stops that work and queued diagnoses.
|
||||
fn ensure_mcp_supervisor(&mut self) {
|
||||
if self.mcp_supervisor_rx.is_some() {
|
||||
return;
|
||||
@@ -6764,11 +6816,12 @@ impl Engine {
|
||||
let (tx, rx) = mpsc::channel(16);
|
||||
self.mcp_supervisor_rx = Some(rx);
|
||||
let weak = Arc::downgrade(pool);
|
||||
spawn_supervised(
|
||||
let task = spawn_supervised(
|
||||
"mcp-supervisor",
|
||||
std::panic::Location::caller(),
|
||||
McpPool::supervise_pool(weak, tx),
|
||||
);
|
||||
self.mcp_supervisor_task = Some(task.abort_handle());
|
||||
}
|
||||
|
||||
/// Apply one supervisor sweep: deaths and failures refresh the engine's
|
||||
@@ -6886,12 +6939,34 @@ impl Engine {
|
||||
return false;
|
||||
}
|
||||
self.mcp_boot_generation = None;
|
||||
self.mcp_boot_task = None;
|
||||
self.mcp_boot_in_flight = false;
|
||||
self.mcp_boot_rx = None;
|
||||
self.mcp_boot_done = None;
|
||||
true
|
||||
}
|
||||
|
||||
/// Drop the engine-owned MCP pool at a session or workspace boundary.
|
||||
/// The in-flight boot pass dials into the dropped pool: abort it and
|
||||
/// clear its generation, receiver and diagnoses, so the next pool starts
|
||||
/// clean instead of waiting on — or adopting progress and connection
|
||||
/// errors from — the previous conversation's pass (C02-08).
|
||||
fn drop_mcp_pool(&mut self) {
|
||||
self.mcp_pool = None;
|
||||
if let Some(task) = self.mcp_boot_task.take() {
|
||||
task.abort();
|
||||
}
|
||||
if let Some(task) = self.mcp_supervisor_task.take() {
|
||||
task.abort();
|
||||
}
|
||||
self.mcp_supervisor_rx = None;
|
||||
self.mcp_boot_generation = None;
|
||||
self.mcp_boot_in_flight = false;
|
||||
self.mcp_boot_rx = None;
|
||||
self.mcp_boot_done = None;
|
||||
self.mcp_connection_errors.clear();
|
||||
}
|
||||
|
||||
async fn emit_mcp_session_boot(&self, generation: u64, finished: bool) {
|
||||
let Ok(snapshot) = self.mcp_session_snapshot().await else {
|
||||
return;
|
||||
@@ -7337,7 +7412,7 @@ impl Engine {
|
||||
self.emit_mcp_session_boot(generation, false).await;
|
||||
|
||||
let pool_for_task = Arc::clone(&pool);
|
||||
spawn_supervised(
|
||||
let boot_task = spawn_supervised(
|
||||
"mcp-session-boot",
|
||||
std::panic::Location::caller(),
|
||||
async move {
|
||||
@@ -7419,6 +7494,7 @@ impl Engine {
|
||||
let _ = done_tx.send(true);
|
||||
},
|
||||
);
|
||||
self.mcp_boot_task = Some(boot_task.abort_handle());
|
||||
|
||||
Ok(generation)
|
||||
}
|
||||
|
||||
@@ -287,14 +287,36 @@ impl Engine {
|
||||
tool_id: &str,
|
||||
request: UserInputRequest,
|
||||
) -> Result<UserInputResponse, ToolError> {
|
||||
let _ = self
|
||||
// C02-19: a question that never reached a host has nobody to answer
|
||||
// it. Fail now instead of waiting out the timeout — which by default
|
||||
// is no timeout at all.
|
||||
if self
|
||||
.tx_event
|
||||
.send(Event::UserInputRequired {
|
||||
id: tool_id.to_string(),
|
||||
request,
|
||||
})
|
||||
.await;
|
||||
.await
|
||||
.is_err()
|
||||
{
|
||||
return Err(ToolError::execution_failed(
|
||||
"User input request could not reach its host, so nobody was asked. \
|
||||
Continue without the answer or ask in your reply instead."
|
||||
.to_string(),
|
||||
));
|
||||
}
|
||||
// R1, as for tool approval: the per-turn wall-clock budget bounds the
|
||||
// agent's own time, not how long a person takes to answer.
|
||||
self.turn_wall_clock.begin_human_wait();
|
||||
let response = self.await_user_input_decision(tool_id).await;
|
||||
self.turn_wall_clock.end_human_wait();
|
||||
response
|
||||
}
|
||||
|
||||
async fn await_user_input_decision(
|
||||
&mut self,
|
||||
tool_id: &str,
|
||||
) -> Result<UserInputResponse, ToolError> {
|
||||
// #6003: `[tools] user_input_timeout_seconds`. Absent, or an explicit
|
||||
// 0, waits until the person answers or cancels. A positive value is
|
||||
// one absolute deadline for the whole wait: `select!` drops the
|
||||
|
||||
@@ -38,7 +38,7 @@ impl ToolUseState {
|
||||
}
|
||||
}
|
||||
|
||||
/// Maximum total bytes of text/thinking content before aborting the stream.
|
||||
/// Maximum total bytes of text, reasoning and tool-argument content before aborting the stream.
|
||||
pub(super) const STREAM_MAX_CONTENT_BYTES: usize = 10 * 1024 * 1024; // 10 MB
|
||||
/// Sanity backstop for total stream wall-clock duration. **Not** a routine
|
||||
/// kill switch — the stream chunk idle timeout is the primary stall
|
||||
|
||||
@@ -25109,7 +25109,7 @@ async fn reasoning_only_length_stop_fails_without_retry() {
|
||||
/// each carry exactly one extra message over the same baseline.
|
||||
#[tokio::test]
|
||||
async fn the_reasoning_only_nudge_rides_one_request_and_never_joins_the_session() {
|
||||
let (model, _events) = run_reasoning_only_turn_with_reprompts(usize::MAX, "stop", 3).await;
|
||||
let (model, events) = run_reasoning_only_turn_with_reprompts(usize::MAX, "stop", 3).await;
|
||||
|
||||
let requests = model.requests.lock().expect("captured requests").clone();
|
||||
assert_eq!(requests.len(), 4, "one initial request plus three retries");
|
||||
@@ -25144,6 +25144,34 @@ async fn the_reasoning_only_nudge_rides_one_request_and_never_joins_the_session(
|
||||
carries_nudge(&requests[2]),
|
||||
"nudge present once retrying again"
|
||||
);
|
||||
|
||||
// C02-04: model-visible means logged. Every request the nudge rides
|
||||
// leaves a durable (internal) receipt carrying its exact text.
|
||||
let receipts: Vec<&str> = events
|
||||
.iter()
|
||||
.filter_map(|event| match event {
|
||||
Event::Status { message }
|
||||
if message.starts_with(super::turn_loop::REQUEST_NUDGE_RECEIPT_PREFIX) =>
|
||||
{
|
||||
Some(message.as_str())
|
||||
}
|
||||
_ => None,
|
||||
})
|
||||
.collect();
|
||||
assert_eq!(
|
||||
receipts.len(),
|
||||
requests
|
||||
.iter()
|
||||
.filter(|request| carries_nudge(request))
|
||||
.count(),
|
||||
"one receipt per nudged request"
|
||||
);
|
||||
assert!(receipts.iter().all(|receipt| receipt.ends_with(nudge)));
|
||||
assert_eq!(
|
||||
crate::core::events::status_visibility(receipts[0]),
|
||||
crate::core::events::StatusVisibility::Internal,
|
||||
"durable clients keep the receipt, collapsed"
|
||||
);
|
||||
}
|
||||
|
||||
/// A model that only ever returns reasoning is bounded: it retries up to the
|
||||
@@ -27443,6 +27471,8 @@ fn engine_adopts_host_owned_session_id_from_config() {
|
||||
);
|
||||
}
|
||||
|
||||
mod admission_gates;
|
||||
mod runtime_state;
|
||||
mod sse_turn_recovery;
|
||||
mod tool_cancellation;
|
||||
|
||||
|
||||
@@ -0,0 +1,382 @@
|
||||
//! Admission gates between a provider response and tool execution: a failed
|
||||
//! stream (C02-05), the step-budget final report (C02-10), the stream content
|
||||
//! cap over tool arguments (C02-13), and bounded tool errors (C02-12). Every
|
||||
//! fixture counts handler entries; none runs a shell or reaches a provider.
|
||||
|
||||
use super::*;
|
||||
use crate::llm_client::mock::{MockLlmClient, canned};
|
||||
use crate::tools::spec::{ApprovalRequirement, ToolCapability, ToolSpec};
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
|
||||
const WRITE_TOOL: &str = "fixture_write";
|
||||
|
||||
/// A mutating tool that needs no approval, so a call that is admitted runs.
|
||||
struct CountingWriteTool(Arc<AtomicUsize>);
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl ToolSpec for CountingWriteTool {
|
||||
fn name(&self) -> &str {
|
||||
WRITE_TOOL
|
||||
}
|
||||
fn description(&self) -> &str {
|
||||
"Count every execution of a mutating fixture."
|
||||
}
|
||||
fn input_schema(&self) -> Value {
|
||||
json!({"type": "object"})
|
||||
}
|
||||
fn capabilities(&self) -> Vec<ToolCapability> {
|
||||
vec![ToolCapability::WritesFiles]
|
||||
}
|
||||
fn approval_requirement(&self) -> ApprovalRequirement {
|
||||
ApprovalRequirement::Auto
|
||||
}
|
||||
async fn execute(&self, _: Value, _: &ToolContext) -> Result<ToolResult, ToolError> {
|
||||
self.0.fetch_add(1, Ordering::SeqCst);
|
||||
Ok(ToolResult::success("wrote"))
|
||||
}
|
||||
}
|
||||
|
||||
fn write_surface(
|
||||
engine: &Engine,
|
||||
workspace: &Path,
|
||||
executions: Arc<AtomicUsize>,
|
||||
) -> ToolSurfacePolicy {
|
||||
let mut registry = crate::tools::ToolRegistry::new(ToolContext::new(workspace));
|
||||
registry.register(Arc::new(CountingWriteTool(executions)));
|
||||
let tools = Some(registry.to_api_tools_with_cache(true));
|
||||
test_tool_surface(engine, registry, tools, AppMode::Agent)
|
||||
}
|
||||
|
||||
/// Streams a scripted response per call, `Err` items included, which the
|
||||
/// queue-driven mock cannot express.
|
||||
struct ScriptedStreamClient {
|
||||
calls: AtomicUsize,
|
||||
script: fn(usize) -> Vec<anyhow::Result<StreamEvent>>,
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl crate::core::model_client::ModelClient for ScriptedStreamClient {
|
||||
fn provider_name(&self) -> &str {
|
||||
"scripted"
|
||||
}
|
||||
fn model(&self) -> &str {
|
||||
"scripted-model"
|
||||
}
|
||||
async fn create_message(
|
||||
&self,
|
||||
_request: codewhale_models::MessageRequest,
|
||||
) -> anyhow::Result<codewhale_models::MessageResponse> {
|
||||
anyhow::bail!("scripted fixture only streams")
|
||||
}
|
||||
async fn create_message_stream(
|
||||
&self,
|
||||
_request: codewhale_models::MessageRequest,
|
||||
) -> anyhow::Result<crate::llm_client::StreamEventBox> {
|
||||
let call = self.calls.fetch_add(1, Ordering::SeqCst);
|
||||
Ok(Box::pin(futures_util::stream::iter((self.script)(call))))
|
||||
}
|
||||
async fn health_check(&self) -> anyhow::Result<bool> {
|
||||
Ok(true)
|
||||
}
|
||||
}
|
||||
|
||||
async fn drain_events(handle: &EngineHandle) -> Vec<Event> {
|
||||
let mut rx = handle.rx_event.write().await;
|
||||
std::iter::from_fn(|| rx.try_recv().ok()).collect()
|
||||
}
|
||||
|
||||
fn not_started(result: &Result<ToolResult, ToolError>) -> bool {
|
||||
result.as_ref().is_ok_and(|output| {
|
||||
!output.success
|
||||
&& output
|
||||
.metadata
|
||||
.as_ref()
|
||||
.is_some_and(|metadata| metadata["side_effect_status"] == "not_started")
|
||||
})
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn terminal_stream_error_after_a_complete_tool_call_never_reaches_the_handler() {
|
||||
// C02-05: a complete mutating call followed by a provider error frame.
|
||||
let workspace = tempdir().unwrap();
|
||||
let mock = Arc::new(MockLlmClient::new(vec![
|
||||
vec![
|
||||
canned::message_start("tool-then-error"),
|
||||
canned::tool_use_block_start(0, "call-write", WRITE_TOOL),
|
||||
canned::tool_input_delta(0, r#"{"path":"target.txt"}"#),
|
||||
canned::block_stop(0),
|
||||
StreamEvent::Error {
|
||||
error: json!({ "message": "Model not exist." }),
|
||||
},
|
||||
],
|
||||
canned::simple_text_turn("this second request must never be issued"),
|
||||
]));
|
||||
let (mut engine, handle) = Engine::new_with_model_client(
|
||||
deterministic_engine_config(workspace.path()),
|
||||
&Config::default(),
|
||||
mock.clone(),
|
||||
);
|
||||
engine.session.auto_approve = true;
|
||||
let executions = Arc::new(AtomicUsize::new(0));
|
||||
let surface = write_surface(&engine, workspace.path(), executions.clone());
|
||||
let mut turn = crate::core::turn::TurnContext::new(4);
|
||||
|
||||
let (status, error) = engine.run_turn(&mut turn, surface, None, None).await;
|
||||
|
||||
assert_eq!(
|
||||
executions.load(Ordering::SeqCst),
|
||||
0,
|
||||
"the handler must never run"
|
||||
);
|
||||
assert_eq!(status, TurnOutcomeStatus::Failed);
|
||||
assert!(
|
||||
error
|
||||
.as_deref()
|
||||
.is_some_and(|e| e.contains("Model not exist.")),
|
||||
"{error:?}"
|
||||
);
|
||||
assert_eq!(
|
||||
mock.call_count(),
|
||||
1,
|
||||
"a failed response authorizes no further request"
|
||||
);
|
||||
assert_eq!(
|
||||
turn.stop_diagnostics.last_response_tool_calls_suppressed,
|
||||
Some(1)
|
||||
);
|
||||
let completions: Vec<_> = drain_events(&handle)
|
||||
.await
|
||||
.into_iter()
|
||||
.filter_map(|event| match event {
|
||||
Event::ToolCallComplete { result, .. } => Some(result),
|
||||
_ => None,
|
||||
})
|
||||
.collect();
|
||||
assert_eq!(completions.len(), 1, "the call is settled exactly once");
|
||||
assert!(not_started(&completions[0]), "{:?}", completions[0]);
|
||||
assert!(
|
||||
!engine
|
||||
.session
|
||||
.messages
|
||||
.iter()
|
||||
.flat_map(|message| &message.content)
|
||||
.any(|block| matches!(
|
||||
block,
|
||||
ContentBlock::ToolUse { .. } | ContentBlock::ToolResult { .. }
|
||||
)),
|
||||
"no unpaired or fabricated tool history"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn exhausted_stream_resume_never_executes_the_failed_batch() {
|
||||
// C02-05, retry path: every attempt streams a complete call and then
|
||||
// drops. Each resume discards its batch; the last, unretried attempt is
|
||||
// refused at admission instead of executing.
|
||||
fn tool_then_drop(_call: usize) -> Vec<anyhow::Result<StreamEvent>> {
|
||||
vec![
|
||||
Ok(canned::message_start("tool-then-drop")),
|
||||
Ok(canned::tool_use_block_start(0, "call-write", WRITE_TOOL)),
|
||||
Ok(canned::tool_input_delta(0, "{}")),
|
||||
Ok(canned::block_stop(0)),
|
||||
Err(anyhow::anyhow!(
|
||||
"Stream read error: error decoding response body"
|
||||
)),
|
||||
]
|
||||
}
|
||||
let workspace = tempdir().unwrap();
|
||||
let client = Arc::new(ScriptedStreamClient {
|
||||
calls: AtomicUsize::new(0),
|
||||
script: tool_then_drop,
|
||||
});
|
||||
let (mut engine, _handle) = Engine::new_with_model_client(
|
||||
EngineConfig {
|
||||
terminal_chrome_enabled: false,
|
||||
..deterministic_engine_config(workspace.path())
|
||||
},
|
||||
&Config::default(),
|
||||
client.clone(),
|
||||
);
|
||||
engine.session.auto_approve = true;
|
||||
let resumes = engine.config.stream_retry_limits.max_resumes as usize;
|
||||
let executions = Arc::new(AtomicUsize::new(0));
|
||||
let surface = write_surface(&engine, workspace.path(), executions.clone());
|
||||
let mut turn = crate::core::turn::TurnContext::new(4);
|
||||
|
||||
let (status, error) = engine.run_turn(&mut turn, surface, None, None).await;
|
||||
|
||||
assert_eq!(
|
||||
executions.load(Ordering::SeqCst),
|
||||
0,
|
||||
"no attempt's batch may run"
|
||||
);
|
||||
assert_eq!(client.calls.load(Ordering::SeqCst), 1 + resumes);
|
||||
assert_eq!(status, TurnOutcomeStatus::Failed);
|
||||
assert!(
|
||||
error
|
||||
.as_deref()
|
||||
.is_some_and(|e| e.contains("error decoding response body")),
|
||||
"{error:?}"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn tool_argument_bytes_count_toward_the_stream_content_cap() {
|
||||
// C02-13: a response made only of tool-argument JSON must still hit the
|
||||
// per-step content cap, and the capped call must not run.
|
||||
let workspace = tempdir().unwrap();
|
||||
let huge_args = format!(r#"{{"blob":"{}"}}"#, "x".repeat(4096));
|
||||
let mock = Arc::new(MockLlmClient::new(vec![
|
||||
canned::tool_call_turn("call-big", WRITE_TOOL, &huge_args),
|
||||
canned::simple_text_turn("this second request must never be issued"),
|
||||
]));
|
||||
let (mut engine, _handle) = Engine::new_with_model_client(
|
||||
EngineConfig {
|
||||
stream_max_content_bytes: 64,
|
||||
..deterministic_engine_config(workspace.path())
|
||||
},
|
||||
&Config::default(),
|
||||
mock.clone(),
|
||||
);
|
||||
engine.session.auto_approve = true;
|
||||
let executions = Arc::new(AtomicUsize::new(0));
|
||||
let surface = write_surface(&engine, workspace.path(), executions.clone());
|
||||
let mut turn = crate::core::turn::TurnContext::new(4);
|
||||
|
||||
let (status, _error) = engine.run_turn(&mut turn, surface, None, None).await;
|
||||
|
||||
assert_eq!(executions.load(Ordering::SeqCst), 0);
|
||||
assert_eq!(status, TurnOutcomeStatus::Failed);
|
||||
assert_eq!(mock.call_count(), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn step_budget_final_report_is_report_only() {
|
||||
// C02-10: the one response granted after the step budget is spent asks
|
||||
// for no tools, and a call it returns anyway is refused, not executed.
|
||||
let workspace = tempdir().unwrap();
|
||||
let mock = Arc::new(MockLlmClient::new(vec![
|
||||
canned::tool_call_turn("call-step-1", WRITE_TOOL, "{}"),
|
||||
canned::tool_call_turn("call-final-report", WRITE_TOOL, "{}"),
|
||||
]));
|
||||
let (mut engine, handle) = Engine::new_with_model_client(
|
||||
deterministic_engine_config(workspace.path()),
|
||||
&Config::default(),
|
||||
mock.clone(),
|
||||
);
|
||||
engine.session.auto_approve = true;
|
||||
let executions = Arc::new(AtomicUsize::new(0));
|
||||
let surface = write_surface(&engine, workspace.path(), executions.clone());
|
||||
let mut turn = crate::core::turn::TurnContext::new(1);
|
||||
|
||||
let (status, error) = engine.run_turn(&mut turn, surface, None, None).await;
|
||||
|
||||
assert_eq!(
|
||||
executions.load(Ordering::SeqCst),
|
||||
1,
|
||||
"only the in-budget step runs"
|
||||
);
|
||||
assert_eq!(status, TurnOutcomeStatus::Failed);
|
||||
assert!(
|
||||
error
|
||||
.as_deref()
|
||||
.is_some_and(|e| e.contains("Maximum model steps")),
|
||||
"{error:?}"
|
||||
);
|
||||
let requests = mock.captured_requests();
|
||||
assert_eq!(requests.len(), 2);
|
||||
assert_ne!(requests[0].tool_choice, Some(json!("none")));
|
||||
assert_eq!(requests[1].tool_choice, Some(json!("none")));
|
||||
let refused = drain_events(&handle).await.into_iter().any(|event| {
|
||||
matches!(
|
||||
event,
|
||||
Event::ToolCallComplete {
|
||||
model_call: Some(ref call),
|
||||
result: Err(ToolError::PermissionDenied { ref message }),
|
||||
..
|
||||
} if call.provider_id == "call-final-report" && message.contains("final report")
|
||||
)
|
||||
});
|
||||
assert!(refused, "the final-report call is refused with the reason");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn oversized_tool_errors_are_bounded_before_fanout() {
|
||||
// C02-12: an error is published to the event stream and the session like
|
||||
// a result, so an oversized one gets the same bounded projection.
|
||||
struct HugeErrorTool {
|
||||
as_result: bool,
|
||||
}
|
||||
#[async_trait::async_trait]
|
||||
impl ToolSpec for HugeErrorTool {
|
||||
fn name(&self) -> &str {
|
||||
"fixture_huge_error"
|
||||
}
|
||||
fn description(&self) -> &str {
|
||||
"Fail with an oversized message."
|
||||
}
|
||||
fn input_schema(&self) -> Value {
|
||||
json!({"type": "object"})
|
||||
}
|
||||
fn capabilities(&self) -> Vec<ToolCapability> {
|
||||
vec![ToolCapability::ReadOnly]
|
||||
}
|
||||
async fn execute(&self, _: Value, _: &ToolContext) -> Result<ToolResult, ToolError> {
|
||||
let message = format!("FIRST{}LAST", "e".repeat(300_000));
|
||||
if self.as_result {
|
||||
Ok(ToolResult::error(message))
|
||||
} else {
|
||||
Err(ToolError::execution_failed(message))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
with_artifact_home(|home| {
|
||||
tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.build()
|
||||
.unwrap()
|
||||
.block_on(async {
|
||||
for as_result in [false, true] {
|
||||
let mock = Arc::new(MockLlmClient::new(vec![
|
||||
canned::tool_call_turn("call-huge", "fixture_huge_error", "{}"),
|
||||
canned::simple_text_turn("done"),
|
||||
]));
|
||||
let (mut engine, handle) = Engine::new_with_model_client(
|
||||
deterministic_engine_config(home),
|
||||
&Config::default(),
|
||||
mock.clone(),
|
||||
);
|
||||
let mut registry = crate::tools::ToolRegistry::new(ToolContext::new(home));
|
||||
registry.register(Arc::new(HugeErrorTool { as_result }));
|
||||
let tools = Some(registry.to_api_tools_with_cache(true));
|
||||
let surface = test_tool_surface(&engine, registry, tools, AppMode::Agent);
|
||||
let mut turn = crate::core::turn::TurnContext::new(4);
|
||||
let (status, error) = engine.run_turn(&mut turn, surface, None, None).await;
|
||||
assert_eq!(status, TurnOutcomeStatus::Completed, "{error:?}");
|
||||
let published = drain_events(&handle)
|
||||
.await
|
||||
.into_iter()
|
||||
.find_map(|event| match event {
|
||||
Event::ToolCallComplete { result, .. } => Some(match result {
|
||||
Ok(output) => output.content,
|
||||
Err(error) => error.to_string(),
|
||||
}),
|
||||
_ => None,
|
||||
})
|
||||
.expect("the call completes");
|
||||
assert!(
|
||||
published.len() < crate::tools::truncate::SPILLOVER_THRESHOLD_BYTES,
|
||||
"as_result={as_result}: {} bytes published",
|
||||
published.len()
|
||||
);
|
||||
assert!(published.contains("FIRST"), "the head survives");
|
||||
assert!(published.contains("LAST"), "the tail survives");
|
||||
assert!(
|
||||
published.contains(crate::tools::truncate::SPILLOVER_RECOVERY_HINT),
|
||||
"the omitted range stays recoverable"
|
||||
);
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,321 @@
|
||||
//! Engine state that must stay truthful across a boundary: the live MCP
|
||||
//! catalog (C02-07/C02-16), a session switch (C02-08/C02-17), a user-input
|
||||
//! wait (C02-19), an `/edit` whose replacement never starts (C02-02), and a
|
||||
//! malformed MCP config (C02-18). No fixture reaches a provider or a shell.
|
||||
|
||||
use super::*;
|
||||
use crate::llm_client::mock::MockLlmClient;
|
||||
|
||||
fn quiet_engine(config: EngineConfig) -> (Engine, EngineHandle) {
|
||||
Engine::new_with_model_client(
|
||||
config,
|
||||
&Config::default(),
|
||||
Arc::new(MockLlmClient::new(Vec::new())),
|
||||
)
|
||||
}
|
||||
|
||||
async fn session_snapshot(handle: &EngineHandle) -> SessionSnapshot {
|
||||
let (tx, rx) = tokio::sync::oneshot::channel();
|
||||
handle
|
||||
.send(Op::GetSessionSnapshot {
|
||||
tx: Arc::new(StdMutex::new(Some(tx))),
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
tokio::time::timeout(model_turn_event_timeout(), rx)
|
||||
.await
|
||||
.expect("snapshot response")
|
||||
.expect("snapshot")
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn live_mcp_refresh_drops_tools_the_pool_no_longer_lists() {
|
||||
// C02-07: the caller's universe is what the pool lists *now*. After a
|
||||
// live 401 that is only the synthetic login tool — never the dead one.
|
||||
let mut catalog = vec![api_tool("mcp_alpha_search"), api_tool("exec_shell")];
|
||||
let mut active: HashSet<String> = ["mcp_alpha_search", "exec_shell"]
|
||||
.into_iter()
|
||||
.map(str::to_string)
|
||||
.collect();
|
||||
let universe: HashSet<String> = ["mcp_alpha_authenticate".to_string()].into();
|
||||
let changed = replace_runtime_mcp_tools(
|
||||
&mut catalog,
|
||||
&mut active,
|
||||
&universe,
|
||||
vec![api_tool("mcp_alpha_authenticate")],
|
||||
AppMode::Agent,
|
||||
&HashSet::new(),
|
||||
crate::model_profile::ToolSurfaceBudget::Standard,
|
||||
);
|
||||
let names: Vec<&str> = catalog.iter().map(|tool| tool.name.as_str()).collect();
|
||||
assert_eq!(names, vec!["exec_shell", "mcp_alpha_authenticate"]);
|
||||
assert!(
|
||||
!active.contains("mcp_alpha_search"),
|
||||
"a dead tool is not callable"
|
||||
);
|
||||
assert!(changed);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn live_mcp_refresh_reports_a_schema_only_edit() {
|
||||
// C02-16: the same names with an edited description is a change the
|
||||
// prefix check must hear about; an identical refresh is not.
|
||||
let refresh = |catalog: &mut Vec<Tool>, active: &mut HashSet<String>, tool: Tool| {
|
||||
replace_runtime_mcp_tools(
|
||||
catalog,
|
||||
active,
|
||||
&["mcp_alpha_read".to_string()].into(),
|
||||
vec![tool],
|
||||
AppMode::Agent,
|
||||
&HashSet::new(),
|
||||
crate::model_profile::ToolSurfaceBudget::Standard,
|
||||
)
|
||||
};
|
||||
let mut catalog = vec![api_tool("exec_shell")];
|
||||
let mut active = HashSet::new();
|
||||
refresh(&mut catalog, &mut active, api_tool("mcp_alpha_read"));
|
||||
let mut edited = api_tool("mcp_alpha_read");
|
||||
edited.description = "Read a resource (now paginated)".to_string();
|
||||
assert!(refresh(&mut catalog, &mut active, edited.clone()));
|
||||
assert!(!refresh(&mut catalog, &mut active, edited));
|
||||
}
|
||||
|
||||
async fn engine_mid_mcp_boot_with_usage() -> (Engine, EngineHandle, tokio::task::JoinHandle<()>) {
|
||||
let workspace = tempdir().unwrap();
|
||||
let (mut engine, handle) = quiet_engine(deterministic_engine_config(workspace.path()));
|
||||
engine.session.total_usage.input_tokens = 1_000;
|
||||
engine.session.total_usage.output_tokens = 200;
|
||||
let boot = tokio::spawn(std::future::pending::<()>());
|
||||
engine.mcp_boot_task = Some(boot.abort_handle());
|
||||
engine.mcp_boot_in_flight = true;
|
||||
engine.mcp_boot_generation = Some(7);
|
||||
engine.mcp_connection_errors.insert(
|
||||
"previous-server".to_string(),
|
||||
"previous failure".to_string(),
|
||||
);
|
||||
// A same-conversation re-sync keeps its usage and its boot pass.
|
||||
let same = engine.session.id.clone();
|
||||
assert!(engine.install_synced_session_id(same).is_none());
|
||||
assert_eq!(engine.session_snapshot().total_tokens, 1_200);
|
||||
assert!(engine.mcp_boot_in_flight);
|
||||
(engine, handle, boot)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_session_boundary_does_not_inherit_the_previous_usage() {
|
||||
// C02-17: session A's tokens are not session B's.
|
||||
let (mut engine, _handle, _boot) = engine_mid_mcp_boot_with_usage().await;
|
||||
assert!(
|
||||
engine
|
||||
.install_synced_session_id("next-session".to_string())
|
||||
.is_some()
|
||||
);
|
||||
assert_eq!(engine.session_snapshot().total_tokens, 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_session_boundary_abandons_the_previous_mcp_boot() {
|
||||
// C02-08: the dropped pool's pass is aborted and its state cleared.
|
||||
let (mut engine, _handle, boot) = engine_mid_mcp_boot_with_usage().await;
|
||||
assert!(
|
||||
engine
|
||||
.install_synced_session_id("next-session".to_string())
|
||||
.is_some()
|
||||
);
|
||||
assert!(!engine.mcp_boot_in_flight);
|
||||
assert_eq!(engine.mcp_boot_generation, None);
|
||||
assert!(engine.mcp_boot_rx.is_none());
|
||||
assert!(engine.mcp_connection_errors.is_empty());
|
||||
let joined = tokio::time::timeout(Duration::from_secs(5), boot)
|
||||
.await
|
||||
.expect("the previous boot pass must stop promptly");
|
||||
assert!(joined.unwrap_err().is_cancelled(), "the pass was aborted");
|
||||
}
|
||||
|
||||
fn empty_user_input_request() -> crate::tools::user_input::UserInputRequest {
|
||||
crate::tools::user_input::UserInputRequest {
|
||||
questions: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_session_boundary_discards_and_stops_the_previous_mcp_supervisor() {
|
||||
let workspace = tempdir().unwrap();
|
||||
let (mut engine, _handle) = quiet_engine(deterministic_engine_config(workspace.path()));
|
||||
let (tx, rx) = mpsc::channel(1);
|
||||
tx.send(McpSupervisorUpdate {
|
||||
died: vec![("previous-server".to_string(), "stale diagnosis".to_string())],
|
||||
failed: Vec::new(),
|
||||
recovered: Vec::new(),
|
||||
parked: Vec::new(),
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
engine.mcp_supervisor_rx = Some(rx);
|
||||
|
||||
// A reconnect future retains its connection resources across await. The
|
||||
// boundary must abort that future, not merely drop a Weak pool owner.
|
||||
let held = Arc::new(());
|
||||
let weak = Arc::downgrade(&held);
|
||||
let (started_tx, started_rx) = tokio::sync::oneshot::channel();
|
||||
let supervisor = tokio::spawn(async move {
|
||||
let _held = held;
|
||||
started_tx.send(()).unwrap();
|
||||
std::future::pending::<()>().await;
|
||||
});
|
||||
started_rx.await.unwrap();
|
||||
engine.mcp_supervisor_task = Some(supervisor.abort_handle());
|
||||
assert!(
|
||||
engine
|
||||
.install_synced_session_id("next-session".to_string())
|
||||
.is_some()
|
||||
);
|
||||
assert!(
|
||||
engine.mcp_supervisor_rx.is_none(),
|
||||
"queued diagnoses retired"
|
||||
);
|
||||
assert!(engine.mcp_supervisor_task.is_none());
|
||||
let joined = tokio::time::timeout(Duration::from_secs(1), supervisor)
|
||||
.await
|
||||
.expect("the previous supervisor must stop promptly");
|
||||
assert!(joined.unwrap_err().is_cancelled());
|
||||
assert!(weak.upgrade().is_none(), "reconnect resources released");
|
||||
assert!(tx.is_closed(), "old diagnoses cannot cross the boundary");
|
||||
|
||||
engine.ensure_mcp_pool().await.unwrap();
|
||||
assert!(engine.mcp_supervisor_rx.is_some(), "the next pool is armed");
|
||||
assert!(engine.mcp_supervisor_task.is_some());
|
||||
assert!(engine.mcp_connection_errors.is_empty());
|
||||
engine.drop_mcp_pool();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn an_undeliverable_user_input_request_fails_fast() {
|
||||
// C02-19: the question cannot reach a host (its event channel is closed,
|
||||
// while the answer channel stays open), so nobody can ever answer it.
|
||||
let workspace = tempdir().unwrap();
|
||||
let (mut engine, handle) = quiet_engine(deterministic_engine_config(workspace.path()));
|
||||
handle.rx_event.write().await.close();
|
||||
let result = tokio::time::timeout(
|
||||
Duration::from_secs(5),
|
||||
engine.await_user_input("ask-undeliverable", empty_user_input_request()),
|
||||
)
|
||||
.await
|
||||
.expect("an undeliverable question must not wait for an answer");
|
||||
assert!(
|
||||
matches!(result, Err(ToolError::ExecutionFailed { .. })),
|
||||
"{result:?}"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn an_open_user_input_question_is_not_charged_to_the_turn() {
|
||||
// C02-19: a delivered question the person leaves open is human time, not
|
||||
// the agent's; the per-turn wall clock does not advance across it.
|
||||
let workspace = tempdir().unwrap();
|
||||
let wait = Duration::from_millis(400);
|
||||
let (mut engine, _handle) = quiet_engine(EngineConfig {
|
||||
user_input_timeout: Some(wait),
|
||||
..deterministic_engine_config(workspace.path())
|
||||
});
|
||||
let before = engine.turn_wall_clock.spent();
|
||||
let result = engine
|
||||
.await_user_input("ask-unanswered", empty_user_input_request())
|
||||
.await;
|
||||
assert!(
|
||||
matches!(result, Err(ToolError::Timeout { .. })),
|
||||
"{result:?}"
|
||||
);
|
||||
let charged = engine.turn_wall_clock.spent().saturating_sub(before);
|
||||
assert!(
|
||||
charged < wait / 2,
|
||||
"the human wait was charged to the turn budget: {charged:?}"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[allow(clippy::await_holding_lock)]
|
||||
async fn edit_last_turn_restores_the_exchange_when_the_replacement_never_starts() {
|
||||
// C02-02: `/edit` cuts the last exchange, then dispatches the new text.
|
||||
// A dispatch that never starts must hand the cut exchange back.
|
||||
let _lock = lock_test_env();
|
||||
let tmp = tempdir().unwrap();
|
||||
let api_config = Config::default().with_legacy_root(
|
||||
Some("test-key".to_string()),
|
||||
Some("http://127.0.0.1:9".to_string()),
|
||||
);
|
||||
let (mut engine, handle) = Engine::new_with_model_client(
|
||||
EngineConfig {
|
||||
workspace: tmp.path().to_path_buf(),
|
||||
model: "deepseek-v4-pro".to_string(),
|
||||
snapshots_enabled: false,
|
||||
subagents_enabled: false,
|
||||
..Default::default()
|
||||
},
|
||||
&api_config,
|
||||
Arc::new(MockLlmClient::new(Vec::new())),
|
||||
);
|
||||
// No model client: the replacement send returns NotStarted.
|
||||
engine.model_client = None;
|
||||
let run = tokio::spawn(engine.run());
|
||||
let text = |role: Role, text: &str| Message {
|
||||
role,
|
||||
content: vec![ContentBlock::Text {
|
||||
text: text.to_string(),
|
||||
cache_control: None,
|
||||
}],
|
||||
};
|
||||
handle
|
||||
.send(Op::SyncSession {
|
||||
session_id: Some("edit-restore".to_string()),
|
||||
messages: vec![
|
||||
text(Role::User, "original prompt"),
|
||||
text(Role::Assistant, "original answer"),
|
||||
],
|
||||
system_prompt: None,
|
||||
system_prompt_override: false,
|
||||
model: "deepseek-v4-pro".to_string(),
|
||||
workspace: tmp.path().to_path_buf(),
|
||||
mode: AppMode::Agent,
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
let before = session_snapshot(&handle).await;
|
||||
assert_eq!(before.messages.len(), 2);
|
||||
handle
|
||||
.send(Op::EditLastTurn {
|
||||
new_message: "edited prompt".to_string(),
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
let after = session_snapshot(&handle).await;
|
||||
assert_eq!(
|
||||
after.messages, before.messages,
|
||||
"a replacement that never started must not cost the original exchange"
|
||||
);
|
||||
handle.send(Op::Shutdown).await.unwrap();
|
||||
run.await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn malformed_mcp_config_is_reported_instead_of_silently_empty() {
|
||||
// C02-18: an unreadable config still yields an empty, reloadable pool,
|
||||
// but the person is told the configured servers are gone.
|
||||
let workspace = tempdir().unwrap();
|
||||
let config_path = workspace.path().join("mcp.json");
|
||||
fs::write(&config_path, "{ this is not json").unwrap();
|
||||
let (mut engine, handle) = quiet_engine(deterministic_engine_config(workspace.path()));
|
||||
engine.session.mcp_config_path = config_path;
|
||||
engine
|
||||
.ensure_mcp_pool()
|
||||
.await
|
||||
.expect("a reloadable empty pool still exists");
|
||||
let mut rx = handle.rx_event.write().await;
|
||||
let reported = std::iter::from_fn(|| rx.try_recv().ok()).any(|event| {
|
||||
matches!(
|
||||
event,
|
||||
Event::Status { ref message } if message.contains("MCP config could not be loaded")
|
||||
)
|
||||
});
|
||||
assert!(reported, "a malformed MCP config must not fail silently");
|
||||
}
|
||||
@@ -112,13 +112,39 @@ async fn preserve_tool_output_before_fanout(
|
||||
tool_id: &str,
|
||||
tool_name: &str,
|
||||
) -> Result<RichToolResult, ToolError> {
|
||||
let mut rich = result?;
|
||||
let model = model.to_owned();
|
||||
let session_id = session_id.to_owned();
|
||||
let tool_id = tool_id.to_owned();
|
||||
let tool_name = tool_name.to_owned();
|
||||
let mut rich = match result {
|
||||
Ok(rich) => rich,
|
||||
// C02-12: an error is fanned out to the event stream and the session
|
||||
// exactly like a result, so an oversized one gets the same bounded
|
||||
// head/tail projection and saved artifact. Ordinary short errors
|
||||
// stay byte-identical (the projection only engages past the
|
||||
// spillover threshold).
|
||||
Err(mut error) => {
|
||||
if tool_error_message_mut(&mut error).is_none_or(|message| {
|
||||
message.len() <= crate::tools::truncate::SPILLOVER_THRESHOLD_BYTES
|
||||
}) {
|
||||
return Err(error);
|
||||
}
|
||||
return tokio::task::spawn_blocking(move || {
|
||||
bound_oversized_tool_error(error, &tool_id, &tool_name, &session_id)
|
||||
})
|
||||
.await
|
||||
.map_err(|join_error| {
|
||||
ToolError::execution_failed(format!(
|
||||
"Tool output preservation failed: {join_error}"
|
||||
))
|
||||
})
|
||||
.and_then(Err);
|
||||
}
|
||||
};
|
||||
tokio::task::spawn_blocking(move || {
|
||||
if let Some(path) = crate::tools::truncate::apply_spillover_with_artifact(
|
||||
// Failed results are bounded too: the fan-out cost of a huge one is
|
||||
// the same whether or not the tool called it a failure (C02-12).
|
||||
if let Some(path) = crate::tools::truncate::apply_spillover_with_artifact_including_errors(
|
||||
&mut rich.result,
|
||||
&tool_id,
|
||||
&tool_name,
|
||||
@@ -155,6 +181,42 @@ async fn preserve_tool_output_before_fanout(
|
||||
})
|
||||
}
|
||||
|
||||
/// The free-form text of a tool error, when its variant carries one.
|
||||
fn tool_error_message_mut(error: &mut ToolError) -> Option<&mut String> {
|
||||
match error {
|
||||
ToolError::InvalidInput { message }
|
||||
| ToolError::ExecutionFailed { message, .. }
|
||||
| ToolError::Cancelled { message }
|
||||
| ToolError::NotAvailable { message }
|
||||
| ToolError::PermissionDenied { message } => Some(message),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Give an oversized tool error message the bounded head/tail projection and
|
||||
/// saved artifact a failed result gets (C02-12). Blocking: it may write the
|
||||
/// artifact, so callers run it under `spawn_blocking`. A failed artifact
|
||||
/// write degrades to the original message, as spillover does for results.
|
||||
fn bound_oversized_tool_error(
|
||||
mut error: ToolError,
|
||||
tool_id: &str,
|
||||
tool_name: &str,
|
||||
session_id: &str,
|
||||
) -> ToolError {
|
||||
let Some(message) = tool_error_message_mut(&mut error) else {
|
||||
return error;
|
||||
};
|
||||
let mut projected = ToolResult::error(std::mem::take(message));
|
||||
crate::tools::truncate::apply_spillover_with_artifact_including_errors(
|
||||
&mut projected,
|
||||
tool_id,
|
||||
tool_name,
|
||||
session_id,
|
||||
);
|
||||
*message = projected.content;
|
||||
error
|
||||
}
|
||||
|
||||
fn approval_intent_summary(text: &str) -> Option<String> {
|
||||
let trimmed = text.trim();
|
||||
if trimmed.is_empty() {
|
||||
@@ -405,6 +467,28 @@ fn incomplete_tool_result(reason: &str) -> ToolResult {
|
||||
}
|
||||
}
|
||||
|
||||
/// Status receipt carried by the one request a reasoning-only / empty-stop
|
||||
/// nudge rides (C02-04). The nudge itself never joins the session.
|
||||
pub(super) const REQUEST_NUDGE_RECEIPT_PREFIX: &str =
|
||||
"Continuing — this retry carries a request-scoped nudge (not saved to the conversation): ";
|
||||
|
||||
/// The not-executed result for a call collected from a response whose stream
|
||||
/// failed before it completed (C02-05). Same shape as
|
||||
/// [`incomplete_tool_result`]: nothing started, so nothing needs undoing.
|
||||
fn stream_failed_tool_result(error: &str) -> ToolResult {
|
||||
ToolResult {
|
||||
content: format!(
|
||||
"Not executed: the provider stream failed before the model response completed ({error})."
|
||||
),
|
||||
success: false,
|
||||
metadata: Some(json!({
|
||||
"side_effect_status": "not_started",
|
||||
"error_category": "model_stream_failed",
|
||||
"model_output_incomplete": true,
|
||||
})),
|
||||
}
|
||||
}
|
||||
|
||||
fn registered_tool_requires_non_bypassable_approval(tool_name: &str) -> bool {
|
||||
// `rlm_eval` (and the unified `rlm` tool whose eval action inherits the
|
||||
// same Required approval) must never bypass explicit approval (#3866).
|
||||
@@ -414,9 +498,11 @@ fn registered_tool_requires_non_bypassable_approval(tool_name: &str) -> bool {
|
||||
/// Replace the runtime-MCP slice of the tool catalog wholesale. An additive
|
||||
/// merge could never remove anything: the synthetic `mcp_<server>_
|
||||
/// authenticate` entry would survive its own successful login, and tools
|
||||
/// killed by a live 401 would stay callable in name. `universe` is every
|
||||
/// name the pool can own; entries inside it are the pool's to manage, and
|
||||
/// the refreshed list is the new truth.
|
||||
/// killed by a live 401 would stay callable in name. The pool owns `universe`
|
||||
/// (every name it can list now) and every MCP name already in the catalog —
|
||||
/// the second half is what lets a tool whose server vanished or lost its
|
||||
/// authorization leave (C02-07); `universe` alone only names survivors. The
|
||||
/// refreshed list is the new truth for all of them.
|
||||
///
|
||||
/// The refreshed slice is shaped exactly like the turn's initial catalog —
|
||||
/// the same deferral pass, the same surface budget, the same always-load
|
||||
@@ -425,6 +511,10 @@ fn registered_tool_requires_non_bypassable_approval(tool_name: &str) -> bool {
|
||||
/// projection carries `defer_loading = false` on every tool, so pushing it
|
||||
/// in unshaped put every MCP tool definition into every remaining request
|
||||
/// of the turn (#5939).
|
||||
///
|
||||
/// Returns whether the catalog or its active set changed in any way — a
|
||||
/// schema or description edit included, not only a count change (C02-16) —
|
||||
/// so the caller can declare the tool-surface change to the prefix check.
|
||||
pub(super) fn replace_runtime_mcp_tools(
|
||||
tool_catalog: &mut Vec<Tool>,
|
||||
active_tool_names: &mut std::collections::HashSet<String>,
|
||||
@@ -433,11 +523,12 @@ pub(super) fn replace_runtime_mcp_tools(
|
||||
mode: AppMode,
|
||||
always_load: &std::collections::HashSet<String>,
|
||||
surface_budget: crate::model_profile::ToolSurfaceBudget,
|
||||
) -> usize {
|
||||
let before = tool_catalog.len();
|
||||
) -> bool {
|
||||
let catalog_before = tool_catalog.clone();
|
||||
let active_before = active_tool_names.clone();
|
||||
let mut previously_active = std::collections::HashSet::new();
|
||||
tool_catalog.retain(|tool| {
|
||||
let owned = universe.contains(&tool.name);
|
||||
let owned = universe.contains(&tool.name) || McpPool::is_mcp_tool(&tool.name);
|
||||
if owned && active_tool_names.remove(&tool.name) {
|
||||
previously_active.insert(tool.name.clone());
|
||||
}
|
||||
@@ -455,7 +546,7 @@ pub(super) fn replace_runtime_mcp_tools(
|
||||
}
|
||||
tool_catalog.push(tool);
|
||||
}
|
||||
tool_catalog.len().abs_diff(before)
|
||||
*tool_catalog != catalog_before || *active_tool_names != active_before
|
||||
}
|
||||
|
||||
impl Engine {
|
||||
@@ -600,6 +691,18 @@ impl Engine {
|
||||
None
|
||||
}
|
||||
|
||||
/// R1: the turn-ending error once the per-turn wall-clock budget is spent.
|
||||
/// Checked wherever the loop is about to authorize a provider request.
|
||||
fn turn_wall_clock_exhausted_error(&self) -> Option<String> {
|
||||
self.turn_wall_clock.exhausted().then(|| {
|
||||
format!(
|
||||
"Per-turn wall-clock budget exhausted after {}s (limit: {}s). The turn was stopped before another model request; work already done is in the transcript. Send another message to continue, or raise `[tui].turn_wall_clock_secs`.",
|
||||
self.turn_wall_clock.spent().as_secs(),
|
||||
self.turn_wall_clock.budget().as_secs(),
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
/// A connection completed during inference must be discoverable in this
|
||||
/// turn, without widening its command policy or making every MCP tool eager.
|
||||
pub(super) async fn refresh_boot_mcp_catalog(
|
||||
@@ -615,23 +718,17 @@ impl Engine {
|
||||
let Some(pool) = self.mcp_pool.as_ref() else {
|
||||
return;
|
||||
};
|
||||
let (mut universe, mut refreshed) = {
|
||||
let (universe, mut refreshed) = {
|
||||
let pool = pool.lock().await;
|
||||
let refreshed = pool.to_api_tools();
|
||||
(pool.model_tool_names(&refreshed), refreshed)
|
||||
};
|
||||
// A config/authority change during handshake can remove a server;
|
||||
// its previous names must also leave this turn's catalog.
|
||||
universe.extend(
|
||||
catalog
|
||||
.iter()
|
||||
.filter(|tool| McpPool::is_mcp_tool(&tool.name))
|
||||
.map(|tool| tool.name.clone()),
|
||||
);
|
||||
// `replace_runtime_mcp_tools` owns every MCP name already in the
|
||||
// catalog, so its previous names leave this turn's catalog too.
|
||||
refreshed
|
||||
.retain(|tool| policy.passes_allow_list(&tool.name) && !policy.denies_tool(&tool.name));
|
||||
let before = catalog.clone();
|
||||
replace_runtime_mcp_tools(
|
||||
if replace_runtime_mcp_tools(
|
||||
catalog,
|
||||
active,
|
||||
&universe,
|
||||
@@ -640,8 +737,7 @@ impl Engine {
|
||||
&self.config.tools_always_load,
|
||||
self.turn_tool_surface_budget
|
||||
.unwrap_or(crate::model_profile::ToolSurfaceBudget::Standard),
|
||||
);
|
||||
if *catalog != before {
|
||||
) {
|
||||
self.session.pending_prefix_change_reason = Some("mcp-session-boot".to_string());
|
||||
}
|
||||
}
|
||||
@@ -997,8 +1093,11 @@ impl Engine {
|
||||
// retry after that carries a nudge — attached to one outbound request
|
||||
// and dropped, never added to the session. Writing it to the session
|
||||
// would put a message the user never sent into the transcript, the
|
||||
// exports, and every later turn's context.
|
||||
let mut reasoning_only_nudge: Option<Message> = None;
|
||||
// exports, and every later turn's context. It is still model-visible,
|
||||
// so the request that carries it also emits a durable internal
|
||||
// status receipt with its exact text (C02-04, "model-visible means
|
||||
// logged"): the runtime event log can reconstruct the request.
|
||||
let mut reasoning_only_nudge: Option<String> = None;
|
||||
// Outer stream-retry budget: when the chunked-transfer connection
|
||||
// dies mid-stream and either nothing useful was streamed (#103
|
||||
// Phase 3), the host slept mid-turn (#2990), or a host hit a
|
||||
@@ -1034,12 +1133,7 @@ impl Engine {
|
||||
// result already produced stays in the transcript. Hitting it is
|
||||
// never a clean success — the turn ends `Failed` with the limit
|
||||
// named, matching how the step ceiling below reports.
|
||||
if self.turn_wall_clock.exhausted() {
|
||||
let error = format!(
|
||||
"Per-turn wall-clock budget exhausted after {}s (limit: {}s). The turn was stopped before another model request; work already done is in the transcript. Send another message to continue, or raise `[tui].turn_wall_clock_secs`.",
|
||||
self.turn_wall_clock.spent().as_secs(),
|
||||
self.turn_wall_clock.budget().as_secs(),
|
||||
);
|
||||
if let Some(error) = self.turn_wall_clock_exhausted_error() {
|
||||
let _ = self.tx_event.send(Event::status(error.clone())).await;
|
||||
return (TurnOutcomeStatus::Failed, Some(error));
|
||||
}
|
||||
@@ -1418,6 +1512,13 @@ impl Engine {
|
||||
}
|
||||
}
|
||||
self.finish_compaction(&compaction_id);
|
||||
// C02-06: a compaction pass has its own bound, not the
|
||||
// turn's. Recheck the wall clock before it can authorize the
|
||||
// provider request below.
|
||||
if let Some(error) = self.turn_wall_clock_exhausted_error() {
|
||||
let _ = self.tx_event.send(Event::status(error.clone())).await;
|
||||
return (TurnOutcomeStatus::Failed, Some(error));
|
||||
}
|
||||
}
|
||||
|
||||
// The guard measures what the compaction gate measures: the honest
|
||||
@@ -1728,14 +1829,18 @@ impl Engine {
|
||||
let fleet_report_response = fleet_denial_guard
|
||||
.as_ref()
|
||||
.is_some_and(FleetDenialGuard::report_only);
|
||||
// `take` is what keeps this request-scoped: the nudge is spent
|
||||
// here and never reaches `self.session.messages`.
|
||||
let request_nudge = reasoning_only_nudge.take();
|
||||
let mut request = prepare_primary_turn_request(PrimaryTurnRequest {
|
||||
model: self.session.model.clone(),
|
||||
messages: {
|
||||
let mut messages = self.messages_with_turn_metadata();
|
||||
// `take` is what keeps this request-scoped: the nudge is
|
||||
// spent here and never reaches `self.session.messages`.
|
||||
if let Some(nudge) = reasoning_only_nudge.take() {
|
||||
messages.push(nudge);
|
||||
if let Some(nudge) = request_nudge.as_ref() {
|
||||
messages.push(self.runtime_text_message_with_turn_metadata(
|
||||
nudge.clone(),
|
||||
UserInputProvenance::Runtime,
|
||||
));
|
||||
}
|
||||
messages
|
||||
},
|
||||
@@ -1748,10 +1853,11 @@ impl Engine {
|
||||
system: self.session.system_prompt.clone(),
|
||||
tools: active_tools.clone(),
|
||||
tool_choice: if active_tools.is_some() {
|
||||
if fleet_report_response {
|
||||
if fleet_report_response || turn.budget_exhausted_final_report {
|
||||
// Keep the pinned tool prefix; only this request's
|
||||
// choice changes. Admission below also enforces this
|
||||
// if a provider ignores the report-only request.
|
||||
// if a provider ignores the report-only request
|
||||
// (C02-10: the step-budget final report included).
|
||||
Some(json!("none"))
|
||||
} else if strict_tool_mode {
|
||||
Some(json!("required"))
|
||||
@@ -1820,6 +1926,16 @@ impl Engine {
|
||||
snapshot: tool_request_snapshot,
|
||||
})
|
||||
.await;
|
||||
if let Some(nudge) = request_nudge {
|
||||
// The "Continuing — " prefix classifies the receipt as
|
||||
// internal: durable clients keep it, collapsed.
|
||||
let _ = self
|
||||
.tx_event
|
||||
.send(Event::status(format!(
|
||||
"{REQUEST_NUDGE_RECEIPT_PREFIX}{nudge}"
|
||||
)))
|
||||
.await;
|
||||
}
|
||||
if let Some(mut route) = turn.pending_route.take() {
|
||||
if let Some(billing) = route.billing.as_mut() {
|
||||
// Freeze the exact provider-live row at CodeWhale's
|
||||
@@ -2015,6 +2131,11 @@ impl Engine {
|
||||
None,
|
||||
Some(super::turn_heartbeat::PREPARING_PHASE_BOUND),
|
||||
);
|
||||
// C02-05: a response whose stream failed — a provider error frame,
|
||||
// a transport error, a stall, or a cap — is not a complete
|
||||
// response. Unless the retry below re-issues the request, nothing
|
||||
// it collected may execute or continue the turn.
|
||||
let response_stream_failed = stream_error.is_some();
|
||||
turn_error = turn_error.or(stream_error);
|
||||
turn.stop_diagnostics
|
||||
.observe_provider_response(stop_reason.as_deref(), tool_uses.len());
|
||||
@@ -2115,26 +2236,8 @@ impl Engine {
|
||||
output_limit_truncated = Some(reason.to_string());
|
||||
// Fall through to the normal content/tool dispatch below.
|
||||
} else {
|
||||
for tool in &tool_uses {
|
||||
let _ = self
|
||||
.tx_event
|
||||
.send(Event::ToolCallStarted {
|
||||
id: tool.execution_id.clone(),
|
||||
model_call: Some(tool.model_call()),
|
||||
name: tool.name.clone(),
|
||||
input: final_tool_input(tool),
|
||||
})
|
||||
.await;
|
||||
let _ = self
|
||||
.tx_event
|
||||
.send(Event::ToolCallComplete {
|
||||
id: tool.execution_id.clone(),
|
||||
model_call: Some(tool.model_call()),
|
||||
name: tool.name.clone(),
|
||||
result: Ok(incomplete_tool_result(reason)),
|
||||
})
|
||||
.await;
|
||||
}
|
||||
self.settle_unadmitted_tool_calls(&tool_uses, &incomplete_tool_result(reason))
|
||||
.await;
|
||||
// Do not emit MessageComplete: hosts must retain the visible
|
||||
// fragment as interrupted/failed rather than recording it as
|
||||
// a completed assistant item.
|
||||
@@ -2334,6 +2437,28 @@ impl Engine {
|
||||
}
|
||||
}
|
||||
|
||||
// C02-05: the one admission authority for a failed stream. Calls
|
||||
// collected before the failure are announced with an explicit
|
||||
// not-executed result and never reach planning, approval or a
|
||||
// handler; the visible text is kept as an interrupted fragment
|
||||
// and no tool_use enters history, so the transcript stays paired.
|
||||
// A retry never gets here: it `continue`d above with this batch
|
||||
// dropped, and the re-issued request streams its own calls.
|
||||
if response_stream_failed && !tool_uses.is_empty() {
|
||||
let error = turn_error
|
||||
.clone()
|
||||
.unwrap_or_else(|| "provider stream failed".to_string());
|
||||
self.settle_unadmitted_tool_calls(
|
||||
&tool_uses,
|
||||
&stream_failed_tool_result(&summarize_text(&error, 200)),
|
||||
)
|
||||
.await;
|
||||
self.add_interrupted_assistant_text(¤t_text_visible)
|
||||
.await;
|
||||
turn.stop_diagnostics.last_response_tool_calls_suppressed = Some(tool_uses.len());
|
||||
return (TurnOutcomeStatus::Failed, Some(error));
|
||||
}
|
||||
|
||||
for tool in &tool_uses {
|
||||
let _ = self
|
||||
.tx_event
|
||||
@@ -2484,6 +2609,15 @@ impl Engine {
|
||||
.await;
|
||||
}
|
||||
|
||||
// C02-05: a failed stream that carried no tool call keeps its text
|
||||
// (above) and ends the turn with the stream's error. It never
|
||||
// runs a ```repl fence from the failed text, and never authorizes
|
||||
// another request for an output-limit continuation, a steer, a
|
||||
// sub-agent completion or a goal continuation.
|
||||
if response_stream_failed {
|
||||
break;
|
||||
}
|
||||
|
||||
// A truncated response with no tool call cannot continue through
|
||||
// tool execution: surface the truncation as a bounded observation
|
||||
// and resume the loop so the model can act on it instead of the
|
||||
@@ -3031,11 +3165,7 @@ impl Engine {
|
||||
crate::config::DEFAULT_REASONING_ONLY_REPROMPT_MESSAGE.to_string()
|
||||
});
|
||||
if !text.trim().is_empty() {
|
||||
reasoning_only_nudge =
|
||||
Some(self.runtime_text_message_with_turn_metadata(
|
||||
text,
|
||||
UserInputProvenance::Runtime,
|
||||
));
|
||||
reasoning_only_nudge = Some(text);
|
||||
}
|
||||
}
|
||||
let how = if nudged {
|
||||
@@ -3093,11 +3223,7 @@ impl Engine {
|
||||
.to_string()
|
||||
});
|
||||
if !text.trim().is_empty() {
|
||||
reasoning_only_nudge =
|
||||
Some(self.runtime_text_message_with_turn_metadata(
|
||||
text,
|
||||
UserInputProvenance::Runtime,
|
||||
));
|
||||
reasoning_only_nudge = Some(text);
|
||||
}
|
||||
"re-requesting the answer with a nudge"
|
||||
}
|
||||
@@ -3555,6 +3681,18 @@ impl Engine {
|
||||
blocked_error = guard.admission_error(&tool_name, &tool_input);
|
||||
}
|
||||
|
||||
// C02-10: the response granted after the step budget ran out is
|
||||
// report-only. A provider that ignores `tool_choice: none` still
|
||||
// gets no execution; the next loop pass ends the turn at the
|
||||
// exhausted budget.
|
||||
if blocked_error.is_none() && turn.budget_exhausted_final_report {
|
||||
blocked_error = Some(ToolError::permission_denied(format!(
|
||||
"Model-step budget exhausted (limit: {}, {}): this is the final report response, so no tool may execute. Report what you did, what you found, what remains, and the evidence.",
|
||||
turn.max_steps,
|
||||
turn.budget_source.key_label(),
|
||||
)));
|
||||
}
|
||||
|
||||
if blocked_error.is_none()
|
||||
&& let Some(error) = tool.input_parse_error.clone()
|
||||
{
|
||||
@@ -5533,7 +5671,7 @@ impl Engine {
|
||||
self.current_mode,
|
||||
&self.config.tools_always_load,
|
||||
surface_budget,
|
||||
) > 0;
|
||||
);
|
||||
}
|
||||
// Any of the legitimate mid-turn tool-surface changes above
|
||||
// re-pin the header under a declared `change:tool_surface`
|
||||
@@ -5840,18 +5978,6 @@ impl Engine {
|
||||
break;
|
||||
}
|
||||
|
||||
// Guard: max accumulated content bytes
|
||||
if stream_content_bytes > max_content_bytes {
|
||||
let envelope = StreamError::Overflow {
|
||||
limit_bytes: max_content_bytes,
|
||||
}
|
||||
.into_envelope();
|
||||
crate::logging::warn(&envelope.message);
|
||||
stream_error.get_or_insert(envelope.message.clone());
|
||||
let _ = self.tx_event.send(Event::error(envelope)).await;
|
||||
break;
|
||||
}
|
||||
|
||||
let event = match event_result {
|
||||
Ok(e) => {
|
||||
self.turn_heartbeat.stream_progress(
|
||||
@@ -5895,6 +6021,15 @@ impl Engine {
|
||||
last_progress_mono.elapsed(),
|
||||
wall_elapsed,
|
||||
));
|
||||
// Like the network-drop resumes below, keep the real
|
||||
// error as the prospective outcome: the retry clears
|
||||
// it, and an exhausted resume budget then fails the
|
||||
// turn with it instead of admitting the partial
|
||||
// response as if it had completed.
|
||||
stream_error.get_or_insert(stream_read_error_user_message(
|
||||
&message,
|
||||
any_content_received,
|
||||
));
|
||||
pending_resume = Some(StreamResume::AfterSleep);
|
||||
break;
|
||||
}
|
||||
@@ -6040,6 +6175,24 @@ impl Engine {
|
||||
}
|
||||
};
|
||||
|
||||
// Guard: max accumulated content bytes (C02-13). Counted per
|
||||
// event and checked before the event is applied, so the delta
|
||||
// that crosses the cap is never forwarded or accumulated — even
|
||||
// when it is the stream's last — and tool-argument JSON counts
|
||||
// like text and reasoning.
|
||||
stream_content_bytes =
|
||||
stream_content_bytes.saturating_add(stream_event_content_bytes(&event));
|
||||
if stream_content_bytes > max_content_bytes {
|
||||
let envelope = StreamError::Overflow {
|
||||
limit_bytes: max_content_bytes,
|
||||
}
|
||||
.into_envelope();
|
||||
crate::logging::warn(&envelope.message);
|
||||
stream_error.get_or_insert(envelope.message.clone());
|
||||
let _ = self.tx_event.send(Event::error(envelope)).await;
|
||||
break;
|
||||
}
|
||||
|
||||
match event {
|
||||
StreamEvent::ToolProjectionWarning {
|
||||
provider,
|
||||
@@ -6150,7 +6303,6 @@ impl Engine {
|
||||
},
|
||||
StreamEvent::ContentBlockDelta { index, delta } => match delta {
|
||||
Delta::TextDelta { text } => {
|
||||
stream_content_bytes = stream_content_bytes.saturating_add(text.len());
|
||||
current_text_raw.push_str(&text);
|
||||
let filtered =
|
||||
filter_tool_call_delta_with_state(&text, &mut tool_call_filter);
|
||||
@@ -6173,7 +6325,6 @@ impl Engine {
|
||||
}
|
||||
}
|
||||
Delta::ThinkingDelta { thinking } => {
|
||||
stream_content_bytes = stream_content_bytes.saturating_add(thinking.len());
|
||||
current_thinking.push_str(&thinking);
|
||||
if !thinking.is_empty() {
|
||||
let _ = self
|
||||
@@ -6338,6 +6489,32 @@ impl Engine {
|
||||
}
|
||||
}
|
||||
|
||||
/// Announce every call of a response that will not be admitted, each
|
||||
/// paired with its not-executed `result`, so a host never shows a started
|
||||
/// call without a completion. Nothing here plans, approves or executes.
|
||||
async fn settle_unadmitted_tool_calls(&self, tool_uses: &[ToolUseState], result: &ToolResult) {
|
||||
for tool in tool_uses {
|
||||
let _ = self
|
||||
.tx_event
|
||||
.send(Event::ToolCallStarted {
|
||||
id: tool.execution_id.clone(),
|
||||
model_call: Some(tool.model_call()),
|
||||
name: tool.name.clone(),
|
||||
input: final_tool_input(tool),
|
||||
})
|
||||
.await;
|
||||
let _ = self
|
||||
.tx_event
|
||||
.send(Event::ToolCallComplete {
|
||||
id: tool.execution_id.clone(),
|
||||
model_call: Some(tool.model_call()),
|
||||
name: tool.name.clone(),
|
||||
result: Ok(result.clone()),
|
||||
})
|
||||
.await;
|
||||
}
|
||||
}
|
||||
|
||||
/// Finalize one streamed tool call's input from its accumulated buffer.
|
||||
///
|
||||
/// The parse that lands here must be structurally intact: a value that
|
||||
@@ -7448,6 +7625,37 @@ fn stream_event_has_actionable_content(event: &StreamEvent) -> bool {
|
||||
}
|
||||
}
|
||||
|
||||
/// Bytes an event adds to the response the engine accumulates: text,
|
||||
/// reasoning, tool calls (id, name and argument JSON, whether the arguments
|
||||
/// arrive whole in the block start or as `InputJsonDelta`s) and replay
|
||||
/// signatures. Opaque reasoning state is provider-owned and not counted.
|
||||
fn stream_event_content_bytes(event: &StreamEvent) -> usize {
|
||||
fn initial_input_bytes(input: &Value) -> usize {
|
||||
let empty = input.is_null() || input.as_object().is_some_and(serde_json::Map::is_empty);
|
||||
if empty { 0 } else { input.to_string().len() }
|
||||
}
|
||||
match event {
|
||||
StreamEvent::ContentBlockStart { content_block, .. } => match content_block {
|
||||
ContentBlockStart::Text { text } => text.len(),
|
||||
ContentBlockStart::Thinking { thinking } => thinking.len(),
|
||||
ContentBlockStart::ToolUse {
|
||||
id, name, input, ..
|
||||
}
|
||||
| ContentBlockStart::ServerToolUse { id, name, input } => {
|
||||
id.len() + name.len() + initial_input_bytes(input)
|
||||
}
|
||||
},
|
||||
StreamEvent::ContentBlockDelta { delta, .. } => match delta {
|
||||
Delta::TextDelta { text } => text.len(),
|
||||
Delta::ThinkingDelta { thinking } => thinking.len(),
|
||||
Delta::InputJsonDelta { partial_json } => partial_json.len(),
|
||||
Delta::SignatureDelta { signature } => signature.len(),
|
||||
Delta::ReasoningStateDelta { .. } => 0,
|
||||
},
|
||||
_ => 0,
|
||||
}
|
||||
}
|
||||
|
||||
/// Sentinel reasoning-effort value meaning "let the auto-reasoning system
|
||||
/// decide" (#4158).
|
||||
pub(super) const REASONING_EFFORT_AUTO: &str = "auto";
|
||||
|
||||
+410
-85
@@ -1104,11 +1104,30 @@ fn resolve_exec_resume_session_id(args: &ExecArgs, workspace: &Path) -> Result<O
|
||||
}
|
||||
|
||||
fn load_exec_resume_session(session_id: &str) -> Result<session_manager::SavedSession> {
|
||||
SessionManager::default_location()
|
||||
match SessionManager::default_location()
|
||||
.context("could not open session manager for resume")?
|
||||
.resume_session_by_prefix(session_id)
|
||||
.map(|recovery| recovery.session)
|
||||
.with_context(|| exec_resume_load_error(session_id))
|
||||
.attach_session_by_prefix(session_id)
|
||||
{
|
||||
Ok((recovery, lease)) => {
|
||||
// This exec run owns the session until it exits.
|
||||
lease.commit();
|
||||
Ok(recovery.session)
|
||||
}
|
||||
// Resuming a session a TUI has open would give its document two
|
||||
// writers; the TUI's next autosave would drop this run's turns.
|
||||
Err(error) if error.kind() == io::ErrorKind::ResourceBusy => {
|
||||
bail!(exec_resume_busy_error(session_id))
|
||||
}
|
||||
Err(error) => Err(error).with_context(|| exec_resume_load_error(session_id)),
|
||||
}
|
||||
}
|
||||
|
||||
fn exec_resume_busy_error(session_id: &str) -> String {
|
||||
format!(
|
||||
"session {} is open in another Codewhale window. Continue it there, or run \
|
||||
`codewhale fork <SESSION_ID>` and resume the copy.",
|
||||
exec_stream_session_ref(session_id)
|
||||
)
|
||||
}
|
||||
|
||||
/// The typed `--resume` value stays redacted in every output mode: exec runs
|
||||
@@ -1312,7 +1331,7 @@ struct ScorecardArgs {
|
||||
#[arg(long, value_name = "FILE")]
|
||||
baseline: Option<PathBuf>,
|
||||
/// Regression threshold, in percent increase over the baseline.
|
||||
#[arg(long, default_value_t = 5.0)]
|
||||
#[arg(long, default_value_t = 5.0, value_parser = parse_regression_threshold)]
|
||||
threshold: f64,
|
||||
/// Emit machine-readable JSON instead of the human summary.
|
||||
#[arg(long, default_value_t = false)]
|
||||
@@ -1996,7 +2015,7 @@ fn run_with_args(args: Vec<String>) -> Result<()> {
|
||||
plugin_registry = Some(discovery.registry_for_workspace(&workspace));
|
||||
plugin_discovery = Some(discovery);
|
||||
},
|
||||
warn_on_workspace_dotenv_result,
|
||||
|| warn_on_workspace_dotenv_result(&workspace),
|
||||
);
|
||||
let plugin_discovery = plugin_discovery
|
||||
.expect("plugin discovery initialization must precede workspace dotenv loading");
|
||||
@@ -2962,8 +2981,8 @@ struct WorkspaceDotenvReport {
|
||||
/// MCP servers, plugin trust, executable lookup, sandbox/approval posture, or
|
||||
/// network destinations. Shell-exported values and config/CLI arguments remain
|
||||
/// the explicit surfaces for those controls.
|
||||
fn warn_on_workspace_dotenv_result() {
|
||||
match load_workspace_dotenv_credentials() {
|
||||
fn warn_on_workspace_dotenv_result(workspace: &Path) {
|
||||
match load_workspace_dotenv_credentials(workspace) {
|
||||
Ok(Some(report)) if !report.ignored.is_empty() => {
|
||||
eprintln!(
|
||||
"Codewhale ignored non-credential settings in {}: {}. Use config.toml, CLI flags, or the launching shell for control settings.",
|
||||
@@ -3003,21 +3022,31 @@ fn display_env_key_set(keys: &BTreeSet<String>) -> String {
|
||||
labels.join(", ")
|
||||
}
|
||||
|
||||
fn load_workspace_dotenv_credentials() -> Result<Option<WorkspaceDotenvReport>> {
|
||||
let Some(path) = find_workspace_dotenv()? else {
|
||||
fn load_workspace_dotenv_credentials(workspace: &Path) -> Result<Option<WorkspaceDotenvReport>> {
|
||||
let Some(path) = find_workspace_dotenv(workspace)? else {
|
||||
return Ok(None);
|
||||
};
|
||||
load_workspace_dotenv_credentials_from_path(&path).map(Some)
|
||||
}
|
||||
|
||||
fn find_workspace_dotenv() -> Result<Option<PathBuf>> {
|
||||
let cwd = std::env::current_dir().context("could not resolve the current workspace")?;
|
||||
let boundary = cwd
|
||||
/// The nearest `.env` from the resolved launch workspace (`--workspace`, else
|
||||
/// the current directory) up to its repository root. Searching from the
|
||||
/// process directory instead would load another tree's credentials when the
|
||||
/// two differ.
|
||||
fn find_workspace_dotenv(workspace: &Path) -> Result<Option<PathBuf>> {
|
||||
let start = if workspace.is_absolute() {
|
||||
workspace.to_path_buf()
|
||||
} else {
|
||||
std::env::current_dir()
|
||||
.context("could not resolve the current workspace")?
|
||||
.join(workspace)
|
||||
};
|
||||
let boundary = start
|
||||
.ancestors()
|
||||
.find(|ancestor| std::fs::symlink_metadata(ancestor.join(".git")).is_ok())
|
||||
.unwrap_or(cwd.as_path());
|
||||
.unwrap_or(start.as_path());
|
||||
|
||||
for ancestor in cwd.ancestors() {
|
||||
for ancestor in start.ancestors() {
|
||||
let candidate = ancestor.join(".env");
|
||||
match std::fs::symlink_metadata(&candidate) {
|
||||
Ok(_) => return Ok(Some(candidate)),
|
||||
@@ -3346,6 +3375,20 @@ fn run_eval(args: EvalArgs) -> Result<()> {
|
||||
}
|
||||
}
|
||||
|
||||
/// A regression gate threshold must be a finite percentage: `NaN` compares
|
||||
/// false against every change (a gate that always passes) and infinity
|
||||
/// disables the gate outright.
|
||||
fn parse_regression_threshold(raw: &str) -> Result<f64, String> {
|
||||
let value: f64 = raw
|
||||
.trim()
|
||||
.parse()
|
||||
.map_err(|_| format!("`{raw}` is not a number"))?;
|
||||
if !value.is_finite() {
|
||||
return Err(format!("`{raw}` is not a finite percentage"));
|
||||
}
|
||||
Ok(value)
|
||||
}
|
||||
|
||||
/// Score a run's token/cache/cost from recorded turns and (optionally) flag
|
||||
/// regressions against a committed baseline. Offline: reads recorded usage from
|
||||
/// a JSON file, reuses the pricing layer, never calls a model. Exits non-zero
|
||||
@@ -11192,10 +11235,26 @@ fn run_sandbox_command(args: SandboxArgs) -> Result<()> {
|
||||
.stdout(Stdio::piped())
|
||||
.stderr(Stdio::piped());
|
||||
child_env::apply_to_command(&mut cmd, child_env::string_map_env(&exec_env.env));
|
||||
// Lead a process group so the timeout ends everything the command started.
|
||||
#[cfg(unix)]
|
||||
{
|
||||
use std::os::unix::process::CommandExt as _;
|
||||
cmd.process_group(0);
|
||||
}
|
||||
|
||||
let mut child = cmd
|
||||
.spawn()
|
||||
.map_err(|e| anyhow::anyhow!("Failed to run command: {e}"))?;
|
||||
// The sandbox run is the tree's lifetime: dropping `tree` on return ends
|
||||
// anything the command left running, as `contained_output` does.
|
||||
let tree = match crate::process_tree::ProcessTree::attach(&child) {
|
||||
Ok(tree) => tree,
|
||||
Err(error) => {
|
||||
let _ = child.kill();
|
||||
let _ = child.wait();
|
||||
bail!("Failed to contain the sandboxed command: {error}");
|
||||
}
|
||||
};
|
||||
let stdout_handle = child
|
||||
.stdout
|
||||
.take()
|
||||
@@ -11205,48 +11264,78 @@ fn run_sandbox_command(args: SandboxArgs) -> Result<()> {
|
||||
.take()
|
||||
.ok_or_else(|| anyhow::anyhow!("stderr unavailable"))?;
|
||||
|
||||
let timeout = exec_env.timeout;
|
||||
let stdout_thread = std::thread::spawn(move || {
|
||||
// Output streams straight through instead of being buffered whole: a
|
||||
// command's output size is unbounded, so only a bounded stderr tail is
|
||||
// kept for sandbox-denial detection.
|
||||
const STDERR_TAIL_BYTES: usize = 64 * 1024;
|
||||
let (done_tx, done_rx) = std::sync::mpsc::channel::<()>();
|
||||
let stdout_done = done_tx.clone();
|
||||
std::thread::spawn(move || {
|
||||
let mut reader = stdout_handle;
|
||||
let mut buf = Vec::new();
|
||||
let _ = reader.read_to_end(&mut buf);
|
||||
buf
|
||||
let _ = io::copy(&mut reader, &mut io::stdout());
|
||||
let _ = stdout_done.send(());
|
||||
});
|
||||
let stderr_thread = std::thread::spawn(move || {
|
||||
let stderr_tail = Arc::new(std::sync::Mutex::new(Vec::<u8>::new()));
|
||||
let tail = Arc::clone(&stderr_tail);
|
||||
std::thread::spawn(move || {
|
||||
let mut reader = stderr_handle;
|
||||
let mut buf = Vec::new();
|
||||
let _ = reader.read_to_end(&mut buf);
|
||||
buf
|
||||
let mut chunk = [0_u8; 8192];
|
||||
loop {
|
||||
let read = match reader.read(&mut chunk) {
|
||||
Ok(0) => break,
|
||||
Ok(read) => read,
|
||||
Err(error) if error.kind() == io::ErrorKind::Interrupted => continue,
|
||||
Err(_) => break,
|
||||
};
|
||||
let _ = io::stderr().write_all(&chunk[..read]);
|
||||
if let Ok(mut tail) = tail.lock() {
|
||||
tail.extend_from_slice(&chunk[..read]);
|
||||
let excess = tail.len().saturating_sub(STDERR_TAIL_BYTES);
|
||||
tail.drain(..excess);
|
||||
}
|
||||
}
|
||||
let _ = done_tx.send(());
|
||||
});
|
||||
|
||||
if let Some(status) = child.wait_timeout(timeout)? {
|
||||
let stdout = stdout_thread.join().unwrap_or_default();
|
||||
let stderr = stderr_thread.join().unwrap_or_default();
|
||||
let stderr_str = String::from_utf8_lossy(&stderr);
|
||||
let exit_code = status.code().unwrap_or(-1);
|
||||
let sandbox_type = exec_env.sandbox_type;
|
||||
let sandbox_denied = SandboxManager::was_denied(sandbox_type, exit_code, &stderr_str);
|
||||
|
||||
if !stdout.is_empty() {
|
||||
print!("{}", String::from_utf8_lossy(&stdout));
|
||||
}
|
||||
if !stderr.is_empty() {
|
||||
eprint!("{stderr_str}");
|
||||
}
|
||||
if sandbox_denied {
|
||||
eprintln!(
|
||||
"{}",
|
||||
SandboxManager::denial_message(sandbox_type, &stderr_str)
|
||||
);
|
||||
}
|
||||
|
||||
if !status.success() {
|
||||
bail!("Command failed with exit code {exit_code}");
|
||||
}
|
||||
} else {
|
||||
let timeout = exec_env.timeout;
|
||||
let deadline = Instant::now() + timeout;
|
||||
let Some(status) = child.wait_timeout(timeout)? else {
|
||||
let _ = tree.kill();
|
||||
let _ = child.kill();
|
||||
let _ = child.wait();
|
||||
bail!("Command timed out after {}ms", timeout.as_millis());
|
||||
};
|
||||
// A descendant may still hold the output pipes. Let it finish inside the
|
||||
// same budget, then end the tree so the drain always reaches EOF.
|
||||
let mut drained = 0;
|
||||
while drained < 2 {
|
||||
match done_rx.recv_timeout(deadline.saturating_duration_since(Instant::now())) {
|
||||
Ok(()) => drained += 1,
|
||||
Err(_) => break,
|
||||
}
|
||||
}
|
||||
if drained < 2 {
|
||||
let _ = tree.kill();
|
||||
while drained < 2 && done_rx.recv_timeout(Duration::from_secs(1)).is_ok() {
|
||||
drained += 1;
|
||||
}
|
||||
}
|
||||
|
||||
let stderr = stderr_tail
|
||||
.lock()
|
||||
.map(|tail| tail.clone())
|
||||
.unwrap_or_default();
|
||||
let stderr_str = String::from_utf8_lossy(&stderr);
|
||||
let exit_code = status.code().unwrap_or(-1);
|
||||
let sandbox_type = exec_env.sandbox_type;
|
||||
if SandboxManager::was_denied(sandbox_type, exit_code, &stderr_str) {
|
||||
eprintln!(
|
||||
"{}",
|
||||
SandboxManager::denial_message(sandbox_type, &stderr_str)
|
||||
);
|
||||
}
|
||||
if !status.success() {
|
||||
bail!("Command failed with exit code {exit_code}");
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -11400,6 +11489,20 @@ fn load_recent_checkpoints(manager: &session_manager::SessionManager) -> Vec<Rec
|
||||
let refs = manager.list_checkpoints().unwrap_or_default();
|
||||
let mut recent = Vec::new();
|
||||
for checkpoint_ref in refs {
|
||||
// A session open in another terminal refreshes its own checkpoint
|
||||
// mid-turn. It is not interrupted: promoting or clearing it would
|
||||
// take that session's only crash-recovery record while it runs. The
|
||||
// legacy slot is checked by the session it names, before anything
|
||||
// prunes, promotes or migrates it over that session's live state.
|
||||
let owner = match &checkpoint_ref.source {
|
||||
session_manager::CheckpointSource::Session(id) => Some(id.clone()),
|
||||
session_manager::CheckpointSource::Legacy => {
|
||||
manager.legacy_checkpoint_origin().ok().flatten()
|
||||
}
|
||||
};
|
||||
if owner.is_some_and(|id| manager.is_session_live_anywhere(&id)) {
|
||||
continue;
|
||||
}
|
||||
let Ok(age) = std::time::SystemTime::now().duration_since(checkpoint_ref.modified) else {
|
||||
continue;
|
||||
};
|
||||
@@ -11507,6 +11610,15 @@ fn recover_interrupted_checkpoint_for_resume(launch_workspace: &Path) -> Option<
|
||||
|
||||
let session_id = best.session.metadata.id.clone();
|
||||
|
||||
// Take the session's live lease before promoting or clearing anything:
|
||||
// the liveness filter above is a check, and another terminal can attach
|
||||
// between it and these writes. The TUI's own attach then finds the lease
|
||||
// already held by this process. Losing the race leaves every file alone.
|
||||
match manager.reserve_session_for_attach(&session_id) {
|
||||
Ok(lease) => lease.commit(),
|
||||
Err(_) => return None,
|
||||
}
|
||||
|
||||
// Persist the checkpoint as a regular session so the TUI can load it by
|
||||
// id — unless a newer regular session file for the same id already
|
||||
// exists (e.g. `--continue` ran before and the session advanced since).
|
||||
@@ -11601,7 +11713,19 @@ fn preserve_interrupted_checkpoint_for_explicit_resume(launch_workspace: &Path)
|
||||
/// else falls back to the global value.
|
||||
#[cfg(test)]
|
||||
fn merge_project_config(config: &mut Config, workspace: &Path) {
|
||||
merge_project_config_with_approval_baseline(config, workspace, None);
|
||||
merge_project_config_with_approval_baseline(config, workspace, None)
|
||||
.expect("project config applies");
|
||||
}
|
||||
|
||||
/// A project config that exists but cannot be applied is an error, not an
|
||||
/// absent file: it may be the thing tightening approval, sandbox or shell for
|
||||
/// this workspace, and launching on the looser user baseline without it would
|
||||
/// fail open. The reason never quotes file contents.
|
||||
fn project_config_unusable(path: &Path, reason: &str) -> anyhow::Error {
|
||||
anyhow!(
|
||||
"Project config {} could not be applied ({reason}), so its approval, sandbox and shell restrictions are not in effect. Fix the file, or launch with --no-project-config to ignore it.",
|
||||
path.display()
|
||||
)
|
||||
}
|
||||
|
||||
/// Apply project config while evaluating approval tightening against the
|
||||
@@ -11612,7 +11736,7 @@ fn merge_project_config_with_approval_baseline(
|
||||
config: &mut Config,
|
||||
workspace: &Path,
|
||||
saved_permission_posture: Option<&str>,
|
||||
) {
|
||||
) -> Result<()> {
|
||||
// When the workspace is the user's home directory, the project-scope
|
||||
// config file is also the global config file. Skip the merge to avoid
|
||||
// redundant processing and a misleading "project-scope config key
|
||||
@@ -11624,46 +11748,41 @@ fn merge_project_config_with_approval_baseline(
|
||||
)
|
||||
&& w == h
|
||||
{
|
||||
return;
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
// v0.8.44: prefer .codewhale/config.toml, fall back to .deepseek/
|
||||
let path = workspace
|
||||
let primary = workspace
|
||||
.join(codewhale_config::CODEWHALE_APP_DIR)
|
||||
.join("config.toml");
|
||||
let raw = match read_project_config_file(&path) {
|
||||
Ok(Some(r)) => r,
|
||||
let (path, raw) = match read_project_config_file(&primary) {
|
||||
Ok(Some(raw)) => (primary, raw),
|
||||
Ok(None) => {
|
||||
let legacy = workspace
|
||||
.join(codewhale_config::LEGACY_APP_DIR)
|
||||
.join("config.toml");
|
||||
match read_project_config_file(&legacy) {
|
||||
Ok(Some(r)) => r,
|
||||
Ok(None) => return,
|
||||
Err(err) => {
|
||||
eprintln!(
|
||||
"warning: failed to read project-scope config {}: {err}",
|
||||
legacy.display()
|
||||
);
|
||||
return;
|
||||
}
|
||||
Ok(Some(raw)) => (legacy, raw),
|
||||
Ok(None) => return Ok(()),
|
||||
Err(err) => return Err(project_config_unusable(&legacy, &err.to_string())),
|
||||
}
|
||||
}
|
||||
Err(err) => {
|
||||
eprintln!(
|
||||
"warning: failed to read project-scope config {}: {err}",
|
||||
path.display()
|
||||
);
|
||||
return;
|
||||
}
|
||||
Err(err) => return Err(project_config_unusable(&primary, &err.to_string())),
|
||||
};
|
||||
let project: toml::Value = match toml::from_str(&raw) {
|
||||
Ok(v) => v,
|
||||
Err(_) => return,
|
||||
};
|
||||
let table = match project.as_table() {
|
||||
Some(t) => t,
|
||||
None => return,
|
||||
let project: toml::Value = toml::from_str(&raw).map_err(|err| {
|
||||
// Position only: the parser's message can quote the offending value.
|
||||
let reason = err.span().map_or_else(
|
||||
|| "invalid TOML".to_string(),
|
||||
|span| {
|
||||
let prefix = &raw.as_bytes()[..span.start.min(raw.len())];
|
||||
let line = prefix.iter().filter(|byte| **byte == b'\n').count() + 1;
|
||||
format!("invalid TOML at line {line}")
|
||||
},
|
||||
);
|
||||
project_config_unusable(&path, &reason)
|
||||
})?;
|
||||
let Some(table) = project.as_table() else {
|
||||
return Err(project_config_unusable(&path, "not a TOML table"));
|
||||
};
|
||||
|
||||
// #417: dangerous keys are denied at project scope. A malicious
|
||||
@@ -11779,6 +11898,7 @@ fn merge_project_config_with_approval_baseline(
|
||||
(See #417.)"
|
||||
);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Maximum bytes read from a project config file. Configs are kilobytes.
|
||||
@@ -12012,7 +12132,7 @@ async fn run_interactive_with_notice(
|
||||
&mut merged_config,
|
||||
&workspace,
|
||||
saved_permission_posture.as_deref(),
|
||||
);
|
||||
)?;
|
||||
}
|
||||
if resume_session_id.is_none() {
|
||||
let explicit_route_override = crate::config::explicit_launch_provider_override().is_some()
|
||||
@@ -18757,6 +18877,56 @@ api_key = "test-only-key"
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scorecard_rejects_a_threshold_that_cannot_gate() {
|
||||
for bad in ["NaN", "inf", "-inf"] {
|
||||
assert!(
|
||||
Cli::try_parse_from([
|
||||
"codewhale",
|
||||
"scorecard",
|
||||
"--input",
|
||||
"t.json",
|
||||
"--threshold",
|
||||
bad
|
||||
])
|
||||
.is_err(),
|
||||
"--threshold {bad} must be refused"
|
||||
);
|
||||
}
|
||||
assert!(
|
||||
Cli::try_parse_from([
|
||||
"codewhale",
|
||||
"scorecard",
|
||||
"--input",
|
||||
"t.json",
|
||||
"--threshold",
|
||||
"2.5"
|
||||
])
|
||||
.is_ok()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn workspace_dotenv_is_found_from_the_launch_workspace_not_the_process_directory() {
|
||||
let workspace = tempfile::tempdir().expect("workspace tempdir");
|
||||
let nested = workspace.path().join("crates/app");
|
||||
std::fs::create_dir_all(&nested).expect("mkdir nested");
|
||||
std::fs::create_dir_all(workspace.path().join(".git")).expect("mkdir .git");
|
||||
std::fs::write(workspace.path().join(".env"), "OPENAI_API_KEY=workspace\n")
|
||||
.expect("write .env");
|
||||
|
||||
// `--workspace <dir>` from a process directory in another tree.
|
||||
assert_eq!(
|
||||
find_workspace_dotenv(workspace.path()).expect("search"),
|
||||
Some(workspace.path().join(".env"))
|
||||
);
|
||||
// Nested launch directory walks up to the repository root.
|
||||
assert_eq!(
|
||||
find_workspace_dotenv(&nested).expect("search"),
|
||||
Some(workspace.path().join(".env"))
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn workspace_dotenv_loads_only_provider_credentials_and_preserves_shell_values() {
|
||||
let _lock = crate::test_support::lock_test_env();
|
||||
@@ -19995,7 +20165,10 @@ mod project_config_tests {
|
||||
..Config::default()
|
||||
};
|
||||
|
||||
merge_project_config(&mut config, workspace.path());
|
||||
let error =
|
||||
merge_project_config_with_approval_baseline(&mut config, workspace.path(), None)
|
||||
.expect_err("a symlinked primary project config must stop the launch");
|
||||
assert!(error.to_string().contains("--no-project-config"), "{error}");
|
||||
|
||||
assert_eq!(
|
||||
config.default_text_model.as_deref(),
|
||||
@@ -20321,7 +20494,8 @@ approval_policy = "on-request"
|
||||
);
|
||||
let mut config = Config::default();
|
||||
|
||||
merge_project_config_with_approval_baseline(&mut config, tmp.path(), Some("full-access"));
|
||||
merge_project_config_with_approval_baseline(&mut config, tmp.path(), Some("full-access"))
|
||||
.expect("valid project config tightens the saved baseline");
|
||||
|
||||
assert_eq!(
|
||||
config.approval_policy.as_deref(),
|
||||
@@ -20602,14 +20776,22 @@ max_subagents = -3
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn project_overlay_skips_malformed_toml() {
|
||||
let tmp = workspace_with_project_config("this is not valid TOML !!");
|
||||
fn project_overlay_refuses_malformed_toml_instead_of_dropping_its_restrictions() {
|
||||
let tmp = workspace_with_project_config(
|
||||
"approval_policy = \"on-request\"\nallow_shell = false\nthis is not valid TOML !!",
|
||||
);
|
||||
let mut config = Config {
|
||||
provider: Some("codewhale".to_string()),
|
||||
..Config::default()
|
||||
};
|
||||
merge_project_config(&mut config, tmp.path());
|
||||
// Untouched on parse error — better to fall back to global than crash.
|
||||
// A broken file may be the one tightening this workspace; launching on
|
||||
// the looser user baseline without it would fail open.
|
||||
let error = merge_project_config_with_approval_baseline(&mut config, tmp.path(), None)
|
||||
.expect_err("a malformed project config must stop the launch");
|
||||
let message = error.to_string();
|
||||
assert!(message.contains("invalid TOML at line 3"), "{message}");
|
||||
assert!(message.contains("--no-project-config"), "{message}");
|
||||
assert!(!message.contains("this is not valid"), "{message}");
|
||||
assert_eq!(config.provider.as_deref(), Some("codewhale"));
|
||||
}
|
||||
|
||||
@@ -21195,6 +21377,9 @@ mod setup_helper_tests {
|
||||
std::env::set_var("USERPROFILE", home);
|
||||
}
|
||||
let result = f();
|
||||
// `--continue` recovery takes the recovered session's live lease for
|
||||
// the process; release it with the temporary home it lives in.
|
||||
crate::session_manager::set_live_session(None);
|
||||
unsafe {
|
||||
match prev_home {
|
||||
Some(value) => std::env::set_var("HOME", value),
|
||||
@@ -21328,6 +21513,72 @@ mod setup_helper_tests {
|
||||
});
|
||||
}
|
||||
|
||||
/// `--continue` in a second terminal must not take the session the first
|
||||
/// terminal is still running: no promotion or clear of its checkpoint, no
|
||||
/// silent swap to an older session, and the attach is refused by name.
|
||||
#[test]
|
||||
fn continue_leaves_a_session_live_in_another_terminal_alone() {
|
||||
let _guard = crate::test_support::lock_test_env();
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let workspace = tmp.path().join("workspace");
|
||||
std::fs::create_dir_all(&workspace).unwrap();
|
||||
|
||||
with_home(tmp.path(), || {
|
||||
let manager = SessionManager::default_location().expect("manager");
|
||||
let message = |text: &str| {
|
||||
vec![Message {
|
||||
role: Role::User,
|
||||
content: vec![ContentBlock::Text {
|
||||
text: text.to_string(),
|
||||
cache_control: None,
|
||||
}],
|
||||
}]
|
||||
};
|
||||
let older = create_saved_session(&message("older"), "test-model", &workspace, 0, None);
|
||||
manager.save_session(&older).expect("save older");
|
||||
std::thread::sleep(std::time::Duration::from_millis(20));
|
||||
let session =
|
||||
create_saved_session(&message("still running"), "test-model", &workspace, 0, None);
|
||||
let session_id = session.metadata.id.clone();
|
||||
manager.save_session(&session).expect("save session");
|
||||
manager.save_checkpoint(&session).expect("save checkpoint");
|
||||
|
||||
let lease = manager.hold_live_lease_elsewhere(&session_id);
|
||||
let resolved = resolve_continue_session_id(&workspace, true);
|
||||
assert_eq!(
|
||||
resolved.as_deref(),
|
||||
Some(session_id.as_str()),
|
||||
"the newest session is named, not swapped for an older one"
|
||||
);
|
||||
assert!(
|
||||
manager
|
||||
.load_session_checkpoint(&session_id)
|
||||
.expect("load checkpoint")
|
||||
.is_some(),
|
||||
"the live session keeps its crash-recovery checkpoint"
|
||||
);
|
||||
let refusal = manager
|
||||
.attach_session(&session_id)
|
||||
.expect_err("attaching to it is refused");
|
||||
assert_eq!(refusal.kind(), io::ErrorKind::ResourceBusy);
|
||||
assert!(refusal.to_string().contains(&session_id), "{refusal}");
|
||||
assert!(
|
||||
load_exec_resume_session(&session_id)
|
||||
.expect_err("exec --continue is refused too")
|
||||
.to_string()
|
||||
.contains("open in another Codewhale window")
|
||||
);
|
||||
drop(lease);
|
||||
|
||||
// Once that session has exited, --continue recovers it as before.
|
||||
assert_eq!(
|
||||
resolve_continue_session_id(&workspace, true).as_deref(),
|
||||
Some(session_id.as_str())
|
||||
);
|
||||
crate::session_manager::set_live_session(None);
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn continue_without_interactive_terminal_leaves_checkpoint_for_a_real_launch() {
|
||||
// `codewhale --continue </dev/null` (and `run --continue`) used to
|
||||
@@ -21391,6 +21642,80 @@ mod setup_helper_tests {
|
||||
std::fs::write(checkpoints.join("latest.json"), content).expect("write legacy checkpoint");
|
||||
}
|
||||
|
||||
/// The legacy `latest.json` slot names a session. When that session is
|
||||
/// open in another terminal, neither `--continue` nor a plain launch may
|
||||
/// promote the slot over its document, overwrite its per-session
|
||||
/// checkpoint, or consume the slot.
|
||||
#[test]
|
||||
fn legacy_checkpoint_of_a_session_live_elsewhere_is_left_alone() {
|
||||
let _guard = crate::test_support::lock_test_env();
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let workspace = tmp.path().join("workspace");
|
||||
std::fs::create_dir_all(&workspace).unwrap();
|
||||
|
||||
with_home(tmp.path(), || {
|
||||
let manager = SessionManager::default_location().expect("manager");
|
||||
let message = |text: &str| {
|
||||
vec![Message {
|
||||
role: Role::User,
|
||||
content: vec![ContentBlock::Text {
|
||||
text: text.to_string(),
|
||||
cache_control: None,
|
||||
}],
|
||||
}]
|
||||
};
|
||||
let live =
|
||||
create_saved_session(&message("live turn"), "test-model", &workspace, 0, None);
|
||||
let session_id = live.metadata.id.clone();
|
||||
let document = manager.save_session(&live).expect("save live document");
|
||||
manager
|
||||
.save_checkpoint(&live)
|
||||
.expect("save live checkpoint");
|
||||
let mut stale = live.clone();
|
||||
stale.messages = message("stale legacy slot");
|
||||
stale.metadata.updated_at = live.metadata.updated_at + chrono::Duration::seconds(60);
|
||||
write_legacy_checkpoint(&manager, &stale);
|
||||
let legacy = manager
|
||||
.sessions_dir()
|
||||
.join("checkpoints")
|
||||
.join("latest.json");
|
||||
let document_before = std::fs::read(&document).expect("document");
|
||||
let checkpoint = || {
|
||||
serde_json::to_string(
|
||||
&manager
|
||||
.load_session_checkpoint(&session_id)
|
||||
.expect("checkpoint")
|
||||
.expect("present")
|
||||
.messages,
|
||||
)
|
||||
.expect("serialize")
|
||||
};
|
||||
let checkpoint_before = checkpoint();
|
||||
|
||||
let lease = manager.hold_live_lease_elsewhere(&session_id);
|
||||
let recovered = recover_interrupted_checkpoint_for_resume(&workspace);
|
||||
preserve_interrupted_checkpoint_for_explicit_resume(&workspace);
|
||||
|
||||
assert_eq!(recovered, None, "nothing is recovered over a live session");
|
||||
assert!(legacy.exists(), "the legacy slot is not consumed");
|
||||
assert_eq!(
|
||||
std::fs::read(&document).expect("document"),
|
||||
document_before,
|
||||
"the live document is not overwritten"
|
||||
);
|
||||
assert_eq!(
|
||||
checkpoint(),
|
||||
checkpoint_before,
|
||||
"the live checkpoint is not replaced by the legacy slot"
|
||||
);
|
||||
assert!(
|
||||
!crate::session_manager::is_live_session(&session_id),
|
||||
"no claim was taken"
|
||||
);
|
||||
drop(lease);
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn continue_recovers_legacy_checkpoint_and_migrates_it() {
|
||||
let _guard = crate::test_support::lock_test_env();
|
||||
|
||||
@@ -1222,6 +1222,17 @@ pub type RetryCallback = Box<dyn Fn(&LlmError, u32, Duration) + Send + Sync>;
|
||||
/// * `Ok(T)` - The successful result from the operation
|
||||
/// * `Err(RetryError)` - All retries exhausted or non-retryable error encountered
|
||||
///
|
||||
/// # Known limitation: ambiguous failures are replayed
|
||||
///
|
||||
/// A timeout or connection loss after the request was written is retried
|
||||
/// like a connect failure, so a provider that already accepted the first
|
||||
/// attempt may bill a second completion. Every caller sends model inference
|
||||
/// (messages, FIM, translation, speech, provider web search): a replay costs
|
||||
/// compute but has no external side effect, and the provider APIs used here
|
||||
/// expose no idempotency key for these requests that could dedupe it. An
|
||||
/// operation with an external side effect must not be retried through this
|
||||
/// helper without an idempotency key the server honors.
|
||||
///
|
||||
/// # Example
|
||||
///
|
||||
/// ```ignore
|
||||
|
||||
@@ -826,13 +826,17 @@ impl McpOAuthRuntime {
|
||||
/// the rotated token while keeping our dead grant would make the next
|
||||
/// `invalid_grant` compare an "unchanged" store and delete the newer
|
||||
/// valid credential.
|
||||
///
|
||||
/// The comparison and the delete are one step under the secret store's
|
||||
/// entry lock (the same lock every save takes), so a login that lands
|
||||
/// between them is never the credential that gets deleted.
|
||||
async fn clear_stored_tokens(&self, reason: &str) -> Result<()> {
|
||||
let held = { self.inner.last_tokens.lock().await.take() };
|
||||
let Some(held) = held else {
|
||||
return Ok(());
|
||||
};
|
||||
match load_oauth_tokens(&self.inner.server_name, &self.inner.url)? {
|
||||
Some(stored) if stored != held => {
|
||||
match delete_oauth_tokens_if_held(&self.inner.server_name, &self.inner.url, &held)? {
|
||||
Some(stored) => {
|
||||
tracing::debug!(
|
||||
target: "mcp",
|
||||
server = %self.inner.server_name,
|
||||
@@ -845,8 +849,7 @@ impl McpOAuthRuntime {
|
||||
*self.inner.last_tokens.lock().await = Some(stored);
|
||||
*self.inner.rejection.lock().await = None;
|
||||
}
|
||||
_ => {
|
||||
delete_oauth_tokens(&self.inner.server_name, &self.inner.url)?;
|
||||
None => {
|
||||
*self.inner.rejection.lock().await = Some(reason.to_string());
|
||||
}
|
||||
}
|
||||
@@ -863,8 +866,9 @@ impl McpOAuthRuntime {
|
||||
};
|
||||
let Some(credentials) = credentials else {
|
||||
let mut last = self.inner.last_tokens.lock().await;
|
||||
if last.take().is_some() {
|
||||
delete_oauth_tokens(&self.inner.server_name, &self.inner.url)?;
|
||||
if let Some(previous) = last.take() {
|
||||
// Only our own credential goes; a newer login stays.
|
||||
delete_oauth_tokens_if_held(&self.inner.server_name, &self.inner.url, &previous)?;
|
||||
}
|
||||
return Ok(());
|
||||
};
|
||||
@@ -1444,9 +1448,42 @@ pub(crate) fn load_oauth_tokens(
|
||||
else {
|
||||
return Ok(None);
|
||||
};
|
||||
let mut tokens = parse_stored_oauth_tokens(&serialized, server_name)?;
|
||||
decode_stored_oauth_tokens(&serialized, server_name).map(Some)
|
||||
}
|
||||
|
||||
fn decode_stored_oauth_tokens(serialized: &str, server_name: &str) -> Result<StoredMcpOAuthTokens> {
|
||||
let mut tokens = parse_stored_oauth_tokens(serialized, server_name)?;
|
||||
refresh_expires_in_from_timestamp(&mut tokens);
|
||||
Ok(Some(tokens))
|
||||
Ok(tokens)
|
||||
}
|
||||
|
||||
/// Delete the stored credential only while it is still exactly `held`,
|
||||
/// comparing and deleting under the store's entry lock. Returns the different
|
||||
/// credential found in its place (left untouched), or `None` when the entry
|
||||
/// was `held` and is now gone, or was already absent. An unreadable entry is
|
||||
/// an error and is left in place, as a plain load would report it.
|
||||
fn delete_oauth_tokens_if_held(
|
||||
server_name: &str,
|
||||
url: &str,
|
||||
held: &StoredMcpOAuthTokens,
|
||||
) -> Result<Option<StoredMcpOAuthTokens>> {
|
||||
let secrets = codewhale_secrets::Secrets::auto_detect();
|
||||
let key = store_key(server_name, url);
|
||||
secrets
|
||||
.with_entry_transaction(&key, |current| {
|
||||
let Some(serialized) = current.as_deref() else {
|
||||
return Ok(Ok(None));
|
||||
};
|
||||
Ok(match decode_stored_oauth_tokens(serialized, server_name) {
|
||||
Ok(stored) if stored == *held => {
|
||||
*current = None;
|
||||
Ok(None)
|
||||
}
|
||||
Ok(stored) => Ok(Some(stored)),
|
||||
Err(error) => Err(error),
|
||||
})
|
||||
})
|
||||
.with_context(|| format!("clearing the MCP OAuth token for '{server_name}'"))?
|
||||
}
|
||||
|
||||
fn parse_stored_oauth_tokens(serialized: &str, server_name: &str) -> Result<StoredMcpOAuthTokens> {
|
||||
|
||||
+58
-18
@@ -28,6 +28,13 @@ pub(super) struct StdioTransport {
|
||||
pub(super) authority_cancel_watch: Option<tokio::task::JoinHandle<()>>,
|
||||
/// Holds reviewed executable/script handles for the process lifetime.
|
||||
pub(super) _reviewed_launch: Option<super::ReviewedStdioLaunch>,
|
||||
/// Everything the server started: its own process group on Unix, a Job
|
||||
/// Object on Windows. Termination signals the whole tree, so a server's
|
||||
/// grandchild (an `npx` wrapper's node, a shell's background job) dies
|
||||
/// with it on shutdown and on plugin revocation. The last clone's drop
|
||||
/// kills whatever is left. `None` only for a transport built around an
|
||||
/// uncontained child (tests).
|
||||
pub(super) process_tree: Option<Arc<crate::process_tree::ProcessTree>>,
|
||||
}
|
||||
|
||||
/// How long `StdioTransport::shutdown` waits for the child to exit on SIGTERM
|
||||
@@ -172,6 +179,9 @@ impl StdioTransport {
|
||||
child_env::string_map_env(&expanded_env),
|
||||
);
|
||||
}
|
||||
// Lead a process group of its own so teardown can reach descendants.
|
||||
#[cfg(unix)]
|
||||
cmd.process_group(0);
|
||||
|
||||
let mut child = cmd.spawn().map_err(|error| {
|
||||
let message = if error.kind() == std::io::ErrorKind::NotFound
|
||||
@@ -195,6 +205,16 @@ impl StdioTransport {
|
||||
anyhow::Error::new(error).context(message)
|
||||
})?;
|
||||
|
||||
let process_tree = match crate::process_tree::ProcessTree::attach_tokio(&child) {
|
||||
Ok(tree) => Arc::new(tree),
|
||||
Err(error) => {
|
||||
let _ = child.start_kill();
|
||||
return Err(anyhow::Error::new(error).context(format!(
|
||||
"MCP server {server_name} could not be contained with its child processes"
|
||||
)));
|
||||
}
|
||||
};
|
||||
|
||||
let stdin = child.stdin.take().context("Failed to get MCP stdin")?;
|
||||
let stdout = child.stdout.take().context("Failed to get MCP stdout")?;
|
||||
let stderr = child.stderr.take().context("Failed to get MCP stderr")?;
|
||||
@@ -218,9 +238,10 @@ impl StdioTransport {
|
||||
let child = Arc::new(TokioMutex::new(child));
|
||||
let authority_cancel_watch = config.reviewed_plugin.as_ref().map(|_| {
|
||||
let watched_child = Arc::clone(&child);
|
||||
let watched_tree = Arc::clone(&process_tree);
|
||||
tokio::spawn(async move {
|
||||
cancel_token.cancelled().await;
|
||||
terminate_child_for_authority_change(&watched_child).await;
|
||||
terminate_child_for_authority_change(&watched_child, &watched_tree).await;
|
||||
})
|
||||
});
|
||||
|
||||
@@ -232,6 +253,7 @@ impl StdioTransport {
|
||||
stderr_tail,
|
||||
authority_cancel_watch,
|
||||
_reviewed_launch: reviewed_launch,
|
||||
process_tree: Some(process_tree),
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -315,19 +337,22 @@ async fn format_stderr_context(tail: &StderrTail) -> Option<String> {
|
||||
))
|
||||
}
|
||||
|
||||
/// Best-effort SIGTERM. On Unix uses `libc::kill`; on Windows there's no
|
||||
/// equivalent so we let `kill_on_drop` (TerminateProcess) handle it via the
|
||||
/// subsequent Drop. Returns whether a signal was actually sent.
|
||||
fn send_sigterm(child: &Child) -> bool {
|
||||
/// Best-effort SIGTERM. On Unix uses `libc::kill`, addressed to the child's
|
||||
/// whole process group when it leads one (`contained`); on Windows there's no
|
||||
/// equivalent so we let `kill_on_drop` (TerminateProcess) and the Job Object
|
||||
/// handle it. Returns whether a signal was actually sent.
|
||||
fn send_sigterm(child: &Child, contained: bool) -> bool {
|
||||
#[cfg(unix)]
|
||||
{
|
||||
if let Some(pid) = child.id() {
|
||||
// SAFETY: pid was just obtained from `child.id()`. `libc::kill`
|
||||
// with `SIGTERM` is async-signal-safe and never observes invalid
|
||||
// memory. Worst case (pid wrap / process already gone) returns
|
||||
// ESRCH, which we deliberately ignore.
|
||||
let pid = pid as i32;
|
||||
let target = if contained { -pid } else { pid };
|
||||
// SAFETY: pid was just obtained from `child.id()` of an unreaped
|
||||
// child, so neither it nor the group it leads can have been
|
||||
// recycled. `libc::kill` with `SIGTERM` is async-signal-safe and
|
||||
// never observes invalid memory. ESRCH is deliberately ignored.
|
||||
unsafe {
|
||||
let _ = libc::kill(pid as i32, libc::SIGTERM);
|
||||
let _ = libc::kill(target, libc::SIGTERM);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
@@ -335,27 +360,34 @@ fn send_sigterm(child: &Child) -> bool {
|
||||
}
|
||||
#[cfg(not(unix))]
|
||||
{
|
||||
let _ = child;
|
||||
let _ = (child, contained);
|
||||
false
|
||||
}
|
||||
}
|
||||
|
||||
async fn terminate_child_for_authority_change(child: &Arc<TokioMutex<Child>>) {
|
||||
async fn terminate_child_for_authority_change(
|
||||
child: &Arc<TokioMutex<Child>>,
|
||||
tree: &crate::process_tree::ProcessTree,
|
||||
) {
|
||||
let mut child = child.lock().await;
|
||||
terminate_child(&mut child).await;
|
||||
terminate_child(&mut child, Some(tree)).await;
|
||||
}
|
||||
|
||||
async fn terminate_child(child: &mut Child) {
|
||||
async fn terminate_child(child: &mut Child, tree: Option<&crate::process_tree::ProcessTree>) {
|
||||
// Reap an already-exited child before resolving its PID. Until it is
|
||||
// reaped, the OS cannot recycle that identity; after it is reaped there is
|
||||
// nothing left to signal. This avoids a PID-only watcher ever targeting an
|
||||
// unrelated process after rapid PID reuse.
|
||||
if child.try_wait().is_ok_and(|status| status.is_some()) {
|
||||
// The server is gone, but what it started may not be.
|
||||
if let Some(tree) = tree {
|
||||
let _ = tree.kill();
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
send_sigterm(child);
|
||||
send_sigterm(child, tree.is_some());
|
||||
|
||||
#[cfg(not(unix))]
|
||||
let _ = child.start_kill();
|
||||
@@ -369,6 +401,10 @@ async fn terminate_child(child: &mut Child) {
|
||||
let _ = child.wait().await;
|
||||
}
|
||||
}
|
||||
// Descendants that outlived the grace (or ignored SIGTERM) go now.
|
||||
if let Some(tree) = tree {
|
||||
let _ = tree.kill();
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
@@ -450,7 +486,7 @@ impl McpTransport for StdioTransport {
|
||||
/// then force termination and reap the child as the backstop.
|
||||
async fn shutdown(&mut self) {
|
||||
let mut child = self.child.lock().await;
|
||||
terminate_child(&mut child).await;
|
||||
terminate_child(&mut child, self.process_tree.as_deref()).await;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -466,18 +502,22 @@ impl Drop for StdioTransport {
|
||||
if let Ok(runtime) = tokio::runtime::Handle::try_current() {
|
||||
let child = Arc::clone(&self.child);
|
||||
let reviewed_launch = self._reviewed_launch.take();
|
||||
// The task owns the tree so it is not killed before the grace.
|
||||
let tree = self.process_tree.take();
|
||||
runtime.spawn(async move {
|
||||
let _reviewed_launch = reviewed_launch;
|
||||
let mut child = child.lock().await;
|
||||
terminate_child(&mut child).await;
|
||||
terminate_child(&mut child, tree.as_deref()).await;
|
||||
});
|
||||
return;
|
||||
}
|
||||
if let Ok(mut child) = self.child.try_lock()
|
||||
&& !child.try_wait().is_ok_and(|status| status.is_some())
|
||||
{
|
||||
send_sigterm(&child);
|
||||
send_sigterm(&child, self.process_tree.is_some());
|
||||
}
|
||||
// No runtime: dropping the tree below SIGKILLs the group / closes the
|
||||
// job, the same backstop `kill_on_drop` gives the direct child.
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1946,6 +1946,48 @@ async fn plugin_stdio_does_not_surface_reviewed_child_stderr() {
|
||||
assert!(!error.contains("ARBITRARY_PLUGIN_CREDENTIAL"));
|
||||
}
|
||||
|
||||
/// Shutdown reaches what the server started, not only the server: a
|
||||
/// background grandchild (an `npx` wrapper's node, a shell job) must not
|
||||
/// survive the transport.
|
||||
#[cfg(unix)]
|
||||
#[tokio::test]
|
||||
async fn stdio_shutdown_also_terminates_the_servers_grandchildren() {
|
||||
let mut config = test_server_config();
|
||||
config.command = Some("sh".to_string());
|
||||
config.args = vec!["-c".to_string(), "sleep 300 & echo $!; wait".to_string()];
|
||||
let mut transport = StdioTransport::spawn(
|
||||
"grandparent",
|
||||
"sh",
|
||||
&config,
|
||||
tokio_util::sync::CancellationToken::new(),
|
||||
)
|
||||
.unwrap();
|
||||
let line = tokio::time::timeout(Duration::from_secs(5), transport.recv())
|
||||
.await
|
||||
.expect("grandchild pid line")
|
||||
.unwrap();
|
||||
let grandchild: i32 = String::from_utf8(line).unwrap().trim().parse().unwrap();
|
||||
|
||||
transport.shutdown().await;
|
||||
|
||||
let deadline = tokio::time::Instant::now() + Duration::from_secs(3);
|
||||
loop {
|
||||
// SAFETY: signal 0 only probes whether the pid still exists.
|
||||
if unsafe { libc::kill(grandchild, 0) } != 0 {
|
||||
break;
|
||||
}
|
||||
if tokio::time::Instant::now() >= deadline {
|
||||
// Do not leak the sleeper past a failing run.
|
||||
// SAFETY: plain kill(2) of the pid this test started.
|
||||
unsafe {
|
||||
libc::kill(grandchild, libc::SIGKILL);
|
||||
}
|
||||
panic!("the MCP server's grandchild outlived the transport");
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(20)).await;
|
||||
}
|
||||
}
|
||||
|
||||
/// #6187: a crashed stdio child must stop reading as "ready" before any
|
||||
/// call is in flight — `is_ready` probes the child, so the pool rebuilds
|
||||
/// the connection on the next use instead of handing the dead transport
|
||||
@@ -5120,6 +5162,7 @@ async fn stdio_transport_shutdown_terminates_child() {
|
||||
stderr_tail: StderrTail::new(),
|
||||
authority_cancel_watch: None,
|
||||
_reviewed_launch: None,
|
||||
process_tree: None,
|
||||
};
|
||||
|
||||
// shutdown() should send SIGTERM and complete within the grace window.
|
||||
@@ -5249,6 +5292,7 @@ async fn stdio_transport_recv_error_includes_stderr_tail() {
|
||||
stderr_tail,
|
||||
authority_cancel_watch: None,
|
||||
_reviewed_launch: None,
|
||||
process_tree: None,
|
||||
};
|
||||
|
||||
// Give the subprocess time to write its stderr line and exit.
|
||||
|
||||
@@ -64,6 +64,9 @@ pub(crate) enum AutoRouterSetupIssue {
|
||||
Incomplete,
|
||||
/// The route is complete but its credential is missing.
|
||||
MissingKey,
|
||||
/// `thinking` is not a reasoning tier; the classifier call would carry
|
||||
/// an effort the provider rejects or silently reinterprets.
|
||||
InvalidThinking,
|
||||
}
|
||||
|
||||
impl AutoRouterSetupIssue {
|
||||
@@ -76,6 +79,9 @@ impl AutoRouterSetupIssue {
|
||||
}
|
||||
Self::Incomplete => "[auto.router] needs a known provider and a model",
|
||||
Self::MissingKey => "no API key for the router route",
|
||||
Self::InvalidThinking => {
|
||||
"[auto.router] thinking must be auto, off, minimal, low, medium, high, xhigh, ultra, or max"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -322,20 +328,28 @@ impl ModelInventory {
|
||||
router_setup_issue = Some(AutoRouterSetupIssue::UnknownKind);
|
||||
None
|
||||
}
|
||||
Some(AutoRouterKind::Chat) => router_provider_setting
|
||||
.and_then(ApiProvider::parse)
|
||||
.zip(router_model_setting)
|
||||
.map(|(provider, model)| {
|
||||
(
|
||||
provider,
|
||||
model.to_string(),
|
||||
router_table
|
||||
.and_then(|router| router.thinking.as_deref())
|
||||
.map(str::trim)
|
||||
.filter(|t| !t.is_empty())
|
||||
.map(str::to_string),
|
||||
)
|
||||
}),
|
||||
Some(AutoRouterKind::Chat) => {
|
||||
let thinking = router_table
|
||||
.and_then(|router| router.thinking.as_deref())
|
||||
.map(str::trim)
|
||||
.filter(|t| !t.is_empty());
|
||||
// A typo here would otherwise ride every classifier request
|
||||
// as an effort string the provider rejects, failing Auto back
|
||||
// to the local fallback on every turn.
|
||||
if thinking.is_some_and(|thinking| {
|
||||
crate::reasoning_preference::ReasoningEffort::parse_strict(thinking).is_err()
|
||||
}) {
|
||||
router_setup_issue = Some(AutoRouterSetupIssue::InvalidThinking);
|
||||
None
|
||||
} else {
|
||||
router_provider_setting
|
||||
.and_then(ApiProvider::parse)
|
||||
.zip(router_model_setting)
|
||||
.map(|(provider, model)| {
|
||||
(provider, model.to_string(), thinking.map(str::to_string))
|
||||
})
|
||||
}
|
||||
}
|
||||
Some(AutoRouterKind::Decision) => {
|
||||
match router_provider_setting.map(DecisionRouterRoute::parse) {
|
||||
Some(None) => {
|
||||
@@ -1507,6 +1521,30 @@ mod decision_router_inventory_tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_chat_router_with_an_unknown_thinking_tier_is_not_configured() {
|
||||
let _env = hermetic();
|
||||
let chat = |thinking: &str| crate::config::AutoRouterConfig {
|
||||
kind: Some("chat".to_string()),
|
||||
provider: Some("openrouter".to_string()),
|
||||
model: Some("openai/gpt-5-mini".to_string()),
|
||||
thinking: Some(thinking.to_string()),
|
||||
..Default::default()
|
||||
};
|
||||
let typo = ModelInventory::from_config(&with_router(chat("hgih"), true));
|
||||
assert!(!typo.router_configured);
|
||||
assert!(!typo.router_available);
|
||||
assert_eq!(
|
||||
typo.router_setup_issue,
|
||||
Some(AutoRouterSetupIssue::InvalidThinking)
|
||||
);
|
||||
|
||||
let valid = ModelInventory::from_config(&with_router(chat("low"), true));
|
||||
assert!(valid.router_available);
|
||||
assert_eq!(valid.router_setup_issue, None);
|
||||
assert_eq!(valid.router_thinking.as_deref(), Some("low"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_router_table_with_only_tuning_keys_is_declared_and_incomplete() {
|
||||
let _env = hermetic();
|
||||
|
||||
@@ -46,6 +46,14 @@ pub const ENV_MODELS_DEV_PATH: &str = "CODEWHALE_MODELS_DEV_PATH";
|
||||
pub const ENV_DISABLE_FETCH: &str = "CODEWHALE_DISABLE_MODELS_DEV_FETCH";
|
||||
|
||||
const CACHE_SCHEMA_VERSION: u32 = 1;
|
||||
/// Largest catalog body accepted from the network, an override file, or the
|
||||
/// disk cache. The public catalog is a few MiB; anything past this is refused
|
||||
/// instead of being read whole into memory.
|
||||
const MAX_CATALOG_BYTES: usize = 32 * 1024 * 1024;
|
||||
/// Clock skew tolerated on a cache timestamp. A `fetched_at` further in the
|
||||
/// future than this is not "age zero": it is untrustworthy, so the cache is
|
||||
/// published as stale and the next refresh fetches.
|
||||
const MAX_CACHE_CLOCK_SKEW_SECS: u64 = 5 * 60;
|
||||
|
||||
/// Provenance / freshness of the Models.dev live layer for UI chips (#4187).
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
|
||||
@@ -227,11 +235,16 @@ pub fn maybe_load_persisted_cache() {
|
||||
let Some(cache) = load_cache_file(&path) else {
|
||||
return;
|
||||
};
|
||||
let age = now_unix().saturating_sub(cache.fetched_at);
|
||||
let freshness = if age > DEFAULT_MODELS_DEV_TTL_SECS {
|
||||
ModelsDevFreshness::Stale
|
||||
} else {
|
||||
// Live only when the cache is recent by a trustworthy clock *and* came
|
||||
// from the source this process would fetch now. A cache written for
|
||||
// another mirror or override file still publishes (offline startups keep
|
||||
// rows) but as stale, so the next refresh replaces it.
|
||||
let freshness = if within_ttl(cache.fetched_at, now_unix())
|
||||
&& cache.source_fingerprint == current_source_fingerprint()
|
||||
{
|
||||
ModelsDevFreshness::Live
|
||||
} else {
|
||||
ModelsDevFreshness::Stale
|
||||
};
|
||||
if let Err(err) = publish_from_body(
|
||||
&cache.body,
|
||||
@@ -269,7 +282,7 @@ pub async fn refresh(force_network: bool) -> Result<usize, ModelsDevRefreshError
|
||||
if current.freshness == ModelsDevFreshness::Live
|
||||
&& current
|
||||
.fetched_at
|
||||
.is_some_and(|ts| now_unix().saturating_sub(ts) < DEFAULT_MODELS_DEV_TTL_SECS)
|
||||
.is_some_and(|ts| within_ttl(ts, now_unix()))
|
||||
{
|
||||
return Ok(current.offering_count);
|
||||
}
|
||||
@@ -302,6 +315,29 @@ pub async fn refresh(force_network: bool) -> Result<usize, ModelsDevRefreshError
|
||||
Ok(count)
|
||||
}
|
||||
|
||||
/// Whether a payload fetched at `fetched_at` is still inside the TTL at
|
||||
/// `now`. A timestamp beyond the tolerated clock skew is never fresh: a
|
||||
/// saturating age would read it as zero and suppress refresh until the clock
|
||||
/// caught up.
|
||||
fn within_ttl(fetched_at: u64, now: u64) -> bool {
|
||||
if fetched_at > now.saturating_add(MAX_CACHE_CLOCK_SKEW_SECS) {
|
||||
return false;
|
||||
}
|
||||
now.saturating_sub(fetched_at) <= DEFAULT_MODELS_DEV_TTL_SECS
|
||||
}
|
||||
|
||||
/// Fingerprint of the source a refresh would read right now: the override
|
||||
/// file when `CODEWHALE_MODELS_DEV_PATH` is set, else the catalog URL. Same
|
||||
/// derivation `refresh` stores in the cache.
|
||||
fn current_source_fingerprint() -> String {
|
||||
match std::env::var(ENV_MODELS_DEV_PATH) {
|
||||
Ok(path) if !path.trim().is_empty() => {
|
||||
base_url_fingerprint(&format!("file:{}", Path::new(path.trim()).display()))
|
||||
}
|
||||
_ => base_url_fingerprint(&resolve_catalog_url()),
|
||||
}
|
||||
}
|
||||
|
||||
/// Best-effort background refresh: never panics, never blocks callers.
|
||||
pub fn spawn_background_refresh() {
|
||||
if env_truthy(ENV_DISABLE_FETCH) && std::env::var(ENV_MODELS_DEV_PATH).is_err() {
|
||||
@@ -328,10 +364,9 @@ pub fn spawn_background_refresh() {
|
||||
}
|
||||
|
||||
async fn refresh_from_path(path: &Path) -> Result<usize, ModelsDevRefreshError> {
|
||||
let body = match tokio::fs::read_to_string(path).await {
|
||||
let body = match read_catalog_file(path).await {
|
||||
Ok(body) => body,
|
||||
Err(err) => {
|
||||
let mapped = ModelsDevRefreshError::Io(err.to_string());
|
||||
Err(mapped) => {
|
||||
mark_failed(mapped.clone());
|
||||
return Err(mapped);
|
||||
}
|
||||
@@ -375,10 +410,56 @@ async fn fetch_catalog_body(url: &str) -> Result<String, ModelsDevRefreshError>
|
||||
return Err(ModelsDevRefreshError::HttpStatus(status.as_u16()));
|
||||
}
|
||||
|
||||
response
|
||||
.text()
|
||||
read_catalog_response(response, MAX_CATALOG_BYTES).await
|
||||
}
|
||||
|
||||
fn oversized_catalog(limit: usize) -> ModelsDevRefreshError {
|
||||
ModelsDevRefreshError::InvalidResponse(format!("catalog exceeds {limit} bytes"))
|
||||
}
|
||||
|
||||
/// Read a catalog response body without ever holding more than `limit` bytes.
|
||||
async fn read_catalog_response(
|
||||
mut response: reqwest::Response,
|
||||
limit: usize,
|
||||
) -> Result<String, ModelsDevRefreshError> {
|
||||
if response
|
||||
.content_length()
|
||||
.is_some_and(|length| length > limit as u64)
|
||||
{
|
||||
return Err(oversized_catalog(limit));
|
||||
}
|
||||
let mut bytes = Vec::new();
|
||||
while let Some(chunk) = response
|
||||
.chunk()
|
||||
.await
|
||||
.map_err(|err| ModelsDevRefreshError::Network(err.to_string()))
|
||||
.map_err(|err| ModelsDevRefreshError::Network(err.to_string()))?
|
||||
{
|
||||
if bytes.len().saturating_add(chunk.len()) > limit {
|
||||
return Err(oversized_catalog(limit));
|
||||
}
|
||||
bytes.extend_from_slice(&chunk);
|
||||
}
|
||||
String::from_utf8(bytes)
|
||||
.map_err(|_| ModelsDevRefreshError::InvalidResponse("catalog is not UTF-8".into()))
|
||||
}
|
||||
|
||||
/// Read an override catalog file, refusing one larger than the catalog limit.
|
||||
async fn read_catalog_file(path: &Path) -> Result<String, ModelsDevRefreshError> {
|
||||
use tokio::io::AsyncReadExt as _;
|
||||
|
||||
let file = tokio::fs::File::open(path)
|
||||
.await
|
||||
.map_err(|err| ModelsDevRefreshError::Io(err.to_string()))?;
|
||||
let mut bytes = Vec::new();
|
||||
file.take(MAX_CATALOG_BYTES as u64 + 1)
|
||||
.read_to_end(&mut bytes)
|
||||
.await
|
||||
.map_err(|err| ModelsDevRefreshError::Io(err.to_string()))?;
|
||||
if bytes.len() > MAX_CATALOG_BYTES {
|
||||
return Err(oversized_catalog(MAX_CATALOG_BYTES));
|
||||
}
|
||||
String::from_utf8(bytes)
|
||||
.map_err(|_| ModelsDevRefreshError::InvalidResponse("catalog is not UTF-8".into()))
|
||||
}
|
||||
|
||||
fn publish_from_body(
|
||||
@@ -430,7 +511,20 @@ fn mark_failed(err: ModelsDevRefreshError) {
|
||||
/// with an escaped body) formats. Returns metadata and the *unescaped* body
|
||||
/// without copying it in the v2 path.
|
||||
fn load_cache_file(path: &Path) -> Option<PersistedModelsDevCache> {
|
||||
let bytes = std::fs::read(path).ok()?;
|
||||
use std::io::Read as _;
|
||||
|
||||
// Header line plus body; a larger file is not ours to trust or to read
|
||||
// whole, so the bundled rows stand instead.
|
||||
const MAX_CACHE_FILE_BYTES: u64 = MAX_CATALOG_BYTES as u64 + 64 * 1024;
|
||||
let mut bytes = Vec::new();
|
||||
std::fs::File::open(path)
|
||||
.ok()?
|
||||
.take(MAX_CACHE_FILE_BYTES + 1)
|
||||
.read_to_end(&mut bytes)
|
||||
.ok()?;
|
||||
if bytes.len() as u64 > MAX_CACHE_FILE_BYTES {
|
||||
return None;
|
||||
}
|
||||
// v2: single-line JSON header terminated by a newline, then the verbatim
|
||||
// catalog body. One small parse, zero body copies.
|
||||
if bytes.first() == Some(&b'{') && bytes.contains(&b'\n') {
|
||||
@@ -735,6 +829,80 @@ mod tests {
|
||||
clear_live_snapshot();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_future_or_foreign_cache_is_never_live() {
|
||||
let now = 2_000_000_000;
|
||||
assert!(within_ttl(now - 60, now));
|
||||
assert!(within_ttl(now + 30, now), "small skew is tolerated");
|
||||
assert!(
|
||||
!within_ttl(now + 3 * 24 * 60 * 60, now),
|
||||
"a future timestamp is not age zero"
|
||||
);
|
||||
assert!(!within_ttl(now - DEFAULT_MODELS_DEV_TTL_SECS - 1, now));
|
||||
|
||||
let _lock = lock_test_env();
|
||||
let _live = lock_live_snapshot();
|
||||
clear_live_snapshot();
|
||||
let dir = tempfile::tempdir().expect("tempdir");
|
||||
let home = dir.path().join("home");
|
||||
let _home = EnvVarGuard::set("CODEWHALE_HOME", &home);
|
||||
let _path = EnvVarGuard::remove(ENV_MODELS_DEV_PATH);
|
||||
let _url = EnvVarGuard::remove(ENV_MODELS_DEV_URL);
|
||||
let cache_dir = home.join("catalog");
|
||||
std::fs::create_dir_all(&cache_dir).expect("mkdir");
|
||||
let cache = cache_dir.join(CACHE_FILE);
|
||||
|
||||
// Fresh by the clock, but fetched from another source.
|
||||
let foreign = PersistedModelsDevCache {
|
||||
schema_version: CACHE_SCHEMA_VERSION,
|
||||
fetched_at: now_unix(),
|
||||
source_fingerprint: base_url_fingerprint("https://mirror.example/catalog.json"),
|
||||
source_label: "https://mirror.example/catalog.json".into(),
|
||||
body: FIXTURE.into(),
|
||||
};
|
||||
save_cache_file(&cache, &foreign).expect("save");
|
||||
maybe_load_persisted_cache();
|
||||
assert_eq!(status().freshness, ModelsDevFreshness::Stale);
|
||||
|
||||
// From the current source, but stamped far in the future.
|
||||
let future = PersistedModelsDevCache {
|
||||
fetched_at: now_unix() + 30 * 24 * 60 * 60,
|
||||
source_fingerprint: base_url_fingerprint(MODELS_DEV_CATALOG_URL),
|
||||
source_label: MODELS_DEV_CATALOG_URL.into(),
|
||||
..foreign.clone()
|
||||
};
|
||||
save_cache_file(&cache, &future).expect("save");
|
||||
maybe_load_persisted_cache();
|
||||
assert_eq!(status().freshness, ModelsDevFreshness::Stale);
|
||||
|
||||
// The same cache stamped now is live.
|
||||
let current = PersistedModelsDevCache {
|
||||
fetched_at: now_unix(),
|
||||
..future
|
||||
};
|
||||
save_cache_file(&cache, ¤t).expect("save");
|
||||
maybe_load_persisted_cache();
|
||||
assert_eq!(status().freshness, ModelsDevFreshness::Live);
|
||||
clear_live_snapshot();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_oversized_override_catalog_is_refused() {
|
||||
let dir = tempfile::tempdir().expect("tempdir");
|
||||
let path = dir.path().join("huge.json");
|
||||
let file = std::fs::File::create(&path).expect("create");
|
||||
file.set_len(MAX_CATALOG_BYTES as u64 + 1)
|
||||
.expect("sparse size");
|
||||
let rt = tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.build()
|
||||
.expect("runtime");
|
||||
let error = rt
|
||||
.block_on(read_catalog_file(&path))
|
||||
.expect_err("oversized catalog must be refused");
|
||||
assert!(error.to_string().contains("exceeds"), "{error}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn network_failure_keeps_prior_rows_and_marks_failed() {
|
||||
let _lock = lock_test_env();
|
||||
|
||||
@@ -165,6 +165,10 @@ static MERGED_CACHE: RwLock<Option<MergedCacheEntry>> = RwLock::new(None);
|
||||
/// 600+ row OpenRouter catalog for every route candidate.
|
||||
static RUNTIME_RESOLVER_CACHE: RwLock<BTreeMap<String, RuntimeResolverCacheEntry>> =
|
||||
RwLock::new(BTreeMap::new());
|
||||
/// Most distinct endpoints the resolver cache holds at once. Entries from an
|
||||
/// older catalog generation can never hit again and are dropped on insert;
|
||||
/// this caps the endpoints of the current generation.
|
||||
const MAX_RUNTIME_RESOLVER_CACHE_ENTRIES: usize = 64;
|
||||
|
||||
#[derive(Clone)]
|
||||
struct RuntimeResolverCacheEntry {
|
||||
@@ -408,27 +412,61 @@ pub fn live_catalog_origin(provider: ApiProvider, wire_model_id: &str) -> Option
|
||||
let Ok(guard) = LIVE_SNAPSHOT.read() else {
|
||||
return None;
|
||||
};
|
||||
let matches = |row: &CatalogOffering| {
|
||||
row.provider.eq_ignore_ascii_case(catalog_id)
|
||||
&& row.wire_model_id.eq_ignore_ascii_case(needle)
|
||||
};
|
||||
if guard
|
||||
.per_provider
|
||||
.get(&owner)
|
||||
.is_some_and(|snap| snap.offerings.iter().any(matches))
|
||||
{
|
||||
return Some(LiveSource::PerProvider);
|
||||
}
|
||||
if guard
|
||||
.models_dev
|
||||
.as_ref()
|
||||
.is_some_and(|snap| snap.offerings.iter().any(|row| matches(row)))
|
||||
{
|
||||
return Some(LiveSource::ModelsDev);
|
||||
// Wire ids are opaque and case-sensitive: an exact row in either
|
||||
// partition decides before a case-folded one does.
|
||||
for exact in [true, false] {
|
||||
let matches = |row: &CatalogOffering| {
|
||||
row.provider.eq_ignore_ascii_case(catalog_id)
|
||||
&& if exact {
|
||||
row.wire_model_id == needle
|
||||
} else {
|
||||
row.wire_model_id.eq_ignore_ascii_case(needle)
|
||||
}
|
||||
};
|
||||
if guard
|
||||
.per_provider
|
||||
.get(&owner)
|
||||
.is_some_and(|snap| snap.offerings.iter().any(matches))
|
||||
{
|
||||
return Some(LiveSource::PerProvider);
|
||||
}
|
||||
if guard
|
||||
.models_dev
|
||||
.as_ref()
|
||||
.is_some_and(|snap| snap.offerings.iter().any(|row| matches(row)))
|
||||
{
|
||||
return Some(LiveSource::ModelsDev);
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// The row whose wire id is exactly `needle`, else the one wire id equal to
|
||||
/// it ignoring ASCII case. Wire ids are opaque and case-sensitive, so the
|
||||
/// fallback only forgives a case slip when it cannot choose between two
|
||||
/// distinct ids; otherwise it answers nothing rather than another model's
|
||||
/// metadata.
|
||||
fn find_wire_model<'a>(
|
||||
rows: impl IntoIterator<Item = &'a CatalogOffering>,
|
||||
needle: &str,
|
||||
) -> Option<&'a CatalogOffering> {
|
||||
let mut folded: Option<&'a CatalogOffering> = None;
|
||||
let mut ambiguous = false;
|
||||
for row in rows {
|
||||
if row.wire_model_id == needle {
|
||||
return Some(row);
|
||||
}
|
||||
if row.wire_model_id.eq_ignore_ascii_case(needle) {
|
||||
match folded {
|
||||
None => folded = Some(row),
|
||||
Some(first) if first.wire_model_id != row.wire_model_id => ambiguous = true,
|
||||
Some(_) => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
folded.filter(|_| !ambiguous)
|
||||
}
|
||||
|
||||
/// Serialize tests that mutate the process-wide live snapshot.
|
||||
///
|
||||
/// Lock ordering: this takes the test env barrier FIRST (skipped when the
|
||||
@@ -938,6 +976,12 @@ pub(crate) fn runtime_catalog_resolver_for_identity(
|
||||
|
||||
let resolver = RouteResolver::from_offerings(route_offerings.into_values().collect());
|
||||
if let Ok(mut cache) = RUNTIME_RESOLVER_CACHE.write() {
|
||||
cache.retain(|_, entry| {
|
||||
entry.generation == generation && entry.cloud_generation == cloud_generation
|
||||
});
|
||||
if cache.len() >= MAX_RUNTIME_RESOLVER_CACHE_ENTRIES && !cache.contains_key(&cache_key) {
|
||||
cache.pop_first();
|
||||
}
|
||||
cache.insert(
|
||||
cache_key,
|
||||
RuntimeResolverCacheEntry {
|
||||
@@ -1110,14 +1154,14 @@ pub fn catalog_offering_for_model_identity(
|
||||
return None;
|
||||
}
|
||||
if provider == ApiProvider::Custom {
|
||||
return exact_custom_offerings(catalog_id.as_ref())
|
||||
.into_iter()
|
||||
.find(|row| row.wire_model_id.eq_ignore_ascii_case(needle));
|
||||
let rows = exact_custom_offerings(catalog_id.as_ref());
|
||||
return find_wire_model(&rows, needle).cloned();
|
||||
}
|
||||
offerings_for_provider_identity(&merged_snapshot(), catalog_id.as_ref())
|
||||
.into_iter()
|
||||
.find(|row| row.wire_model_id.eq_ignore_ascii_case(needle))
|
||||
.cloned()
|
||||
find_wire_model(
|
||||
offerings_for_provider_identity(&merged_snapshot(), catalog_id.as_ref()),
|
||||
needle,
|
||||
)
|
||||
.cloned()
|
||||
}
|
||||
|
||||
/// Metadata from the exact route, without borrowing another endpoint's live facts.
|
||||
@@ -1252,11 +1296,11 @@ pub fn bundled_catalog_offering_for_model(
|
||||
if needle.is_empty() {
|
||||
return None;
|
||||
}
|
||||
bundled_snapshot()
|
||||
.offerings_for_provider(catalog_id)
|
||||
.into_iter()
|
||||
.find(|row| row.wire_model_id.eq_ignore_ascii_case(needle))
|
||||
.cloned()
|
||||
find_wire_model(
|
||||
bundled_snapshot().offerings_for_provider(catalog_id),
|
||||
needle,
|
||||
)
|
||||
.cloned()
|
||||
}
|
||||
|
||||
/// Count of merged-catalog models for one provider (catalog view / dashboard).
|
||||
@@ -1860,6 +1904,22 @@ mod tests {
|
||||
use crate::config::{DEFAULT_TOGETHER_FLASH_MODEL, DEFAULT_TOGETHER_MODEL};
|
||||
use codewhale_config::catalog::CatalogSource;
|
||||
|
||||
#[test]
|
||||
fn wire_model_lookup_is_exact_first_and_never_guesses_between_case_variants() {
|
||||
let row = |id: &str| CatalogOffering {
|
||||
provider: "p".to_string(),
|
||||
wire_model_id: id.to_string(),
|
||||
..Default::default()
|
||||
};
|
||||
let rows = vec![row("org/Model-A"), row("org/model-a"), row("org/Model-B")];
|
||||
let found =
|
||||
|needle: &str| find_wire_model(&rows, needle).map(|row| row.wire_model_id.clone());
|
||||
assert_eq!(found("org/model-a").as_deref(), Some("org/model-a"));
|
||||
assert_eq!(found("org/Model-A").as_deref(), Some("org/Model-A"));
|
||||
assert_eq!(found("ORG/MODEL-A"), None, "two distinct ids fold together");
|
||||
assert_eq!(found("org/model-b").as_deref(), Some("org/Model-B"));
|
||||
}
|
||||
|
||||
fn catalog_test_config(first_url: &str, second_url: &str) -> Config {
|
||||
use crate::config::{ProviderConfig, ProvidersConfig};
|
||||
Config {
|
||||
|
||||
@@ -3290,6 +3290,9 @@ async fn relay_worker(
|
||||
sync_tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
|
||||
let mut runtime_retry_delay = RUNTIME_UPLOAD_RETRY_INTERVAL;
|
||||
let mut runtime_retry_not_before = Instant::now();
|
||||
// One credential refresh per rejection streak (see
|
||||
// `RUNTIME_REFRESHED_CREDENTIAL_REFUSED`).
|
||||
let mut runtime_refreshed_since_accept = false;
|
||||
|
||||
loop {
|
||||
tokio::select! {
|
||||
@@ -3412,11 +3415,15 @@ async fn relay_worker(
|
||||
RuntimeFlushOutcome::Accepted { run_id, cursor } => {
|
||||
runtime_retry_delay = RUNTIME_UPLOAD_RETRY_INTERVAL;
|
||||
runtime_retry_not_before = Instant::now();
|
||||
runtime_refreshed_since_accept = false;
|
||||
event_tx
|
||||
.send(RemoteEvent::RuntimeCursor { run_id, cursor })
|
||||
.map_err(|_| "The terminal remote-control owner stopped.".to_string())?;
|
||||
}
|
||||
RuntimeFlushOutcome::AccessTokenExpired => {
|
||||
if std::mem::replace(&mut runtime_refreshed_since_accept, true) {
|
||||
return Err(RUNTIME_REFRESHED_CREDENTIAL_REFUSED.to_string());
|
||||
}
|
||||
refresh_enrollment_and_reconnect(
|
||||
&client,
|
||||
&mut enrollment,
|
||||
@@ -3694,6 +3701,12 @@ impl RuntimeTransportOutbox {
|
||||
}
|
||||
}
|
||||
|
||||
/// A runtime event refused (401/403) again right after a successful
|
||||
/// credential refresh is a terminal rejection, not an expired token: a
|
||||
/// revoked runner or a run this runner may not write. Refreshing again would
|
||||
/// loop at the upload retry interval forever.
|
||||
const RUNTIME_REFRESHED_CREDENTIAL_REFUSED: &str = "The remote-control server refused runtime events again with a freshly refreshed credential; stopping instead of retrying.";
|
||||
|
||||
/// Flushes every queued runtime envelope through the server-confirmed cursor
|
||||
/// before a stop may be acknowledged. Emits `RuntimeCursor` events so the
|
||||
/// controller compacts its journal as acknowledgements land. Failing to drain
|
||||
@@ -3710,6 +3723,7 @@ async fn drain_runtime_outbox_for_stop(
|
||||
deadline: Instant,
|
||||
) -> Result<(), String> {
|
||||
let mut delay = RUNTIME_UPLOAD_RETRY_INTERVAL;
|
||||
let mut refreshed_since_accept = false;
|
||||
while !outbox.events.is_empty() {
|
||||
if Instant::now() >= deadline {
|
||||
return Err(
|
||||
@@ -3721,6 +3735,7 @@ async fn drain_runtime_outbox_for_stop(
|
||||
RuntimeFlushOutcome::Idle => break,
|
||||
RuntimeFlushOutcome::Accepted { run_id, cursor } => {
|
||||
delay = RUNTIME_UPLOAD_RETRY_INTERVAL;
|
||||
refreshed_since_accept = false;
|
||||
let _ = event_tx.send(RemoteEvent::RuntimeCursor { run_id, cursor });
|
||||
}
|
||||
RuntimeFlushOutcome::Retryable => {
|
||||
@@ -3728,6 +3743,9 @@ async fn drain_runtime_outbox_for_stop(
|
||||
delay = delay.saturating_mul(2).min(RUNTIME_UPLOAD_MAX_BACKOFF);
|
||||
}
|
||||
RuntimeFlushOutcome::AccessTokenExpired => {
|
||||
if std::mem::replace(&mut refreshed_since_accept, true) {
|
||||
return Err(RUNTIME_REFRESHED_CREDENTIAL_REFUSED.to_string());
|
||||
}
|
||||
refresh_enrollment_and_reconnect(client, enrollment, runner_id, start, event_tx)
|
||||
.await?;
|
||||
}
|
||||
@@ -7938,6 +7956,84 @@ mod tests {
|
||||
assert_eq!(bodies[0], bodies[1], "retries must be byte-identical");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn stop_drain_stops_when_a_fresh_credential_is_refused_again() {
|
||||
let _env = crate::test_support::lock_test_env();
|
||||
let secrets_root = tempfile::tempdir().expect("isolated remote-control secrets");
|
||||
let _codewhale_home =
|
||||
crate::test_support::EnvVarGuard::set("CODEWHALE_HOME", secrets_root.path());
|
||||
let server = MockServer::start().await;
|
||||
// A terminal refusal: the runner may not write this run, and a new
|
||||
// access token does not change that.
|
||||
Mock::given(method("POST"))
|
||||
.and(path(
|
||||
"/api/local-runners/runner_fixture/runs/run_fixture/events",
|
||||
))
|
||||
.respond_with(ResponseTemplate::new(403))
|
||||
.expect(2)
|
||||
.mount(&server)
|
||||
.await;
|
||||
let access_token = crate::test_support::future_test_jwt(&"a".repeat(40));
|
||||
Mock::given(method("POST"))
|
||||
.and(path("/api/runner/enrollments/token"))
|
||||
.respond_with(ResponseTemplate::new(200).set_body_json(json!({
|
||||
"enrollment": {
|
||||
"id": "enrollment_fixture",
|
||||
"userId": "account_fixture",
|
||||
"deviceId": "device_fixture",
|
||||
"runtimeVersion": "0.9.6",
|
||||
"runtimeCommit": "a".repeat(40),
|
||||
"capabilities": CAPABILITIES,
|
||||
},
|
||||
"credential": { "accessToken": access_token },
|
||||
})))
|
||||
.expect(1)
|
||||
.mount(&server)
|
||||
.await;
|
||||
Mock::given(method("POST"))
|
||||
.and(path("/api/local-runners/connect"))
|
||||
.respond_with(ResponseTemplate::new(200).set_body_json(fixture_connection_response()))
|
||||
.mount(&server)
|
||||
.await;
|
||||
let mut enrollment = fixture_enrollment(&format!("{}/", server.uri()));
|
||||
let mut runner_id = "runner_fixture".to_string();
|
||||
let client = crate::tls::reqwest_client_builder()
|
||||
.redirect(reqwest::redirect::Policy::none())
|
||||
.build()
|
||||
.expect("fixture client");
|
||||
let (event_tx, _event_rx) = mpsc::unbounded_channel();
|
||||
let mut outbox = RuntimeTransportOutbox::default();
|
||||
outbox
|
||||
.enqueue(
|
||||
"run_fixture",
|
||||
runtime_envelope(
|
||||
1,
|
||||
"turn.completed",
|
||||
Some("turn_fixture"),
|
||||
"2026-08-08T12:00:00Z".to_string(),
|
||||
json!({ "turn": { "status": "completed", "usage": {} } }),
|
||||
),
|
||||
)
|
||||
.expect("queue terminal envelope");
|
||||
|
||||
let error = drain_runtime_outbox_for_stop(
|
||||
&client,
|
||||
&mut enrollment,
|
||||
&mut runner_id,
|
||||
&fixture_start(),
|
||||
&event_tx,
|
||||
&mut outbox,
|
||||
Instant::now() + Duration::from_secs(5),
|
||||
)
|
||||
.await
|
||||
.expect_err("a refusal that survives a refresh must end the drain");
|
||||
assert_eq!(error, RUNTIME_REFRESHED_CREDENTIAL_REFUSED);
|
||||
assert!(
|
||||
!outbox.events.is_empty(),
|
||||
"the refused envelope stays queued for a later, authorized resend"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn stop_drain_deadline_failure_refuses_to_confirm_stop() {
|
||||
let server = MockServer::start().await;
|
||||
|
||||
@@ -295,6 +295,7 @@ fn lighthouse_plan(inputs: &DeployInputs) -> Vec<ProvisionStep> {
|
||||
fn azure_plan(inputs: &DeployInputs) -> Vec<ProvisionStep> {
|
||||
let rg = format!("{}-rg", inputs.instance_name);
|
||||
let vault = format!("{}-kv", inputs.instance_name);
|
||||
let nsg = format!("{}-nsg", inputs.instance_name);
|
||||
let provider_secret = format!("codewhale-{}-key", inputs.provider_slug);
|
||||
vec![
|
||||
ProvisionStep::new(
|
||||
@@ -355,20 +356,40 @@ fn azure_plan(inputs: &DeployInputs) -> Vec<ProvisionStep> {
|
||||
"--custom-data",
|
||||
"cloud-init.yaml",
|
||||
"--assign-identity",
|
||||
// A Linux `vm create` otherwise adds an NSG rule admitting SSH
|
||||
// from any address; the next step adds the caller-only rule.
|
||||
"--nsg",
|
||||
&nsg,
|
||||
"--nsg-rule",
|
||||
"NONE",
|
||||
],
|
||||
),
|
||||
ProvisionStep::new(
|
||||
"Scope the NSG to SSH (22) from the caller IP only; 7878 stays on 127.0.0.1",
|
||||
"Allow SSH (22) from the caller IP only (replace <your-public-ip>); 7878 stays on 127.0.0.1",
|
||||
"az",
|
||||
&[
|
||||
"vm",
|
||||
"open-port",
|
||||
"network",
|
||||
"nsg",
|
||||
"rule",
|
||||
"create",
|
||||
"--resource-group",
|
||||
&rg,
|
||||
"--nsg-name",
|
||||
&nsg,
|
||||
"--name",
|
||||
&inputs.instance_name,
|
||||
"--port",
|
||||
"allow-ssh-from-caller",
|
||||
"--priority",
|
||||
"1000",
|
||||
"--direction",
|
||||
"Inbound",
|
||||
"--access",
|
||||
"Allow",
|
||||
"--protocol",
|
||||
"Tcp",
|
||||
"--destination-port-ranges",
|
||||
"22",
|
||||
"--source-address-prefixes",
|
||||
"<your-public-ip>/32",
|
||||
],
|
||||
),
|
||||
]
|
||||
@@ -540,6 +561,45 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn azure_plan_admits_ssh_from_the_caller_only() {
|
||||
let steps = (AZURE.plan)(&DeployInputs::default());
|
||||
let arg_after = |step: &ProvisionStep, flag: &str| {
|
||||
let at = step.args.iter().position(|arg| arg == flag)?;
|
||||
step.args.get(at + 1).cloned()
|
||||
};
|
||||
// `open-port` has no source restriction: it admits the whole internet.
|
||||
assert!(
|
||||
steps
|
||||
.iter()
|
||||
.all(|s| !s.args.iter().any(|a| a == "open-port")),
|
||||
"Azure plan must not open a port to every source"
|
||||
);
|
||||
let create = steps
|
||||
.iter()
|
||||
.find(|s| {
|
||||
s.args
|
||||
.starts_with(&["vm".to_string(), "create".to_string()])
|
||||
})
|
||||
.expect("vm create step");
|
||||
assert_eq!(arg_after(create, "--nsg-rule").as_deref(), Some("NONE"));
|
||||
let ssh_rules: Vec<_> = steps
|
||||
.iter()
|
||||
.filter(|s| arg_after(s, "--destination-port-ranges").as_deref() == Some("22"))
|
||||
.collect();
|
||||
assert_eq!(ssh_rules.len(), 1, "exactly one SSH rule");
|
||||
let source = arg_after(ssh_rules[0], "--source-address-prefixes").unwrap();
|
||||
assert!(
|
||||
source.ends_with("/32") && !source.starts_with('*') && !source.starts_with("0.0.0.0"),
|
||||
"SSH source must be a single caller address, got {source}"
|
||||
);
|
||||
assert_eq!(
|
||||
arg_after(ssh_rules[0], "--nsg-name"),
|
||||
arg_after(create, "--nsg"),
|
||||
"the rule must land on the NSG the VM was created with"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn display_command_redacts_secret_args() {
|
||||
let mut step =
|
||||
|
||||
@@ -355,7 +355,14 @@ fn classify(
|
||||
) -> BillingPresentation {
|
||||
match provider {
|
||||
ApiProvider::Ollama | ApiProvider::Sglang | ApiProvider::Vllm => BillingPresentation::Local,
|
||||
ApiProvider::OpenaiCodex => BillingPresentation::Subscription("Codex OAuth quota"),
|
||||
// The Codex OAuth quota is a fact of the ChatGPT backend, not of the
|
||||
// provider name: a custom `[providers.openai_codex] base_url` (a
|
||||
// gateway, a proxy) sells its own terms, so it is Unknown rather than
|
||||
// a subscription that would hide metered spend.
|
||||
ApiProvider::OpenaiCodex if is_chatgpt_codex_backend(base_url) => {
|
||||
BillingPresentation::Subscription("Codex OAuth quota")
|
||||
}
|
||||
ApiProvider::OpenaiCodex => BillingPresentation::Unknown,
|
||||
ApiProvider::OpencodeGo => BillingPresentation::Subscription("OpenCode Go quota"),
|
||||
// StepFun already reduces an endpoint to a non-secret billing surface
|
||||
// and fails closed on anything it does not recognize.
|
||||
@@ -608,6 +615,20 @@ fn stepfun_billing_for_endpoint(base_url: Option<&str>) -> BillingPresentation {
|
||||
}
|
||||
}
|
||||
|
||||
/// The ChatGPT backend the Codex OAuth route ships with
|
||||
/// (`https://chatgpt.com/backend-api`, or a path under it).
|
||||
fn is_chatgpt_codex_backend(base_url: &str) -> bool {
|
||||
let Ok(url) = reqwest::Url::parse(base_url.trim()) else {
|
||||
return false;
|
||||
};
|
||||
let path = url.path().trim_end_matches('/');
|
||||
url.scheme() == "https"
|
||||
&& url.host_str() == Some("chatgpt.com")
|
||||
&& url.port().is_none()
|
||||
&& url.username().is_empty()
|
||||
&& (path == "/backend-api" || path.starts_with("/backend-api/"))
|
||||
}
|
||||
|
||||
fn is_zai_coding_plan_endpoint(base_url: &str) -> bool {
|
||||
base_url
|
||||
.trim()
|
||||
@@ -2401,6 +2422,26 @@ mod tests {
|
||||
),
|
||||
BillingPresentation::Subscription("Codex OAuth quota")
|
||||
);
|
||||
// The OAuth token pointed anywhere else proves no Codex quota.
|
||||
for elsewhere in [
|
||||
"https://codex-gateway.example.com/backend-api",
|
||||
"https://chatgpt.com.example.net/backend-api",
|
||||
"http://chatgpt.com/backend-api",
|
||||
"https://chatgpt.com/v1",
|
||||
"",
|
||||
] {
|
||||
assert_eq!(
|
||||
for_dispatched_route(
|
||||
&config,
|
||||
DispatchedRoute {
|
||||
provider: ApiProvider::OpenaiCodex,
|
||||
base_url: elsewhere,
|
||||
},
|
||||
),
|
||||
BillingPresentation::Unknown,
|
||||
"{elsewhere:?}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -105,10 +105,18 @@ pub(super) struct WorkspaceGitMetadata {
|
||||
pub(super) dirty: bool,
|
||||
}
|
||||
|
||||
/// Status is several blocking `git` spawns, so it runs off the async workers
|
||||
/// (#6149) like every other git read on this surface. It deliberately uses
|
||||
/// normal Git (filters, untracked settings) rather than the review command,
|
||||
/// so its counts match `git status` and the operator writes (see `git.rs`).
|
||||
pub(super) async fn workspace_status(
|
||||
State(state): State<RuntimeApiState>,
|
||||
) -> Result<Json<WorkspaceStatusResponse>, ApiError> {
|
||||
Ok(Json(collect_workspace_status(&state.workspace)))
|
||||
let workspace = state.workspace.clone();
|
||||
tokio::task::spawn_blocking(move || collect_workspace_status(&workspace))
|
||||
.await
|
||||
.map(Json)
|
||||
.map_err(|_| ApiError::internal("workspace status failed"))
|
||||
}
|
||||
|
||||
pub(super) fn collect_workspace_status(workspace: &FsPath) -> WorkspaceStatusResponse {
|
||||
|
||||
@@ -648,12 +648,22 @@ impl ScorecardMetrics {
|
||||
/// Flag metrics that grew more than `threshold_pct` over `baseline`. Cost
|
||||
/// and token counts are "lower is better", so only *increases* are
|
||||
/// regressions. (Cache-hit ratio is the opposite, reported separately.)
|
||||
///
|
||||
/// Fails closed on values that cannot be compared: a non-finite threshold
|
||||
/// admits no growth at all (it is treated as 0%), and a metric whose
|
||||
/// change is `NaN` counts as regressed. Every `NaN` comparison is false,
|
||||
/// so without this a `NaN` threshold would pass any run.
|
||||
#[must_use]
|
||||
pub fn regressions_against(
|
||||
&self,
|
||||
baseline: &ScorecardMetrics,
|
||||
threshold_pct: f64,
|
||||
) -> Vec<Regression> {
|
||||
let threshold_pct = if threshold_pct.is_finite() {
|
||||
threshold_pct
|
||||
} else {
|
||||
0.0
|
||||
};
|
||||
let mut out = Vec::new();
|
||||
// A partial/unknown subtotal is not comparable to a complete baseline,
|
||||
// but losing completeness is itself a regression. Otherwise removing
|
||||
@@ -711,7 +721,7 @@ impl ScorecardMetrics {
|
||||
let drop_pct = (baseline.cache_hit_ratio - self.cache_hit_ratio)
|
||||
/ baseline.cache_hit_ratio
|
||||
* 100.0;
|
||||
if drop_pct > threshold_pct {
|
||||
if drop_pct.is_nan() || drop_pct > threshold_pct {
|
||||
out.push(Regression {
|
||||
metric: "cache_hit_ratio_drop".to_string(),
|
||||
baseline: baseline.cache_hit_ratio,
|
||||
@@ -733,7 +743,7 @@ fn push_regression(
|
||||
) {
|
||||
if base > 0.0 {
|
||||
let pct = (cur - base) / base * 100.0;
|
||||
if pct > threshold_pct {
|
||||
if pct.is_nan() || pct > threshold_pct {
|
||||
out.push(Regression {
|
||||
metric: metric.to_string(),
|
||||
baseline: base,
|
||||
@@ -1908,6 +1918,53 @@ mod tests {
|
||||
assert!(!names.contains(&"total_input_tokens")); // under threshold
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_non_finite_threshold_cannot_pass_a_regressed_run() {
|
||||
let baseline = ScorecardMetrics {
|
||||
turns: 1,
|
||||
money_metered_turns: 1,
|
||||
unpriced_turns: 0,
|
||||
cny_unpriced_turns: 0,
|
||||
cost_complete: true,
|
||||
cny_cost_complete: true,
|
||||
unpriced_classes: Vec::new(),
|
||||
total_input_tokens: 1000,
|
||||
total_output_tokens: 1000,
|
||||
total_cache_read_tokens: 0,
|
||||
total_cache_write_tokens: 0,
|
||||
total_reasoning_tokens: 0,
|
||||
total_cost_usd: 0.10,
|
||||
total_cost_cny: 0.7,
|
||||
cache_hit_ratio: 0.5,
|
||||
};
|
||||
let current = ScorecardMetrics {
|
||||
total_output_tokens: 2000,
|
||||
..baseline.clone()
|
||||
};
|
||||
for threshold in [f64::NAN, f64::INFINITY] {
|
||||
let names: Vec<String> = current
|
||||
.regressions_against(&baseline, threshold)
|
||||
.into_iter()
|
||||
.map(|r| r.metric)
|
||||
.collect();
|
||||
assert!(
|
||||
names.iter().any(|name| name == "total_output_tokens"),
|
||||
"threshold {threshold} passed a +100% regression: {names:?}"
|
||||
);
|
||||
}
|
||||
// A metric whose change cannot be computed is not a pass either.
|
||||
let unknown = ScorecardMetrics {
|
||||
cache_hit_ratio: f64::NAN,
|
||||
..baseline.clone()
|
||||
};
|
||||
assert!(
|
||||
unknown
|
||||
.regressions_against(&baseline, 5.0)
|
||||
.iter()
|
||||
.any(|r| r.metric == "cache_hit_ratio_drop")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn regression_flags_loss_of_cost_completeness_without_comparing_subtotals() {
|
||||
let baseline = ScorecardMetrics {
|
||||
|
||||
@@ -44,7 +44,7 @@ const MAX_EMPTY_SESSION_STUBS: usize = 10;
|
||||
/// Maximum session title length, in `char`s. Matches the bound the session
|
||||
/// picker's rename prompt has always enforced.
|
||||
pub const MAX_SESSION_TITLE_CHARS: usize = 100;
|
||||
const WORK_GRAPH_IMPORT_ARCHIVE_DIR: &str = ".work-graph-import-archive";
|
||||
pub(crate) const WORK_GRAPH_IMPORT_ARCHIVE_DIR: &str = ".work-graph-import-archive";
|
||||
const SESSION_GOALS_DIR: &str = ".goals";
|
||||
const CURRENT_SESSION_GOAL_SCHEMA_VERSION: u32 = 2;
|
||||
const MAX_SESSION_GOAL_OBJECTIVE_CHARS: usize = 8_192;
|
||||
@@ -445,6 +445,47 @@ pub fn set_live_session(session_id: Option<&str>) {
|
||||
static LIVE_SESSION_LEASE: std::sync::Mutex<Option<(String, Option<fs::File>)>> =
|
||||
std::sync::Mutex::new(None);
|
||||
|
||||
/// Lock attempts [`SessionManager::reserve_session_for_attach`] makes
|
||||
/// before it treats contention as a running owner.
|
||||
const LIVE_LEASE_ATTACH_ATTEMPTS: u32 = 3;
|
||||
|
||||
/// The refusal for attaching to a session another process has open.
|
||||
pub(crate) fn session_open_elsewhere(session_id: &str) -> std::io::Error {
|
||||
std::io::Error::new(
|
||||
std::io::ErrorKind::ResourceBusy,
|
||||
format!(
|
||||
"session {session_id} is open in another Codewhale window. \
|
||||
Continue it there, or run `codewhale fork {session_id}` to work on a copy"
|
||||
),
|
||||
)
|
||||
}
|
||||
|
||||
/// A reservation of a session's cross-process live lease, taken by
|
||||
/// [`SessionManager::reserve_session_for_attach`] before a surface attaches.
|
||||
/// Dropping it releases the reservation; [`Self::commit`] makes the session
|
||||
/// this process's live session and releases the previous one's lease.
|
||||
#[must_use = "an uncommitted reservation is released when dropped"]
|
||||
#[derive(Debug)]
|
||||
pub struct SessionLease {
|
||||
id: String,
|
||||
/// `None` when this process already holds the session's lease.
|
||||
file: Option<fs::File>,
|
||||
}
|
||||
|
||||
impl SessionLease {
|
||||
/// Make the reserved session this process's live session. Called after
|
||||
/// the attach has been validated and applied, so a failed attach never
|
||||
/// costs the session this process already owns its lease.
|
||||
pub fn commit(self) {
|
||||
set_live_session(Some(&self.id));
|
||||
if let Some(file) = self.file
|
||||
&& let Ok(mut lease) = LIVE_SESSION_LEASE.lock()
|
||||
{
|
||||
*lease = Some((self.id, Some(file)));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Is this session currently owned by **this process's** interactive surface?
|
||||
///
|
||||
/// The registry is process-local. Reclamation must not treat a missing entry
|
||||
@@ -1503,19 +1544,26 @@ impl SessionManager {
|
||||
let Ok(mut lease) = LIVE_SESSION_LEASE.lock() else {
|
||||
return;
|
||||
};
|
||||
if lease.as_ref().is_some_and(|(held, _)| held == id) {
|
||||
return;
|
||||
}
|
||||
// A lease already held is kept. One that could not be taken is tried
|
||||
// again: another process's liveness probe briefly holds the same lock,
|
||||
// and a claim that lost that race once must not run unleased for the
|
||||
// rest of the session.
|
||||
let retry = match lease.as_ref() {
|
||||
Some((held, Some(_))) if held == id => return,
|
||||
Some((held, None)) => held == id,
|
||||
_ => false,
|
||||
};
|
||||
let file = self.live_lease_path(id, true).and_then(|path| {
|
||||
let file = open_private_lock_file(&path)?;
|
||||
Ok(crate::runtime_threads::try_lock_file_exclusive(&file)?.then_some(file))
|
||||
});
|
||||
match &file {
|
||||
Ok(Some(_)) => {}
|
||||
Ok(None) => tracing::warn!(
|
||||
Ok(None) if !retry => tracing::warn!(
|
||||
session_id = id,
|
||||
"another Codewhale process already holds this session open"
|
||||
),
|
||||
Ok(None) => {}
|
||||
Err(error) => {
|
||||
tracing::debug!(session_id = id, %error, "session live lease unavailable");
|
||||
}
|
||||
@@ -1523,6 +1571,59 @@ impl SessionManager {
|
||||
*lease = Some((id.to_string(), file.ok().flatten()));
|
||||
}
|
||||
|
||||
/// Reserve `session_id` before attaching to it (`resume`, `--continue`,
|
||||
/// `exec --resume`, the session picker, `/load`). Unlike
|
||||
/// [`Self::claim_live_session`], which records whatever it gets, this
|
||||
/// refuses with `ResourceBusy` when another process holds the session
|
||||
/// open: attaching anyway gives the document two autosaving writers, and
|
||||
/// the last save silently drops the other's turns. Taking the lock is the
|
||||
/// check, so nothing can claim the session between a check and the attach.
|
||||
///
|
||||
/// The reservation does not touch the session this process owns now: that
|
||||
/// claim and its lease stay until [`SessionLease::commit`], which the
|
||||
/// caller runs only once the new session is loaded and applied. Dropping
|
||||
/// the reservation (a load or apply that failed) releases it and leaves
|
||||
/// the current session leased.
|
||||
///
|
||||
/// It fails closed: a lease that cannot be opened or locked is an error,
|
||||
/// not an unguarded attach.
|
||||
pub fn reserve_session_for_attach(&self, session_id: &str) -> io::Result<SessionLease> {
|
||||
let id = self.validated_session_id(session_id)?;
|
||||
if LIVE_SESSION_LEASE
|
||||
.lock()
|
||||
.is_ok_and(|lease| matches!(lease.as_ref(), Some((held, Some(_))) if held == id))
|
||||
{
|
||||
return Ok(SessionLease {
|
||||
id: id.to_string(),
|
||||
file: None,
|
||||
});
|
||||
}
|
||||
let lease_unavailable = |error: io::Error| {
|
||||
io::Error::new(
|
||||
error.kind(),
|
||||
format!("could not take session {id}'s live lease: {error}"),
|
||||
)
|
||||
};
|
||||
let file = self
|
||||
.live_lease_path(id, true)
|
||||
.and_then(|path| open_private_lock_file(&path))
|
||||
.map_err(lease_unavailable)?;
|
||||
// A liveness probe from another process holds this lock for a moment;
|
||||
// a few short retries tell that apart from a running owner.
|
||||
for attempt in 0..LIVE_LEASE_ATTACH_ATTEMPTS {
|
||||
if crate::runtime_threads::try_lock_file_exclusive(&file).map_err(lease_unavailable)? {
|
||||
return Ok(SessionLease {
|
||||
id: id.to_string(),
|
||||
file: Some(file),
|
||||
});
|
||||
}
|
||||
if attempt + 1 < LIVE_LEASE_ATTACH_ATTEMPTS {
|
||||
std::thread::sleep(std::time::Duration::from_millis(10));
|
||||
}
|
||||
}
|
||||
Err(session_open_elsewhere(id))
|
||||
}
|
||||
|
||||
/// Is `session_id` open in an interactive session in this process *or any
|
||||
/// other*? External writers (the Runtime API, retention) check this before
|
||||
/// rewriting or deleting a document, because the process holding it would
|
||||
@@ -1547,6 +1648,16 @@ impl SessionManager {
|
||||
)
|
||||
}
|
||||
|
||||
/// Hold `session_id`'s live lease the way another running Codewhale
|
||||
/// process does: an OS lock on its own open file description.
|
||||
#[cfg(test)]
|
||||
pub(crate) fn hold_live_lease_elsewhere(&self, session_id: &str) -> fs::File {
|
||||
let path = self.live_lease_path(session_id, true).expect("lease path");
|
||||
let lease = open_private_lock_file(&path).expect("open lease");
|
||||
assert!(crate::runtime_threads::try_lock_file_exclusive(&lease).expect("lock lease"));
|
||||
lease
|
||||
}
|
||||
|
||||
/// Whether a saved document exists for `session_id`.
|
||||
#[must_use]
|
||||
pub fn session_document_exists(&self, session_id: &str) -> bool {
|
||||
@@ -2270,7 +2381,7 @@ impl SessionManager {
|
||||
self.read_checkpoint_file(&path)
|
||||
}
|
||||
|
||||
fn legacy_checkpoint_origin(&self) -> io::Result<Option<String>> {
|
||||
pub(crate) fn legacy_checkpoint_origin(&self) -> io::Result<Option<String>> {
|
||||
use std::io::Read as _;
|
||||
|
||||
let path = self.checkpoints_dir().join(LEGACY_CHECKPOINT_FILE);
|
||||
@@ -2384,7 +2495,12 @@ impl SessionManager {
|
||||
let CheckpointSource::Session(id) = checkpoint.source else {
|
||||
continue;
|
||||
};
|
||||
if Some(id.as_str()) == exclude || !self.session_from_prior_instance(&id) {
|
||||
// A checkpoint another running session is still refreshing is
|
||||
// that session's in-flight work, not a prior crash.
|
||||
if Some(id.as_str()) == exclude
|
||||
|| !self.session_from_prior_instance(&id)
|
||||
|| self.is_session_live_anywhere(&id)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
// One malformed id or unreadable record must not hide a later
|
||||
@@ -2640,6 +2756,49 @@ impl SessionManager {
|
||||
Ok(recovery)
|
||||
}
|
||||
|
||||
/// [`Self::resume_session`] for a surface that will own and autosave the
|
||||
/// session: it first reserves the session's live lease, and refuses with
|
||||
/// `ResourceBusy` when another process has it open
|
||||
/// ([`Self::reserve_session_for_attach`]). The caller commits the
|
||||
/// returned lease once the session is applied.
|
||||
pub fn attach_session(&self, id: &str) -> std::io::Result<(SessionRecovery, SessionLease)> {
|
||||
let lease = self.reserve_session_for_attach(id)?;
|
||||
let recovery = self.resume_session(id)?;
|
||||
Ok((recovery, lease))
|
||||
}
|
||||
|
||||
/// [`Self::attach_session`] with a partial-ID prefix.
|
||||
pub fn attach_session_by_prefix(
|
||||
&self,
|
||||
prefix: &str,
|
||||
) -> std::io::Result<(SessionRecovery, SessionLease)> {
|
||||
self.attach_session(&self.resolve_session_id_prefix(prefix)?)
|
||||
}
|
||||
|
||||
/// [`Self::attach_session`] for a session file `/load` has already read.
|
||||
/// Either way the surface becomes the writer of the id the file names, so
|
||||
/// a managed record and a foreign file take the same contract: the id's
|
||||
/// live lease is reserved first and refused while another window has it
|
||||
/// open. A managed record then resumes through the manager so its repair
|
||||
/// is persisted in place; a foreign file is not ours to rewrite, so its
|
||||
/// journal projection and repair stay in memory.
|
||||
pub fn attach_session_file(
|
||||
&self,
|
||||
parsed: SavedSession,
|
||||
path: &Path,
|
||||
) -> std::io::Result<(SavedSession, SessionLease)> {
|
||||
if self.owns_session_path(&parsed.metadata.id, path) {
|
||||
return self
|
||||
.attach_session(&parsed.metadata.id)
|
||||
.map(|(recovery, lease)| (recovery.session, lease));
|
||||
}
|
||||
let lease = self.reserve_session_for_attach(&parsed.metadata.id)?;
|
||||
let mut session = parsed;
|
||||
session.ensure_journal();
|
||||
repair_recovered_session(&mut session);
|
||||
Ok((session, lease))
|
||||
}
|
||||
|
||||
/// [`Self::resume_session`] with a partial-ID prefix.
|
||||
pub fn resume_session_by_prefix(&self, prefix: &str) -> std::io::Result<SessionRecovery> {
|
||||
self.resume_session(&self.resolve_session_id_prefix(prefix)?)
|
||||
@@ -2867,6 +3026,36 @@ impl SessionManager {
|
||||
self.remove_session(id, SessionRemoval::Explicit)
|
||||
}
|
||||
|
||||
/// Remove the pre-import copy of `session_path` that the first
|
||||
/// graph-bearing write kept (see `archive_before_first_graph_write`): it is
|
||||
/// the same transcript, so a deleted session must not survive in it.
|
||||
///
|
||||
/// A linked archive directory is not followed, as the late-usage store
|
||||
/// refuses one: deletion must not reach a same-named file elsewhere.
|
||||
fn remove_import_archive_copy(&self, session_path: &Path) -> io::Result<()> {
|
||||
let Some(name) = session_path.file_name() else {
|
||||
return Ok(());
|
||||
};
|
||||
let dir = self.sessions_dir.join(WORK_GRAPH_IMPORT_ARCHIVE_DIR);
|
||||
match fs::symlink_metadata(&dir) {
|
||||
Ok(metadata) if crate::plugins::metadata_is_link_or_reparse(&metadata) => {
|
||||
tracing::warn!(
|
||||
path = %dir.display(),
|
||||
"work-graph import archive is a link; its copies were not removed"
|
||||
);
|
||||
return Ok(());
|
||||
}
|
||||
Ok(_) => {}
|
||||
Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(()),
|
||||
Err(error) => return Err(error),
|
||||
}
|
||||
match fs::remove_file(dir.join(name)) {
|
||||
Ok(()) => Ok(()),
|
||||
Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(()),
|
||||
Err(error) => Err(error),
|
||||
}
|
||||
}
|
||||
|
||||
fn remove_session(&self, id: &str, removal: SessionRemoval) -> std::io::Result<()> {
|
||||
let path = self.validated_session_path(id)?;
|
||||
// Older ordinary snapshots may use a name reserved by the checkpoint
|
||||
@@ -2901,11 +3090,13 @@ impl SessionManager {
|
||||
// Retention owns the ordinary snapshot, not crash recovery. Keep
|
||||
// the origin and its accounting/evidence writable for resume.
|
||||
// An unreadable legacy origin cannot justify retiring any id.
|
||||
return match fs::remove_file(&path) {
|
||||
Ok(()) => Ok(()),
|
||||
Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(()),
|
||||
Err(error) => Err(error),
|
||||
};
|
||||
match fs::remove_file(&path) {
|
||||
Ok(()) => {}
|
||||
Err(error) if error.kind() == io::ErrorKind::NotFound => {}
|
||||
Err(error) => return Err(error),
|
||||
}
|
||||
// The pre-import copy is an older ordinary snapshot, not recovery.
|
||||
return self.remove_import_archive_copy(&path);
|
||||
}
|
||||
// The store this document is bound to is named by its binding, not by
|
||||
// its id: a host store usually sits under another conversation's
|
||||
@@ -2944,6 +3135,7 @@ impl SessionManager {
|
||||
Err(error) => return Err(error),
|
||||
}
|
||||
}
|
||||
self.remove_import_archive_copy(&path)?;
|
||||
self.clear_session_boot_owner(id);
|
||||
let session_dir = self.sessions_dir.join(id.trim());
|
||||
if session_dir.exists() {
|
||||
@@ -3755,6 +3947,25 @@ fn json_object_end(bytes: &[u8], start: usize) -> Option<usize> {
|
||||
None
|
||||
}
|
||||
|
||||
/// The longest valid UTF-8 prefix of `buf`. A fixed-size read prefix can end
|
||||
/// inside a multi-byte character (routine for CJK or emoji transcripts, which
|
||||
/// serde writes unescaped); rejecting the whole buffer then forced a full-file
|
||||
/// read of every such session on each listing.
|
||||
///
|
||||
/// Only a character cut at the very end is trimmed. An invalid byte anywhere
|
||||
/// else means a damaged file, and an empty prefix sends it down the full read,
|
||||
/// which reports it.
|
||||
fn utf8_prefix(buf: &[u8]) -> &str {
|
||||
match std::str::from_utf8(buf) {
|
||||
Ok(s) => s,
|
||||
// `valid_up_to` marks a char boundary, so this slice is valid.
|
||||
Err(error) if error.error_len().is_none() => {
|
||||
std::str::from_utf8(&buf[..error.valid_up_to()]).unwrap_or_default()
|
||||
}
|
||||
Err(_) => "",
|
||||
}
|
||||
}
|
||||
|
||||
/// String-scan a JSON byte buffer for the top-level `"metadata":{...}`
|
||||
/// block and return it parsed. Returns `None` if no balanced metadata
|
||||
/// object is present in the buffer.
|
||||
@@ -3764,7 +3975,7 @@ fn json_object_end(bytes: &[u8], start: usize) -> Option<usize> {
|
||||
/// `}` appearing inside a string literal doesn't perturb the depth
|
||||
/// count.
|
||||
fn extract_top_level_metadata(buf: &[u8]) -> Option<SessionMetadata> {
|
||||
let s = std::str::from_utf8(buf).ok()?;
|
||||
let s = utf8_prefix(buf);
|
||||
let bytes = s.as_bytes();
|
||||
const KEY: &[u8] = b"\"metadata\"";
|
||||
let start = json_value_start(bytes, find_json_key(bytes, KEY)?, KEY.len(), b'{')?;
|
||||
@@ -3776,9 +3987,7 @@ fn extract_top_level_metadata(buf: &[u8]) -> Option<SessionMetadata> {
|
||||
/// whether the array was seen to end. A message the prefix cut in half is
|
||||
/// simply absent; nothing is reconstructed.
|
||||
fn extract_leading_messages(buf: &[u8], max: usize) -> (Vec<Message>, bool) {
|
||||
let Ok(s) = std::str::from_utf8(buf) else {
|
||||
return (Vec::new(), false);
|
||||
};
|
||||
let s = utf8_prefix(buf);
|
||||
let bytes = s.as_bytes();
|
||||
const KEY: &[u8] = b"\"messages\"";
|
||||
let Some(key_offset) = find_json_key(bytes, KEY) else {
|
||||
@@ -6640,6 +6849,103 @@ mod tests {
|
||||
assert!(!sessions_dir.join(&session_id).exists());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn delete_session_removes_its_work_graph_import_archive() {
|
||||
let tmp = tempdir().expect("tempdir");
|
||||
let sessions_dir = tmp.path().join("sessions");
|
||||
let manager = SessionManager::new(sessions_dir.clone()).expect("new");
|
||||
let session = create_saved_session(
|
||||
&[make_test_message("user", "archived transcript")],
|
||||
"test-model",
|
||||
tmp.path(),
|
||||
0,
|
||||
None,
|
||||
);
|
||||
let session_id = session.metadata.id.clone();
|
||||
let path = manager.save_session(&session).expect("save");
|
||||
let archive_dir = sessions_dir.join(WORK_GRAPH_IMPORT_ARCHIVE_DIR);
|
||||
fs::create_dir_all(&archive_dir).expect("archive dir");
|
||||
let archive = archive_dir.join(path.file_name().expect("file name"));
|
||||
fs::copy(&path, &archive).expect("archive copy");
|
||||
let other = archive_dir.join("other-session.json");
|
||||
fs::write(&other, "{}").expect("other archive");
|
||||
|
||||
manager.delete_session(&session_id).expect("delete");
|
||||
|
||||
assert!(
|
||||
!archive.exists(),
|
||||
"the deleted transcript's copy is removed"
|
||||
);
|
||||
assert!(other.exists(), "other sessions' copies are untouched");
|
||||
}
|
||||
|
||||
/// Retention keeps a session's crash-recovery checkpoint but retires the
|
||||
/// ordinary snapshot, and the pre-import copy is an ordinary snapshot.
|
||||
#[test]
|
||||
fn retention_removes_the_import_archive_copy_of_a_recoverable_session() {
|
||||
let tmp = tempdir().expect("tempdir");
|
||||
let sessions_dir = tmp.path().join("sessions");
|
||||
let manager = SessionManager::new(sessions_dir.clone()).expect("new");
|
||||
let session = create_saved_session(
|
||||
&[make_test_message("user", "archived transcript")],
|
||||
"test-model",
|
||||
tmp.path(),
|
||||
0,
|
||||
None,
|
||||
);
|
||||
let session_id = session.metadata.id.clone();
|
||||
let path = manager.save_session(&session).expect("save");
|
||||
manager.save_checkpoint(&session).expect("checkpoint");
|
||||
let archive_dir = sessions_dir.join(WORK_GRAPH_IMPORT_ARCHIVE_DIR);
|
||||
fs::create_dir_all(&archive_dir).expect("archive dir");
|
||||
let archive = archive_dir.join(path.file_name().expect("file name"));
|
||||
fs::copy(&path, &archive).expect("archive copy");
|
||||
|
||||
manager
|
||||
.remove_session(&session_id, SessionRemoval::Retention)
|
||||
.expect("retention");
|
||||
|
||||
assert!(!path.exists(), "the ordinary snapshot is retired");
|
||||
assert!(!archive.exists(), "so is its pre-import copy");
|
||||
assert!(
|
||||
manager
|
||||
.load_session_checkpoint(&session_id)
|
||||
.expect("checkpoint")
|
||||
.is_some(),
|
||||
"crash recovery is kept"
|
||||
);
|
||||
}
|
||||
|
||||
/// Deletion does not follow a linked archive directory to a same-named
|
||||
/// file somewhere else.
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn delete_does_not_follow_a_linked_import_archive() {
|
||||
let tmp = tempdir().expect("tempdir");
|
||||
let sessions_dir = tmp.path().join("sessions");
|
||||
let manager = SessionManager::new(sessions_dir.clone()).expect("new");
|
||||
let session = create_saved_session(
|
||||
&[make_test_message("user", "transcript")],
|
||||
"test-model",
|
||||
tmp.path(),
|
||||
0,
|
||||
None,
|
||||
);
|
||||
let session_id = session.metadata.id.clone();
|
||||
let path = manager.save_session(&session).expect("save");
|
||||
let elsewhere = tmp.path().join("elsewhere");
|
||||
fs::create_dir_all(&elsewhere).expect("elsewhere");
|
||||
let unrelated = elsewhere.join(path.file_name().expect("file name"));
|
||||
fs::write(&unrelated, "{}").expect("unrelated file");
|
||||
std::os::unix::fs::symlink(&elsewhere, sessions_dir.join(WORK_GRAPH_IMPORT_ARCHIVE_DIR))
|
||||
.expect("link");
|
||||
|
||||
manager.delete_session(&session_id).expect("delete");
|
||||
|
||||
assert!(!path.exists());
|
||||
assert!(unrelated.exists(), "a file behind the link is untouched");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_session_id_rejects_invalid_characters() {
|
||||
let tmp = tempdir().expect("tempdir");
|
||||
@@ -7419,6 +7725,205 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
fn hold_live_lease(manager: &SessionManager, id: &str) -> fs::File {
|
||||
manager.hold_live_lease_elsewhere(id)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn interrupted_workspace_session_skips_a_session_live_elsewhere() {
|
||||
let tmp = tempdir().expect("tempdir");
|
||||
let manager = SessionManager::new(tmp.path().join("sessions")).expect("manager");
|
||||
let workspace = tmp.path().join("ws");
|
||||
fs::create_dir_all(&workspace).expect("workspace");
|
||||
write_prior_interrupted_session(&manager, "sess-crashed", &workspace);
|
||||
std::thread::sleep(std::time::Duration::from_millis(20));
|
||||
write_prior_interrupted_session(&manager, "sess-running", &workspace);
|
||||
|
||||
let lease = hold_live_lease(&manager, "sess-running");
|
||||
assert_eq!(
|
||||
manager
|
||||
.interrupted_workspace_session(&workspace, None)
|
||||
.map(|meta| meta.id),
|
||||
Some("sess-crashed".to_string()),
|
||||
"a checkpoint another terminal is refreshing is not a crash"
|
||||
);
|
||||
drop(lease);
|
||||
assert_eq!(
|
||||
manager
|
||||
.interrupted_workspace_session(&workspace, None)
|
||||
.map(|meta| meta.id),
|
||||
Some("sess-running".to_string())
|
||||
);
|
||||
}
|
||||
|
||||
/// Attaching to a session another process has open is refused, naming
|
||||
/// it, rather than giving the document a second autosaving writer.
|
||||
#[test]
|
||||
fn attach_refuses_a_session_open_in_another_process() {
|
||||
let _env = crate::test_support::lock_test_env();
|
||||
let tmp = tempdir().expect("tempdir");
|
||||
let manager = SessionManager::new(tmp.path().join("sessions")).expect("manager");
|
||||
let workspace = tmp.path().join("ws");
|
||||
fs::create_dir_all(&workspace).expect("workspace");
|
||||
write_prior_interrupted_session(&manager, "sess-open", &workspace);
|
||||
|
||||
let lease = hold_live_lease(&manager, "sess-open");
|
||||
let error = manager
|
||||
.reserve_session_for_attach("sess-open")
|
||||
.expect_err("a session open elsewhere is refused");
|
||||
assert_eq!(error.kind(), io::ErrorKind::ResourceBusy);
|
||||
assert!(error.to_string().contains("sess-open"), "{error}");
|
||||
assert!(!is_live_session("sess-open"), "nothing was claimed");
|
||||
|
||||
drop(lease);
|
||||
let (_, reserved) = manager
|
||||
.attach_session("sess-open")
|
||||
.expect("attach once the other window has closed");
|
||||
assert!(
|
||||
!try_lock_elsewhere(&manager, "sess-open"),
|
||||
"the attach holds the lease from the start"
|
||||
);
|
||||
reserved.commit();
|
||||
assert!(is_live_session("sess-open"));
|
||||
assert!(!try_lock_elsewhere(&manager, "sess-open"));
|
||||
set_live_session(None);
|
||||
}
|
||||
|
||||
/// Switching from session A to B must not give up A until B is applied:
|
||||
/// a reservation that is dropped (B failed to load or apply) leaves A
|
||||
/// claimed and leased, and B free.
|
||||
#[test]
|
||||
fn a_failed_attach_keeps_the_current_sessions_lease() {
|
||||
let _env = crate::test_support::lock_test_env();
|
||||
let tmp = tempdir().expect("tempdir");
|
||||
let manager = SessionManager::new(tmp.path().join("sessions")).expect("manager");
|
||||
let workspace = tmp.path().join("ws");
|
||||
fs::create_dir_all(&workspace).expect("workspace");
|
||||
write_prior_interrupted_session(&manager, "sess-a", &workspace);
|
||||
write_prior_interrupted_session(&manager, "sess-b", &workspace);
|
||||
manager.claim_live_session("sess-a");
|
||||
assert!(!try_lock_elsewhere(&manager, "sess-a"), "A is leased");
|
||||
|
||||
let reserved = manager
|
||||
.reserve_session_for_attach("sess-b")
|
||||
.expect("reserve B");
|
||||
assert!(is_live_session("sess-a"), "A stays claimed while B loads");
|
||||
assert!(!try_lock_elsewhere(&manager, "sess-a"), "and leased");
|
||||
drop(reserved);
|
||||
assert!(is_live_session("sess-a"));
|
||||
assert!(!try_lock_elsewhere(&manager, "sess-a"), "A is still leased");
|
||||
assert!(try_lock_elsewhere(&manager, "sess-b"), "B was released");
|
||||
|
||||
// A B whose document cannot be loaded is refused the same way.
|
||||
let b_path = manager.validated_session_path("sess-b").expect("path");
|
||||
fs::write(&b_path, "{ not json").expect("corrupt B");
|
||||
manager
|
||||
.attach_session("sess-b")
|
||||
.expect_err("a corrupt B fails to attach");
|
||||
assert!(is_live_session("sess-a"));
|
||||
assert!(!try_lock_elsewhere(&manager, "sess-a"), "A is still leased");
|
||||
assert!(try_lock_elsewhere(&manager, "sess-b"), "B was released");
|
||||
set_live_session(None);
|
||||
}
|
||||
|
||||
/// `/load` takes the same contract as `resume`: a managed record and a
|
||||
/// foreign file naming a session open in another window are both
|
||||
/// refused, and neither is claimed.
|
||||
#[test]
|
||||
fn load_of_a_session_open_elsewhere_is_refused_for_managed_and_foreign_files() {
|
||||
let _env = crate::test_support::lock_test_env();
|
||||
let tmp = tempdir().expect("tempdir");
|
||||
let manager = SessionManager::new(tmp.path().join("sessions")).expect("manager");
|
||||
let workspace = tmp.path().join("ws");
|
||||
fs::create_dir_all(&workspace).expect("workspace");
|
||||
let session = write_prior_interrupted_session(&manager, "sess-load", &workspace);
|
||||
let managed = manager.validated_session_path("sess-load").expect("path");
|
||||
let foreign = tmp.path().join("exported.json");
|
||||
fs::write(&foreign, serde_json::to_vec(&session).expect("json")).expect("export");
|
||||
|
||||
let lease = hold_live_lease(&manager, "sess-load");
|
||||
for path in [&managed, &foreign] {
|
||||
let error = manager
|
||||
.attach_session_file(session.clone(), path)
|
||||
.expect_err("a session open elsewhere is not loaded");
|
||||
assert_eq!(
|
||||
error.kind(),
|
||||
io::ErrorKind::ResourceBusy,
|
||||
"{}",
|
||||
path.display()
|
||||
);
|
||||
}
|
||||
assert!(!is_live_session("sess-load"), "nothing was claimed");
|
||||
|
||||
drop(lease);
|
||||
for path in [&managed, &foreign] {
|
||||
let (loaded, reserved) = manager
|
||||
.attach_session_file(session.clone(), path)
|
||||
.expect("loads once the other window has closed");
|
||||
assert_eq!(loaded.metadata.id, "sess-load");
|
||||
assert!(
|
||||
!try_lock_elsewhere(&manager, "sess-load"),
|
||||
"the load holds the lease"
|
||||
);
|
||||
drop(reserved);
|
||||
}
|
||||
set_live_session(None);
|
||||
}
|
||||
|
||||
/// A lease that cannot be taken is an error, not an unguarded attach.
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn attach_fails_closed_when_the_lease_cannot_be_taken() {
|
||||
let _env = crate::test_support::lock_test_env();
|
||||
let tmp = tempdir().expect("tempdir");
|
||||
let sessions_dir = tmp.path().join("sessions");
|
||||
let manager = SessionManager::new(sessions_dir.clone()).expect("manager");
|
||||
let workspace = tmp.path().join("ws");
|
||||
fs::create_dir_all(&workspace).expect("workspace");
|
||||
write_prior_interrupted_session(&manager, "sess-x", &workspace);
|
||||
// The lease directory is a regular file: no lock file can be opened.
|
||||
let _ = fs::remove_dir_all(sessions_dir.join(LATE_USAGE_DIR));
|
||||
fs::write(sessions_dir.join(LATE_USAGE_DIR), "").expect("block lease dir");
|
||||
|
||||
let error = manager
|
||||
.attach_session("sess-x")
|
||||
.expect_err("no lease, no attach");
|
||||
assert!(error.to_string().contains("live lease"), "{error}");
|
||||
assert!(!is_live_session("sess-x"), "nothing was claimed");
|
||||
}
|
||||
|
||||
/// A liveness probe from another process briefly holds the lease lock. A
|
||||
/// claim that lost that race once must take the lease on a later save,
|
||||
/// not run unleased for the whole session.
|
||||
#[test]
|
||||
fn a_claim_that_lost_a_probe_race_takes_the_lease_later() {
|
||||
let _env = crate::test_support::lock_test_env();
|
||||
let tmp = tempdir().expect("tempdir");
|
||||
let manager = SessionManager::new(tmp.path().join("sessions")).expect("manager");
|
||||
|
||||
let probe = hold_live_lease(&manager, "sess-a");
|
||||
manager.claim_live_session("sess-a");
|
||||
drop(probe);
|
||||
assert!(
|
||||
try_lock_elsewhere(&manager, "sess-a"),
|
||||
"the first claim lost"
|
||||
);
|
||||
|
||||
manager.claim_live_session("sess-a");
|
||||
assert!(
|
||||
!try_lock_elsewhere(&manager, "sess-a"),
|
||||
"the next claim takes the lease"
|
||||
);
|
||||
set_live_session(None);
|
||||
}
|
||||
|
||||
/// Whether another open file description can lock `id`'s lease now.
|
||||
fn try_lock_elsewhere(manager: &SessionManager, id: &str) -> bool {
|
||||
let lease = open_private_read_file(&manager.live_lease_path(id, false).expect("path"))
|
||||
.expect("lease file");
|
||||
crate::runtime_threads::try_lock_file_exclusive(&lease).expect("lock")
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn interrupted_workspace_session_ignores_settled_sessions() {
|
||||
let tmp = tempdir().expect("tempdir");
|
||||
@@ -7839,6 +8344,41 @@ mod tests {
|
||||
assert_eq!(extracted.total_tokens, 4096);
|
||||
}
|
||||
|
||||
/// A 64 KB read prefix routinely ends inside a multi-byte character in a
|
||||
/// CJK or emoji transcript. The metadata ahead of the cut is intact and
|
||||
/// must still be read from the prefix, not force a full-file read.
|
||||
#[test]
|
||||
fn extract_top_level_metadata_survives_a_prefix_cut_inside_a_character() {
|
||||
let json = format!(
|
||||
r#"{{"schema_version":1,"metadata":{{"id":"cjk-1","title":"会话","created_at":"2026-01-01T00:00:00Z","updated_at":"2026-01-02T00:00:00Z","message_count":3,"total_tokens":10,"model":"m","workspace":"/tmp"}},"messages":[{{"role":"user","content":"{}"}}]}}"#,
|
||||
"中文".repeat(100)
|
||||
);
|
||||
let bytes = json.as_bytes();
|
||||
let body_start = json.find("中文").expect("body");
|
||||
// One byte into a three-byte character.
|
||||
let cut = &bytes[..body_start + 1];
|
||||
assert!(
|
||||
std::str::from_utf8(cut).is_err(),
|
||||
"the cut splits a character"
|
||||
);
|
||||
|
||||
let extracted = extract_top_level_metadata(cut).expect("metadata from the cut prefix");
|
||||
assert_eq!(extracted.id, "cjk-1");
|
||||
assert_eq!(extracted.title, "会话");
|
||||
}
|
||||
|
||||
/// Only a cut at the end of the prefix is trimmed. An invalid byte ahead
|
||||
/// of it is a damaged file, left to the full read to report.
|
||||
#[test]
|
||||
fn extract_top_level_metadata_rejects_invalid_utf8_inside_the_prefix() {
|
||||
let json = r#"{"schema_version":1,"metadata":{"id":"bad-1","title":"t","created_at":"2026-01-01T00:00:00Z","updated_at":"2026-01-02T00:00:00Z","message_count":3,"total_tokens":10,"model":"m","workspace":"/tmp"},"messages":[{"role":"user","content":"AB"}]}"#;
|
||||
let mut bytes = json.as_bytes().to_vec();
|
||||
let body = json.find("AB").expect("body");
|
||||
bytes[body] = 0xFF;
|
||||
|
||||
assert!(extract_top_level_metadata(&bytes).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extract_top_level_metadata_handles_braces_inside_strings() {
|
||||
// A title containing `{` and `}` inside the metadata block must
|
||||
|
||||
@@ -56,6 +56,15 @@ pub(crate) fn session_files(
|
||||
if let Some(checkpoints) = canonical_store_root(&root.join("checkpoints"), unreadable) {
|
||||
dirs.push(checkpoints);
|
||||
}
|
||||
// Pre-import transcript copies hold the same tool output. Session
|
||||
// deletion does not follow a linked archive directory, so neither does
|
||||
// the scrub: it is reported as not covered instead.
|
||||
let archive = root.join(crate::session_manager::WORK_GRAPH_IMPORT_ARCHIVE_DIR);
|
||||
match std::fs::symlink_metadata(&archive) {
|
||||
Ok(metadata) if metadata.is_dir() => dirs.push(archive),
|
||||
Err(error) if error.kind() == io::ErrorKind::NotFound => {}
|
||||
_ => unreadable.push(archive),
|
||||
}
|
||||
for session in directory_entries(&root, unreadable) {
|
||||
if !entry_has_type(&session, true, unreadable) {
|
||||
continue;
|
||||
@@ -181,6 +190,17 @@ pub(crate) fn configured_secrets(
|
||||
))
|
||||
}
|
||||
|
||||
/// Whether `path` sits directly in `manager`'s pre-import archive directory
|
||||
/// (never a linked one; see [`session_files`]).
|
||||
fn is_import_archive(path: &Path, manager: &crate::session_manager::SessionManager) -> bool {
|
||||
let archive = manager
|
||||
.sessions_dir()
|
||||
.join(crate::session_manager::WORK_GRAPH_IMPORT_ARCHIVE_DIR);
|
||||
std::fs::symlink_metadata(&archive).is_ok_and(|metadata| metadata.is_dir())
|
||||
&& std::fs::canonicalize(&archive)
|
||||
.is_ok_and(|archive| path.parent() == Some(archive.as_path()))
|
||||
}
|
||||
|
||||
fn runtime_root(path: &Path) -> Option<&Path> {
|
||||
let parent = path.parent()?;
|
||||
matches!(parent.file_name()?.to_str()?, "items" | "events")
|
||||
@@ -230,6 +250,18 @@ pub(crate) fn scrub_files(
|
||||
.with_session_file_lock(id, || scrub_file(path, true, secrets).map(|_| ()))
|
||||
}) {
|
||||
Some(Ok(Some(()))) => FileScan::Dirty(redacted),
|
||||
// A pre-import copy an earlier build left behind when it
|
||||
// deleted the session. Finish that deletion: rewriting it
|
||||
// would keep a deleted transcript, and could put back a
|
||||
// copy a concurrent delete had just removed.
|
||||
Some(Ok(None)) if is_import_archive(path, manager) => {
|
||||
match std::fs::remove_file(path) {
|
||||
Ok(()) => {}
|
||||
Err(error) if error.kind() == io::ErrorKind::NotFound => {}
|
||||
Err(error) => return Err(error),
|
||||
}
|
||||
FileScan::Dirty(redacted)
|
||||
}
|
||||
// A deleted session is not resurrected by a rewrite.
|
||||
Some(Ok(None)) => FileScan::Clean,
|
||||
// No lockable session id: leave the file untouched.
|
||||
@@ -437,6 +469,70 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
/// The pre-import copy of a transcript holds the same tool output. It is
|
||||
/// scrubbed with the sessions. A copy an earlier build left behind for a
|
||||
/// session it deleted is removed, finishing that deletion.
|
||||
#[test]
|
||||
fn work_graph_import_archive_copies_are_scrubbed() {
|
||||
let dir = tempfile::tempdir().expect("tempdir");
|
||||
let output = format!("{{\"access_token\": \"{TOKEN}\"}}");
|
||||
let manager =
|
||||
crate::session_manager::SessionManager::new(dir.path().to_path_buf()).expect("manager");
|
||||
std::fs::write(
|
||||
dir.path().join("gone.json"),
|
||||
session_with_tool_output("nothing").to_string(),
|
||||
)
|
||||
.unwrap();
|
||||
manager.delete_session("gone").expect("delete");
|
||||
let archive_dir = dir
|
||||
.path()
|
||||
.join(crate::session_manager::WORK_GRAPH_IMPORT_ARCHIVE_DIR);
|
||||
std::fs::create_dir_all(&archive_dir).unwrap();
|
||||
let kept = archive_dir.join("kept.json");
|
||||
let orphan = archive_dir.join("gone.json");
|
||||
for path in [&kept, &orphan] {
|
||||
std::fs::write(path, session_with_tool_output(&output).to_string()).unwrap();
|
||||
}
|
||||
|
||||
let files = session_files(dir.path(), &dir.path().join("standalone"), &mut Vec::new());
|
||||
let report = scrub_files(&files, Some(&manager), &[]).expect("scrub");
|
||||
|
||||
assert_eq!(report.flagged_files.len(), 2, "{report:?}");
|
||||
let text = std::fs::read_to_string(&kept).unwrap();
|
||||
assert!(!text.contains(TOKEN), "{text}");
|
||||
assert!(!orphan.exists(), "the deleted session's copy is removed");
|
||||
}
|
||||
|
||||
/// A linked archive directory is not followed, matching session deletion.
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn a_linked_import_archive_is_reported_not_followed() {
|
||||
let dir = tempfile::tempdir().expect("tempdir");
|
||||
let sessions = dir.path().join("sessions");
|
||||
let elsewhere = dir.path().join("elsewhere");
|
||||
std::fs::create_dir_all(&sessions).unwrap();
|
||||
std::fs::create_dir_all(&elsewhere).unwrap();
|
||||
let outside = elsewhere.join("gone.json");
|
||||
let output = format!("{{\"access_token\": \"{TOKEN}\"}}");
|
||||
std::fs::write(&outside, session_with_tool_output(&output).to_string()).unwrap();
|
||||
std::os::unix::fs::symlink(
|
||||
&elsewhere,
|
||||
sessions.join(crate::session_manager::WORK_GRAPH_IMPORT_ARCHIVE_DIR),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let mut unreadable = Vec::new();
|
||||
let files = session_files(&sessions, &dir.path().join("standalone"), &mut unreadable);
|
||||
|
||||
assert!(
|
||||
!files.iter().any(|file| file.ends_with("gone.json")),
|
||||
"{files:?}"
|
||||
);
|
||||
assert_eq!(unreadable.len(), 1, "{unreadable:?}");
|
||||
assert!(unreadable[0].ends_with(crate::session_manager::WORK_GRAPH_IMPORT_ARCHIVE_DIR));
|
||||
assert!(std::fs::read_to_string(&outside).unwrap().contains(TOKEN));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn busy_runtime_does_not_abort_scrub() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
|
||||
@@ -32,14 +32,15 @@ pub fn tool_output_status(messages: &[Message], artifacts: &[ArtifactRecord]) ->
|
||||
for message in messages {
|
||||
for block in &message.content {
|
||||
if let ContentBlock::ToolResult { content, .. } = block {
|
||||
if looks_like_receipt(content) {
|
||||
// The prefix is text any tool output can carry, so it never
|
||||
// exempts a result from the size check: a "receipt" above the
|
||||
// threshold is raw pressure like any other large result.
|
||||
let chars = content.chars().count();
|
||||
if chars > RAW_TOOL_OUTPUT_RECEIPT_THRESHOLD_CHARS {
|
||||
status.raw_large_count += 1;
|
||||
status.raw_large_chars = status.raw_large_chars.saturating_add(chars);
|
||||
} else if looks_like_receipt(content) {
|
||||
status.receipt_count += 1;
|
||||
} else {
|
||||
let chars = content.chars().count();
|
||||
if chars > RAW_TOOL_OUTPUT_RECEIPT_THRESHOLD_CHARS {
|
||||
status.raw_large_count += 1;
|
||||
status.raw_large_chars = status.raw_large_chars.saturating_add(chars);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -151,4 +152,12 @@ mod tests {
|
||||
assert!(rendered.contains("compact receipt"));
|
||||
assert!(rendered.contains("artifact"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_receipt_prefix_does_not_hide_a_large_raw_result() {
|
||||
let forged = format!("[TOOL_OUTPUT_RECEIPT]\n{}", "RAW\n".repeat(4_000));
|
||||
let status = tool_output_status(&[tool_result_message("call-forged", &forged)], &[]);
|
||||
assert_eq!(status.raw_large_count, 1);
|
||||
assert_eq!(status.receipt_count, 0);
|
||||
}
|
||||
}
|
||||
@@ -11277,7 +11277,9 @@ async fn spawn_subagent_from_input(
|
||||
)));
|
||||
}
|
||||
|
||||
if let Some(remaining) = crate::retry_status::rate_limit_remaining() {
|
||||
if let Some(remaining) =
|
||||
crate::retry_status::rate_limit_remaining(&runtime.client.rate_limit_scope())
|
||||
{
|
||||
let seconds = remaining.as_secs() + u64::from(remaining.subsec_nanos() > 0);
|
||||
return Err(ToolError::execution_failed(format!(
|
||||
"Provider is rate-limiting; sub-agent spawning is paused for {seconds}s. \
|
||||
|
||||
@@ -12605,11 +12605,14 @@ async fn rate_limit_pause_blocks_subagent_spawn() {
|
||||
let _clear = ClearRateLimitOnDrop;
|
||||
crate::retry_status::clear();
|
||||
crate::retry_status::clear_rate_limit();
|
||||
crate::retry_status::note_rate_limit(Duration::from_secs(30));
|
||||
|
||||
let tmp = tempdir().expect("tempdir");
|
||||
let mut runtime = stub_runtime();
|
||||
runtime.context = ToolContext::new(tmp.path().to_path_buf());
|
||||
crate::retry_status::note_rate_limit(
|
||||
&runtime.client.rate_limit_scope(),
|
||||
Duration::from_secs(30),
|
||||
);
|
||||
let manager = new_shared_subagent_manager(tmp.path().to_path_buf(), 2);
|
||||
|
||||
let err = spawn_subagent_from_input(
|
||||
@@ -18453,6 +18456,32 @@ async fn spawn_budget_capped_worker(
|
||||
String,
|
||||
Arc<AtomicUsize>,
|
||||
tokio::task::JoinHandle<()>,
|
||||
) {
|
||||
spawn_budget_capped_worker_with_pause(
|
||||
workspace,
|
||||
prompt_tokens,
|
||||
completion_tokens,
|
||||
max_steps,
|
||||
wall_time,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
/// As [`spawn_budget_capped_worker`], optionally opening a rate-limit pause
|
||||
/// on the worker's own route before it starts.
|
||||
async fn spawn_budget_capped_worker_with_pause(
|
||||
workspace: &Path,
|
||||
prompt_tokens: u64,
|
||||
completion_tokens: u64,
|
||||
max_steps: u32,
|
||||
wall_time: Duration,
|
||||
pause: Option<Duration>,
|
||||
) -> (
|
||||
Arc<RwLock<SubAgentManager>>,
|
||||
String,
|
||||
Arc<AtomicUsize>,
|
||||
tokio::task::JoinHandle<()>,
|
||||
) {
|
||||
let manager = Arc::new(RwLock::new(SubAgentManager::new(
|
||||
workspace.to_path_buf(),
|
||||
@@ -18502,6 +18531,9 @@ async fn spawn_budget_capped_worker(
|
||||
launch_gate: None,
|
||||
_foreground_child_registration: None,
|
||||
};
|
||||
if let Some(pause) = pause {
|
||||
crate::retry_status::note_rate_limit(&runtime.client.rate_limit_scope(), pause);
|
||||
}
|
||||
let task_handle = tokio::spawn(run_subagent_task(task));
|
||||
(manager, agent_id, calls, task_handle)
|
||||
}
|
||||
@@ -18781,11 +18813,17 @@ async fn worker_is_not_stranded_by_transient_global_rate_limit_window() {
|
||||
// in-flight requests promptly.
|
||||
let _guard = crate::retry_status::test_guard();
|
||||
let _clear = ClearRateLimitOnDrop;
|
||||
crate::retry_status::note_rate_limit(Duration::from_secs(30));
|
||||
|
||||
let tmp = tempdir().expect("tempdir");
|
||||
let (manager, agent_id, _calls, task_handle) =
|
||||
spawn_budget_capped_worker(tmp.path(), 60, 40, 4, DEFAULT_CHILD_WALL_TIME).await;
|
||||
let (manager, agent_id, _calls, task_handle) = spawn_budget_capped_worker_with_pause(
|
||||
tmp.path(),
|
||||
60,
|
||||
40,
|
||||
4,
|
||||
DEFAULT_CHILD_WALL_TIME,
|
||||
Some(Duration::from_secs(30)),
|
||||
)
|
||||
.await;
|
||||
|
||||
// Simulate the concurrent test finishing: the window closes shortly
|
||||
// after the worker's first request has already observed it.
|
||||
|
||||
@@ -47,6 +47,10 @@ pub(crate) enum StatusToastKind {
|
||||
BehavioralTip(crate::tui::behavioral_tips::BehavioralTip),
|
||||
PluginSuggestion,
|
||||
ContextPressure(crate::context_budget::PressureLevel),
|
||||
/// A condition that holds until its owner retires it by event id (a
|
||||
/// session save that keeps failing). It never times out: expiring it
|
||||
/// would report recovery that has not happened.
|
||||
Standing,
|
||||
}
|
||||
|
||||
impl StatusToast {
|
||||
@@ -67,6 +71,24 @@ impl StatusToast {
|
||||
self
|
||||
}
|
||||
|
||||
/// A notice for a standing condition, withdrawn only through
|
||||
/// [`App::retire_event_notices`]`(event_id)` once the condition clears.
|
||||
#[must_use]
|
||||
pub(crate) fn standing(
|
||||
text: impl Into<String>,
|
||||
level: StatusToastLevel,
|
||||
event_id: impl Into<String>,
|
||||
) -> Self {
|
||||
Self {
|
||||
text: text.into(),
|
||||
level,
|
||||
created_at: Instant::now(),
|
||||
ttl_ms: None,
|
||||
kind: StatusToastKind::Standing,
|
||||
event_id: Some(event_id.into()),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn for_action(mut self, request_id: impl Into<String>) -> Self {
|
||||
self.kind = StatusToastKind::ActionRequired;
|
||||
self.event_id = Some(request_id.into());
|
||||
@@ -130,8 +152,15 @@ impl App {
|
||||
return;
|
||||
}
|
||||
self.status_toasts.push_back(toast);
|
||||
// Overflow drops the oldest notice that is not a standing condition;
|
||||
// those leave only when their condition clears.
|
||||
while self.status_toasts.len() > 24 {
|
||||
self.status_toasts.pop_front();
|
||||
let oldest = self
|
||||
.status_toasts
|
||||
.iter()
|
||||
.position(|toast| toast.kind != StatusToastKind::Standing)
|
||||
.unwrap_or(0);
|
||||
self.status_toasts.remove(oldest);
|
||||
}
|
||||
self.needs_redraw = true;
|
||||
}
|
||||
@@ -147,6 +176,14 @@ impl App {
|
||||
self.needs_redraw |= self.status_toasts.len() != before;
|
||||
}
|
||||
|
||||
/// Retire the notices pushed with [`StatusToast::for_event`]`(event_id)`.
|
||||
pub(crate) fn retire_event_notices(&mut self, event_id: &str) {
|
||||
let before = self.status_toasts.len();
|
||||
self.status_toasts
|
||||
.retain(|toast| toast.event_id.as_deref() != Some(event_id));
|
||||
self.needs_redraw |= self.status_toasts.len() != before;
|
||||
}
|
||||
|
||||
/// Gate transitions retire their own guidance or failed-write receipt,
|
||||
/// independently of translated text and unrelated requests/errors.
|
||||
pub(crate) fn retire_redaction_gate_notice(&mut self, notice: RedactionGateNotice) {
|
||||
|
||||
@@ -431,7 +431,11 @@ impl<'a> AutoReviewContext<'a> {
|
||||
let tool_name = tool_name.to_owned();
|
||||
let params = params.clone();
|
||||
let workspace = workspace.map(std::path::Path::to_path_buf);
|
||||
#[cfg(test)]
|
||||
let env_ticket = crate::test_support::env_scope_ticket();
|
||||
tokio::task::spawn_blocking(move || {
|
||||
#[cfg(test)]
|
||||
let _membership = crate::test_support::join_env_scope(env_ticket);
|
||||
let trusted = workspace
|
||||
.as_deref()
|
||||
.is_some_and(crate::config::is_workspace_trusted);
|
||||
@@ -2689,6 +2693,36 @@ mod tests {
|
||||
)
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn auto_review_worker_keeps_the_callers_sealed_environment() {
|
||||
use crate::test_support::{EnvVarGuard, lock_test_env};
|
||||
use std::time::Duration;
|
||||
|
||||
let _lock = lock_test_env();
|
||||
let home = tempfile::tempdir().expect("test home");
|
||||
let workspace = tempfile::tempdir().expect("workspace");
|
||||
let _home = EnvVarGuard::set("CODEWHALE_HOME", home.path());
|
||||
crate::config::save_workspace_trust(workspace.path()).expect("save trust");
|
||||
|
||||
let context = tokio::time::timeout(
|
||||
Duration::from_secs(2),
|
||||
AutoReviewContext::from_tool_call_async(
|
||||
"read_file",
|
||||
&json!({ "path": "README.md" }),
|
||||
RunOrigin::Interactive,
|
||||
ApprovalMode::Auto,
|
||||
Some(workspace.path()),
|
||||
),
|
||||
)
|
||||
.await
|
||||
.expect("worker must not wait for its awaiting caller's environment lock")
|
||||
.expect("evidence");
|
||||
assert!(
|
||||
context.workspace_trusted,
|
||||
"worker must read the sealed trust file"
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn auto_review_filesystem_evidence_does_not_park_runtime() {
|
||||
|
||||
@@ -157,6 +157,7 @@ struct SharedPending {
|
||||
struct PersistRequestSender {
|
||||
shared: Arc<std::sync::Mutex<SharedPending>>,
|
||||
cmd_tx: mpsc::Sender<ActorCommand>,
|
||||
health: SessionSaveHealth,
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for PersistRequestSender {
|
||||
@@ -169,6 +170,7 @@ impl std::fmt::Debug for PersistRequestSender {
|
||||
struct PersistRequestReceiver {
|
||||
shared: Arc<std::sync::Mutex<SharedPending>>,
|
||||
cmd_rx: mpsc::Receiver<ActorCommand>,
|
||||
health: SessionSaveHealth,
|
||||
}
|
||||
|
||||
/// Single construction seam for the production persistence request channel.
|
||||
@@ -179,12 +181,18 @@ struct PersistRequestReceiver {
|
||||
fn persistence_request_channel() -> (PersistRequestSender, PersistRequestReceiver) {
|
||||
let (cmd_tx, cmd_rx) = mpsc::channel(ACTOR_COMMAND_CAPACITY);
|
||||
let shared = Arc::new(std::sync::Mutex::new(SharedPending::default()));
|
||||
let health = SessionSaveHealth::default();
|
||||
(
|
||||
PersistRequestSender {
|
||||
shared: Arc::clone(&shared),
|
||||
cmd_tx,
|
||||
health: health.clone(),
|
||||
},
|
||||
PersistRequestReceiver {
|
||||
shared,
|
||||
cmd_rx,
|
||||
health,
|
||||
},
|
||||
PersistRequestReceiver { shared, cmd_rx },
|
||||
)
|
||||
}
|
||||
|
||||
@@ -213,6 +221,11 @@ pub struct PersistActorHandle {
|
||||
}
|
||||
|
||||
impl PersistActorHandle {
|
||||
/// Whether the latest save of each session document landed.
|
||||
pub(crate) fn session_save_health(&self) -> SaveHealthReading {
|
||||
self.tx.health.reading()
|
||||
}
|
||||
|
||||
/// Queue a persistence request without blocking. The request is
|
||||
/// coalesced into the shared pending state immediately (latest-wins per
|
||||
/// session), so repeated snapshots of one session retain only the
|
||||
@@ -357,6 +370,12 @@ fn request_label(request: &PersistRequest) -> &'static str {
|
||||
|
||||
/// Queue persistence and report whether the actor accepted ownership. Work
|
||||
/// Graph projections use this acknowledgement as their publish boundary.
|
||||
/// [`PersistActorHandle::session_save_health`] of the global actor; `None`
|
||||
/// before it starts.
|
||||
pub(crate) fn session_save_health() -> Option<SaveHealthReading> {
|
||||
ACTOR_TX.get().map(PersistActorHandle::session_save_health)
|
||||
}
|
||||
|
||||
pub fn try_persist(request: PersistRequest) -> bool {
|
||||
ACTOR_TX
|
||||
.get()
|
||||
@@ -390,8 +409,9 @@ pub fn spawn_persistence_actor(
|
||||
manager: &SessionManager,
|
||||
pending: &mut PendingState,
|
||||
unreported: &mut FlushReport,
|
||||
health: &SessionSaveHealth,
|
||||
) {
|
||||
let cycle = flush_inner(manager, pending);
|
||||
let cycle = flush_inner(manager, pending, health);
|
||||
log_flush_failures(&cycle);
|
||||
unreported.merge(cycle);
|
||||
}
|
||||
@@ -404,15 +424,15 @@ pub fn spawn_persistence_actor(
|
||||
match command {
|
||||
ActorCommand::WorkReady => {
|
||||
if !pending.is_empty() {
|
||||
flush_cycle(&manager, &mut pending, &mut unreported);
|
||||
flush_cycle(&manager, &mut pending, &mut unreported, &rx.health);
|
||||
}
|
||||
}
|
||||
ActorCommand::FlushAndReport { reply } => {
|
||||
flush_cycle(&manager, &mut pending, &mut unreported);
|
||||
flush_cycle(&manager, &mut pending, &mut unreported, &rx.health);
|
||||
let _ = reply.send(std::mem::take(&mut unreported));
|
||||
}
|
||||
ActorCommand::Shutdown => {
|
||||
flush_cycle(&manager, &mut pending, &mut unreported);
|
||||
flush_cycle(&manager, &mut pending, &mut unreported, &rx.health);
|
||||
return;
|
||||
}
|
||||
}
|
||||
@@ -420,7 +440,7 @@ pub fn spawn_persistence_actor(
|
||||
// Every sender is gone — final flush and exit. Each absorb
|
||||
// guarantees a queued WorkReady, so this is normally empty.
|
||||
let mut pending = rx.take_pending();
|
||||
flush_cycle(&manager, &mut pending, &mut unreported);
|
||||
flush_cycle(&manager, &mut pending, &mut unreported, &rx.health);
|
||||
},
|
||||
);
|
||||
|
||||
@@ -561,7 +581,11 @@ impl PendingState {
|
||||
/// commit clears its session's checkpoint only after that session's own
|
||||
/// write succeeded, so a failed save always leaves the crash-recovery
|
||||
/// checkpoint in place.
|
||||
fn flush_inner(manager: &SessionManager, pending: &mut PendingState) -> FlushReport {
|
||||
fn flush_inner(
|
||||
manager: &SessionManager,
|
||||
pending: &mut PendingState,
|
||||
health: &SessionSaveHealth,
|
||||
) -> FlushReport {
|
||||
let mut report = FlushReport::default();
|
||||
let mut record = |what: String, result: std::io::Result<()>| match result {
|
||||
Ok(()) => report.completed += 1,
|
||||
@@ -569,13 +593,13 @@ fn flush_inner(manager: &SessionManager, pending: &mut PendingState) -> FlushRep
|
||||
};
|
||||
|
||||
for (session_id, session) in std::mem::take(&mut pending.sessions) {
|
||||
record(
|
||||
format!("session:{session_id}"),
|
||||
manager.save_session_owned(session).map(|_| ()),
|
||||
);
|
||||
let result = manager.save_session_owned(session).map(|_| ());
|
||||
health.record(&session_id, &result);
|
||||
record(format!("session:{session_id}"), result);
|
||||
}
|
||||
for (session_id, session) in std::mem::take(&mut pending.completed_commits) {
|
||||
let commit_result = manager.save_session_owned(session);
|
||||
health.record(&session_id, &commit_result);
|
||||
let save_succeeded = commit_result.is_ok();
|
||||
record(
|
||||
format!("completed-commit:{session_id}"),
|
||||
@@ -620,6 +644,61 @@ fn flush_inner(manager: &SessionManager, pending: &mut PendingState) -> FlushRep
|
||||
report
|
||||
}
|
||||
|
||||
/// Which session documents' latest save failed. Only document saves count: a
|
||||
/// failed checkpoint, queue or cleanup write does not lose the conversation.
|
||||
/// A later successful save of the same session clears its entry, so a
|
||||
/// transient failure does not leave a standing alarm.
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub(crate) struct SessionSaveHealth(Arc<std::sync::Mutex<SaveHealthState>>);
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
struct SaveHealthState {
|
||||
/// Bumped whenever `failing` changes, so a poller can tell what is new.
|
||||
generation: u64,
|
||||
failing: BTreeMap<String, std::io::ErrorKind>,
|
||||
}
|
||||
|
||||
/// One reading of [`SessionSaveHealth`].
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub(crate) struct SaveHealthReading {
|
||||
pub(crate) generation: u64,
|
||||
/// A session whose latest save failed, and how, while any has.
|
||||
pub(crate) failing: Option<(String, std::io::ErrorKind)>,
|
||||
}
|
||||
|
||||
impl SessionSaveHealth {
|
||||
fn record<T>(&self, session_id: &str, result: &std::io::Result<T>) {
|
||||
let mut state = self
|
||||
.0
|
||||
.lock()
|
||||
.unwrap_or_else(|poisoned| poisoned.into_inner());
|
||||
let changed = match result {
|
||||
Ok(_) => state.failing.remove(session_id).is_some(),
|
||||
Err(error) => {
|
||||
state.failing.insert(session_id.to_string(), error.kind()) != Some(error.kind())
|
||||
}
|
||||
};
|
||||
if changed {
|
||||
state.generation += 1;
|
||||
}
|
||||
}
|
||||
|
||||
fn reading(&self) -> SaveHealthReading {
|
||||
let state = self
|
||||
.0
|
||||
.lock()
|
||||
.unwrap_or_else(|poisoned| poisoned.into_inner());
|
||||
SaveHealthReading {
|
||||
generation: state.generation,
|
||||
failing: state
|
||||
.failing
|
||||
.iter()
|
||||
.next()
|
||||
.map(|(id, kind)| (id.clone(), *kind)),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Surface flush failures in the log for write cycles that have no caller
|
||||
/// waiting on a [`FlushReport`].
|
||||
fn log_flush_failures(report: &FlushReport) {
|
||||
@@ -917,6 +996,60 @@ mod tests {
|
||||
task.await.expect("persistence actor join");
|
||||
}
|
||||
|
||||
/// The health the UI polls follows real saves through the actor: a failed
|
||||
/// checkpoint does not count as a lost conversation, a failed document
|
||||
/// save does, and a later successful save of that session clears it.
|
||||
#[tokio::test]
|
||||
async fn session_save_health_follows_document_saves_through_the_actor() {
|
||||
let tmp = tempfile::tempdir().expect("tempdir");
|
||||
let sessions_dir = tmp.path().join("sessions");
|
||||
let manager = SessionManager::new(sessions_dir.clone()).expect("manager");
|
||||
std::fs::write(sessions_dir.join("checkpoints"), b"not a directory")
|
||||
.expect("block checkpoints dir");
|
||||
let session = crate::session_manager::create_saved_session_with_mode(
|
||||
&[],
|
||||
"deepseek-v4-pro",
|
||||
tmp.path(),
|
||||
0,
|
||||
None,
|
||||
Some("agent"),
|
||||
);
|
||||
let session_id = session.metadata.id.clone();
|
||||
// A directory where the document goes makes its save fail.
|
||||
let blocker = sessions_dir.join(format!("{session_id}.json"));
|
||||
std::fs::create_dir(&blocker).expect("block session document");
|
||||
let (handle, task) = spawn_persistence_actor(manager);
|
||||
let flush = |handle: &PersistActorHandle| {
|
||||
let (reply_tx, reply_rx) = tokio::sync::oneshot::channel();
|
||||
handle.try_send(PersistRequest::FlushAndReport { reply: reply_tx });
|
||||
reply_rx
|
||||
};
|
||||
|
||||
handle.try_send(PersistRequest::SaveCheckpoint {
|
||||
session: session.clone(),
|
||||
});
|
||||
flush(&handle).await.expect("flush");
|
||||
assert_eq!(handle.session_save_health().failing, None);
|
||||
|
||||
handle.try_send(PersistRequest::SessionSnapshot(session.clone()));
|
||||
flush(&handle).await.expect("flush");
|
||||
let failed = handle.session_save_health();
|
||||
assert_eq!(
|
||||
failed.failing.as_ref().map(|(id, _)| id.as_str()),
|
||||
Some(session_id.as_str())
|
||||
);
|
||||
|
||||
std::fs::remove_dir(&blocker).expect("unblock");
|
||||
handle.try_send(PersistRequest::SessionSnapshot(session));
|
||||
flush(&handle).await.expect("flush");
|
||||
let healed = handle.session_save_health();
|
||||
assert_eq!(healed.failing, None, "a later save clears the failure");
|
||||
assert!(healed.generation > failed.generation);
|
||||
|
||||
handle.try_send(PersistRequest::Shutdown);
|
||||
task.await.expect("persistence actor join");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn flush_and_report_propagates_write_failures() {
|
||||
let tmp = tempfile::tempdir().expect("tempdir");
|
||||
@@ -1227,7 +1360,7 @@ mod tests {
|
||||
});
|
||||
drop(lease); // The old window changed session before the actor ran.
|
||||
assert!(manager.acquire_offline_queue_lease("session-A").is_err());
|
||||
let report = flush_inner(&manager, &mut pending);
|
||||
let report = flush_inner(&manager, &mut pending, &SessionSaveHealth::default());
|
||||
assert!(report.failures.is_empty(), "draft write failed: {report:?}");
|
||||
assert_eq!(report.completed, 1);
|
||||
let _next_editor = manager
|
||||
|
||||
@@ -90,7 +90,12 @@ class PetNative {
|
||||
this.world.voices = [];
|
||||
return this.world.frame.timeMs;
|
||||
}
|
||||
observeEngine(metadataJSON, timeMs) { this.engine.observe(JSON.parse(metadataJSON), timeMs); }
|
||||
/** A rejected observation leaves the reducer exactly as it was, like a batch. */
|
||||
observeEngine(metadataJSON, timeMs) {
|
||||
const next = this.engine.clone();
|
||||
next.observe(JSON.parse(metadataJSON), timeMs);
|
||||
this.engine = next;
|
||||
}
|
||||
observeEngineBatch(metadataJSON, timeMs) {
|
||||
const events = JSON.parse(metadataJSON);
|
||||
if (!Array.isArray(events) || events.length > 64)
|
||||
@@ -1846,6 +1851,9 @@ class PetEngineTelemetry {
|
||||
next.completedTurns = new Set(this.completedTurns);
|
||||
return next;
|
||||
}
|
||||
/** Mutates in place: clock, freshness and the event window change before
|
||||
* type-specific fields are checked. Callers that continue after a throw must
|
||||
* observe on a clone() and swap it in on success, as PetNative does. */
|
||||
observe(value, at) {
|
||||
if (!Number.isFinite(at) || at < this.lastTime || at > pet_sim_js_1.PET_MAX_SECONDS * 1000)
|
||||
throw new Error('Invalid Engine pet clock.');
|
||||
|
||||
@@ -510,6 +510,19 @@ impl SessionPickerView {
|
||||
fn delete_selected(&mut self) -> Option<ViewEvent> {
|
||||
let session = self.selected_session().cloned()?;
|
||||
let manager = SessionManager::default_location().ok()?;
|
||||
// A deleted document the live session still owns refuses every later
|
||||
// save, so the conversation on screen would silently stop persisting.
|
||||
let refusal = if self.current_session_id.as_deref() == Some(session.id.as_str()) {
|
||||
Some(MessageId::SessionsDeleteOpenHere)
|
||||
} else if manager.is_session_live_anywhere(&session.id) {
|
||||
Some(MessageId::SessionsDeleteOpenElsewhere)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
if let Some(reason) = refusal {
|
||||
self.status = Some(tr(self.locale, reason).into_owned());
|
||||
return None;
|
||||
}
|
||||
if let Err(err) = manager.delete_session(&session.id) {
|
||||
self.status = Some(
|
||||
tr(self.locale, MessageId::SessionsDeleteFailed)
|
||||
@@ -1633,6 +1646,59 @@ mod tests {
|
||||
view
|
||||
}
|
||||
|
||||
/// Deleting the session open in this window would leave it unable to
|
||||
/// save anything afterwards, with nothing on screen saying so.
|
||||
#[test]
|
||||
fn delete_refuses_the_session_open_here() {
|
||||
let _lock = crate::test_support::lock_test_env();
|
||||
let tmp = tempfile::tempdir().expect("tempdir");
|
||||
let _home = crate::test_support::EnvVarGuard::set("CODEWHALE_HOME", tmp.path());
|
||||
let manager = SessionManager::default_location().expect("session manager");
|
||||
let mut saved = saved_session_with_messages(vec![text_message("user", "hello")]);
|
||||
saved.metadata.id = "session-open-here".to_string();
|
||||
manager.save_session(&saved).expect("save session");
|
||||
let mut view = picker_with(vec![saved.metadata.clone()], None);
|
||||
view.current_session_id = Some("session-open-here".to_string());
|
||||
|
||||
assert!(view.delete_selected().is_none(), "no deletion event");
|
||||
|
||||
assert_eq!(
|
||||
view.status.as_deref(),
|
||||
Some(tr(view.locale, MessageId::SessionsDeleteOpenHere).as_ref())
|
||||
);
|
||||
assert_eq!(view.sessions.len(), 1, "the row stays listed");
|
||||
assert!(
|
||||
manager.load_session("session-open-here").is_ok(),
|
||||
"the document is not deleted"
|
||||
);
|
||||
}
|
||||
|
||||
/// A session another Codewhale window holds open would be written back
|
||||
/// by that window's next autosave, so deleting it here is refused.
|
||||
#[test]
|
||||
fn delete_refuses_a_session_open_in_another_window() {
|
||||
let _lock = crate::test_support::lock_test_env();
|
||||
let tmp = tempfile::tempdir().expect("tempdir");
|
||||
let _home = crate::test_support::EnvVarGuard::set("CODEWHALE_HOME", tmp.path());
|
||||
let manager = SessionManager::default_location().expect("session manager");
|
||||
let mut saved = saved_session_with_messages(vec![text_message("user", "hello")]);
|
||||
saved.metadata.id = "session-open-elsewhere".to_string();
|
||||
manager.save_session(&saved).expect("save session");
|
||||
let _lease = manager.hold_live_lease_elsewhere("session-open-elsewhere");
|
||||
let mut view = picker_with(vec![saved.metadata.clone()], None);
|
||||
|
||||
assert!(view.delete_selected().is_none(), "no deletion event");
|
||||
|
||||
assert_eq!(
|
||||
view.status.as_deref(),
|
||||
Some(tr(view.locale, MessageId::SessionsDeleteOpenElsewhere).as_ref())
|
||||
);
|
||||
assert!(
|
||||
manager.load_session("session-open-elsewhere").is_ok(),
|
||||
"the document is not deleted"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rename_selected_persists_and_emits_saved_metadata() {
|
||||
let _lock = crate::test_support::lock_test_env();
|
||||
|
||||
@@ -1482,28 +1482,19 @@ async fn apply_command_result_inner(
|
||||
return Ok(false);
|
||||
}
|
||||
};
|
||||
// A managed record resumes through the manager so its repair is
|
||||
// hydrated, applied, and persisted in place. A foreign `/load`
|
||||
// file is not ours to rewrite: hydrate its journal projection
|
||||
// and repair in memory only.
|
||||
let session = match SessionManager::default_location() {
|
||||
Ok(manager) if manager.owns_session_path(&parsed.metadata.id, &path) => {
|
||||
match manager.resume_session(&parsed.metadata.id) {
|
||||
Ok(recovery) => recovery.session,
|
||||
Err(err) => {
|
||||
crate::tui::ui::session_state::surface_session_load_failure(
|
||||
app,
|
||||
format!("Failed to resume session {}: {err}", path.display()),
|
||||
);
|
||||
return Ok(false);
|
||||
}
|
||||
}
|
||||
}
|
||||
_ => {
|
||||
let mut session = parsed;
|
||||
session.ensure_journal();
|
||||
crate::session_manager::repair_recovered_session(&mut session);
|
||||
session
|
||||
// `/load` shares the attach contract of `resume` and the
|
||||
// picker (`SessionManager::attach_session_file`); the lease is
|
||||
// committed only once the session is applied.
|
||||
let attached = SessionManager::default_location()
|
||||
.and_then(|manager| manager.attach_session_file(parsed, &path));
|
||||
let (session, lease) = match attached {
|
||||
Ok(attached) => attached,
|
||||
Err(err) => {
|
||||
crate::tui::ui::session_state::surface_session_load_failure(
|
||||
app,
|
||||
format!("Failed to resume session {}: {err}", path.display()),
|
||||
);
|
||||
return Ok(false);
|
||||
}
|
||||
};
|
||||
let fresh_config =
|
||||
@@ -1527,7 +1518,10 @@ async fn apply_command_result_inner(
|
||||
fresh_config,
|
||||
true,
|
||||
) {
|
||||
Ok(outcome) => outcome,
|
||||
Ok(outcome) => {
|
||||
lease.commit();
|
||||
outcome
|
||||
}
|
||||
Err(err) => {
|
||||
crate::tui::ui::session_state::surface_session_load_failure(
|
||||
app,
|
||||
|
||||
@@ -577,6 +577,8 @@ pub(crate) fn coalesce_scroll_burst(
|
||||
latest = *next;
|
||||
}
|
||||
_ => {
|
||||
// Back to the head, not the tail: `pending` may already hold
|
||||
// later input (typed text, Enter), and this event came first.
|
||||
pending.push_front(next_observed);
|
||||
break;
|
||||
}
|
||||
@@ -600,6 +602,76 @@ pub(crate) fn transcript_cadence_tier(
|
||||
)
|
||||
}
|
||||
|
||||
/// Fold every queued `Resize` behind the one in hand into the final size, so
|
||||
/// one clear and redraw serves the whole drag (#65).
|
||||
///
|
||||
/// The first non-resize event ends the fold and goes back to the head of the
|
||||
/// queue, ahead of any later input already drained into `pending`.
|
||||
pub(crate) fn coalesce_resize_burst(
|
||||
width: u16,
|
||||
height: u16,
|
||||
input: &TerminalInputPump,
|
||||
pending: &mut VecDeque<ObservedTerminalEvent>,
|
||||
) -> std::io::Result<(u16, u16)> {
|
||||
let (mut final_w, mut final_h) = (width, height);
|
||||
while let Some(next_observed) = try_next_terminal_event(input, pending)? {
|
||||
if let Event::Resize(w, h) = next_observed.event {
|
||||
final_w = w;
|
||||
final_h = h;
|
||||
} else {
|
||||
pending.push_front(next_observed);
|
||||
break;
|
||||
}
|
||||
}
|
||||
Ok((final_w, final_h))
|
||||
}
|
||||
|
||||
/// Toast identity for a failing session save, so recovery can retire it.
|
||||
const SESSION_SAVE_FAILURE_TOAST: &str = "session-save-failure";
|
||||
|
||||
/// Keep the save-failure notice in step with the persistence actor's session
|
||||
/// save health. Saves run off the UI thread, so a full disk or an unwritable
|
||||
/// sessions directory used to reach only the log while the user kept working
|
||||
/// unsaved. The notice stays while a session's latest save is failing and is
|
||||
/// withdrawn once a later save lands.
|
||||
pub(crate) fn surface_session_save_health(
|
||||
app: &mut App,
|
||||
reading: Option<crate::tui::persistence_actor::SaveHealthReading>,
|
||||
seen: &mut u64,
|
||||
) {
|
||||
let Some(reading) = reading.filter(|reading| reading.generation != *seen) else {
|
||||
return;
|
||||
};
|
||||
*seen = reading.generation;
|
||||
app.retire_event_notices(SESSION_SAVE_FAILURE_TOAST);
|
||||
if let Some((session_id, kind)) = reading.failing {
|
||||
let text = app
|
||||
.tr(MessageId::SessionSaveFailed)
|
||||
.replace("{id}", crate::session_manager::truncate_id(&session_id))
|
||||
.replace("{error}", &kind.to_string());
|
||||
// Standing, not timed: it must outlast the failure, and only the
|
||||
// recovery reading above withdraws it.
|
||||
app.push_status_toast_record(StatusToast::standing(
|
||||
text,
|
||||
StatusToastLevel::Error,
|
||||
SESSION_SAVE_FAILURE_TOAST,
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
/// The exit line when a session's latest save failed and was never replaced
|
||||
/// by a successful one.
|
||||
pub(crate) fn shutdown_persistence_notice(
|
||||
locale: codewhale_localization::Locale,
|
||||
reading: &crate::tui::persistence_actor::SaveHealthReading,
|
||||
) -> Option<String> {
|
||||
reading.failing.as_ref().map(|(session_id, kind)| {
|
||||
codewhale_localization::tr(locale, MessageId::SessionSaveFailedAtExit)
|
||||
.replace("{id}", session_id)
|
||||
.replace("{error}", &kind.to_string())
|
||||
})
|
||||
}
|
||||
|
||||
/// Run the interactive TUI event loop.
|
||||
///
|
||||
/// # Examples
|
||||
@@ -820,28 +892,38 @@ pub async fn run_tui(
|
||||
&& let Ok(manager) = SessionManager::default_location()
|
||||
{
|
||||
// Try to load by prefix or full ID
|
||||
let load_result: std::io::Result<Option<crate::session_manager::SavedSession>> =
|
||||
let load_result: std::io::Result<
|
||||
Option<(
|
||||
crate::session_manager::SavedSession,
|
||||
crate::session_manager::SessionLease,
|
||||
)>,
|
||||
> =
|
||||
// `attach_*` reserves the session's live lease first, and refuses
|
||||
// a session another window has open instead of becoming its second
|
||||
// autosaving writer. The lease is committed once the session is
|
||||
// applied.
|
||||
if session_id == "latest" {
|
||||
// Special case: resume the most recent session in this workspace.
|
||||
match manager.get_latest_session_for_workspace(&options.workspace) {
|
||||
Ok(Some(meta)) => manager
|
||||
.resume_session(&meta.id)
|
||||
.map(|recovery| Some(recovery.session)),
|
||||
.attach_session(&meta.id)
|
||||
.map(|(recovery, lease)| Some((recovery.session, lease))),
|
||||
Ok(None) => Ok(None),
|
||||
Err(e) => Err(e),
|
||||
}
|
||||
} else {
|
||||
manager
|
||||
.resume_session_by_prefix(session_id)
|
||||
.map(|recovery| Some(recovery.session))
|
||||
.attach_session_by_prefix(session_id)
|
||||
.map(|(recovery, lease)| Some((recovery.session, lease)))
|
||||
};
|
||||
|
||||
match load_result {
|
||||
Ok(Some(saved)) => match manager.load_session_goal(&saved.metadata.id) {
|
||||
Ok(Some((saved, lease))) => match manager.load_session_goal(&saved.metadata.id) {
|
||||
Ok(goal) => {
|
||||
let saved_id = saved.metadata.id.clone();
|
||||
match apply_loaded_session_with_goal(&mut app, config, saved, goal.as_ref()) {
|
||||
Ok(()) => {
|
||||
lease.commit();
|
||||
app.status_message = Some(format!(
|
||||
"Resumed session: {}",
|
||||
crate::session_manager::truncate_id(&saved_id)
|
||||
@@ -1173,6 +1255,7 @@ pub async fn run_tui(
|
||||
// applied), the checkpoint is the only durable record of that work:
|
||||
// clearing it here unconditionally could erase in-flight progress that
|
||||
// never reached a snapshot, so it survives for startup recovery review.
|
||||
let mut shutdown_save_health = None;
|
||||
if let Some((handle, task)) = persistence_runtime {
|
||||
// A quit key can leave the frame before its usual queue comparison.
|
||||
// Capture the final edited draft before the shutdown durability barrier.
|
||||
@@ -1201,6 +1284,9 @@ pub async fn run_tui(
|
||||
"session persistence reported write failures during shutdown",
|
||||
);
|
||||
}
|
||||
// Read after the final flush: whether each session's latest save
|
||||
// landed, not every failure this run has ever seen.
|
||||
shutdown_save_health = Some(handle.session_save_health());
|
||||
handle.try_send(PersistRequest::Shutdown);
|
||||
let _ = task.await;
|
||||
}
|
||||
@@ -1260,7 +1346,24 @@ pub async fn run_tui(
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(notice) = shutdown_save_health
|
||||
.as_ref()
|
||||
.and_then(|reading| shutdown_persistence_notice(app.ui_locale, reading))
|
||||
{
|
||||
// Primary screen, like the settings failures above.
|
||||
#[allow(clippy::print_stderr)]
|
||||
{
|
||||
eprintln!("{notice}");
|
||||
}
|
||||
}
|
||||
|
||||
// `codewhale resume <id>` for a document that never reached disk (every
|
||||
// save failed, or nothing was ever saved) only fails with NotFound.
|
||||
let session_document_exists = app.current_session_id.as_deref().is_some_and(|id| {
|
||||
SessionManager::default_location().is_ok_and(|manager| manager.session_document_exists(id))
|
||||
});
|
||||
if result.is_ok()
|
||||
&& session_document_exists
|
||||
&& let Some(hint) = resume_hint_text(
|
||||
app.ui_locale,
|
||||
app.current_session_id.as_deref(),
|
||||
@@ -1677,6 +1780,7 @@ pub(crate) async fn run_event_loop(
|
||||
}
|
||||
|
||||
let mut pending_subagent_list_refresh = false;
|
||||
let mut session_save_health_seen = 0u64;
|
||||
|
||||
loop {
|
||||
// #6169: first statement of every iteration. The job-control handler can
|
||||
@@ -2024,6 +2128,12 @@ pub(crate) async fn run_event_loop(
|
||||
deliver_constitution_draft_result(app, model_label, draft_locale, outcome);
|
||||
}
|
||||
|
||||
surface_session_save_health(
|
||||
app,
|
||||
crate::tui::persistence_actor::session_save_health(),
|
||||
&mut session_save_health_seen,
|
||||
);
|
||||
|
||||
// Discovery and callback delivery never park terminal input.
|
||||
poll_mcp_login(app);
|
||||
poll_mcp_retries(app);
|
||||
@@ -4747,25 +4857,12 @@ pub(crate) async fn run_event_loop(
|
||||
// common "stale art on the right edge" symptom (#65) caused by
|
||||
// the diff renderer skipping cells that match a stale back
|
||||
// buffer between intermediate sizes.
|
||||
let mut final_w = width;
|
||||
let mut final_h = height;
|
||||
while let Some(next_observed) =
|
||||
try_next_terminal_event(&terminal_input, &mut pending_terminal_events)?
|
||||
{
|
||||
match next_observed.event {
|
||||
Event::Resize(w, h) => {
|
||||
final_w = w;
|
||||
final_h = h;
|
||||
}
|
||||
other => {
|
||||
pending_terminal_events.push_front(ObservedTerminalEvent::new(
|
||||
other,
|
||||
next_observed.observed_at,
|
||||
));
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
let (final_w, final_h) = coalesce_resize_burst(
|
||||
width,
|
||||
height,
|
||||
&terminal_input,
|
||||
&mut pending_terminal_events,
|
||||
)?;
|
||||
|
||||
if final_w == 0 || final_h == 0 {
|
||||
tracing::debug!(
|
||||
|
||||
@@ -1684,8 +1684,10 @@ pub(crate) async fn handle_view_events(
|
||||
}
|
||||
};
|
||||
|
||||
match manager.resume_session(&session_id) {
|
||||
Ok(recovery) => {
|
||||
// Another window's open session is refused, not attached
|
||||
// as a second autosaving writer.
|
||||
match manager.attach_session(&session_id) {
|
||||
Ok((recovery, lease)) => {
|
||||
let session = recovery.session;
|
||||
let next_config = config.clone();
|
||||
let message_count = session.metadata.message_count;
|
||||
@@ -1699,7 +1701,13 @@ pub(crate) async fn handle_view_events(
|
||||
next_config,
|
||||
false,
|
||||
) {
|
||||
Ok(outcome) => outcome,
|
||||
Ok(outcome) => {
|
||||
// Only now does this window give up the
|
||||
// session it had open; a failed restore
|
||||
// above keeps that session's lease.
|
||||
lease.commit();
|
||||
outcome
|
||||
}
|
||||
Err(err) => {
|
||||
crate::tui::ui::session_state::surface_session_load_failure(
|
||||
app,
|
||||
|
||||
@@ -31084,6 +31084,169 @@ fn a_scroll_burst_is_folded_into_one_frame() {
|
||||
));
|
||||
}
|
||||
|
||||
fn observed_key(c: char) -> ObservedTerminalEvent {
|
||||
ObservedTerminalEvent::new(
|
||||
Event::Key(KeyEvent::new(KeyCode::Char(c), KeyModifiers::NONE)),
|
||||
Instant::now(),
|
||||
)
|
||||
}
|
||||
|
||||
fn idle_input_pump() -> TerminalInputPump {
|
||||
let (_tx, rx) = std::sync::mpsc::channel();
|
||||
TerminalInputPump {
|
||||
rx,
|
||||
stop: std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)),
|
||||
paused: std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)),
|
||||
paused_ack: std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)),
|
||||
handle: None,
|
||||
last_alive_at: std::cell::Cell::new(Instant::now()),
|
||||
}
|
||||
}
|
||||
|
||||
fn pending_key_chars(pending: &VecDeque<ObservedTerminalEvent>) -> String {
|
||||
pending
|
||||
.iter()
|
||||
.filter_map(|observed| match &observed.event {
|
||||
Event::Key(KeyEvent {
|
||||
code: KeyCode::Char(c),
|
||||
..
|
||||
}) => Some(*c),
|
||||
_ => None,
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Typing while the wheel scrolls: the loop has already drained the burst and
|
||||
/// the keys into `pending`. The key that ends the burst goes back to the head,
|
||||
/// so the composer receives "abc", not "bca".
|
||||
#[test]
|
||||
fn a_scroll_burst_keeps_following_input_in_order() {
|
||||
let mut app = create_test_app();
|
||||
app.launch.visible = false;
|
||||
app.viewport.last_transcript_area = Some(Rect::new(0, 0, 80, 20));
|
||||
let scroll = crossterm::event::MouseEvent {
|
||||
kind: crossterm::event::MouseEventKind::ScrollDown,
|
||||
column: 10,
|
||||
row: 5,
|
||||
modifiers: KeyModifiers::NONE,
|
||||
};
|
||||
let input = idle_input_pump();
|
||||
let mut pending: VecDeque<ObservedTerminalEvent> = VecDeque::new();
|
||||
pending.push_back(ObservedTerminalEvent::new(
|
||||
Event::Mouse(scroll),
|
||||
Instant::now(),
|
||||
));
|
||||
for c in ['a', 'b', 'c'] {
|
||||
pending.push_back(observed_key(c));
|
||||
}
|
||||
|
||||
super::event_loop::coalesce_scroll_burst(&mut app, scroll, &input, &mut pending)
|
||||
.expect("coalesce");
|
||||
|
||||
assert_eq!(pending_key_chars(&pending), "abc");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_resize_burst_keeps_following_input_in_order() {
|
||||
let input = idle_input_pump();
|
||||
let mut pending: VecDeque<ObservedTerminalEvent> = VecDeque::new();
|
||||
pending.push_back(ObservedTerminalEvent::new(
|
||||
Event::Resize(120, 40),
|
||||
Instant::now(),
|
||||
));
|
||||
pending.push_back(observed_key('a'));
|
||||
pending.push_back(observed_key('b'));
|
||||
|
||||
let size =
|
||||
super::event_loop::coalesce_resize_burst(100, 30, &input, &mut pending).expect("coalesce");
|
||||
|
||||
assert_eq!(size, (120, 40), "the final queued size wins");
|
||||
assert_eq!(pending_key_chars(&pending), "ab");
|
||||
}
|
||||
|
||||
/// A failing session save reaches the user, not only the log, and the notice
|
||||
/// is withdrawn once a later save of that session lands.
|
||||
#[test]
|
||||
fn a_failing_session_save_shows_an_error_until_a_save_lands() {
|
||||
use crate::tui::persistence_actor::SaveHealthReading;
|
||||
let mut app = create_test_app();
|
||||
let mut seen = 0;
|
||||
let failing = SaveHealthReading {
|
||||
generation: 1,
|
||||
failing: Some((
|
||||
"toast-probe".to_string(),
|
||||
std::io::ErrorKind::PermissionDenied,
|
||||
)),
|
||||
};
|
||||
let save_errors = |app: &App| {
|
||||
app.status_toasts
|
||||
.iter()
|
||||
.filter(|toast| toast.level == StatusToastLevel::Error)
|
||||
.count()
|
||||
};
|
||||
|
||||
super::event_loop::surface_session_save_health(&mut app, Some(failing.clone()), &mut seen);
|
||||
assert_eq!(seen, 1);
|
||||
assert_eq!(save_errors(&app), 1, "a failing save is visible");
|
||||
assert!(
|
||||
app.status_toasts
|
||||
.iter()
|
||||
.any(|toast| toast.text.contains("toast-pr")),
|
||||
"{:?}",
|
||||
app.status_toasts
|
||||
);
|
||||
|
||||
// Polling the same reading again does not repeat it.
|
||||
super::event_loop::surface_session_save_health(&mut app, Some(failing), &mut seen);
|
||||
assert_eq!(save_errors(&app), 1);
|
||||
|
||||
// The failure outlives any toast lifetime: well past the sticky TTL, and
|
||||
// behind a full queue of newer notices, it is still there and shown once
|
||||
// they expire, because nothing has recovered.
|
||||
for toast in app.status_toasts.iter_mut() {
|
||||
toast.created_at = std::time::Instant::now()
|
||||
- std::time::Duration::from_millis(App::STICKY_ERROR_TTL_MS * 10);
|
||||
}
|
||||
for i in 0..30 {
|
||||
app.push_status_toast(format!("newer {i}"), StatusToastLevel::Info, Some(1));
|
||||
}
|
||||
std::thread::sleep(std::time::Duration::from_millis(5));
|
||||
let shown = app
|
||||
.active_status_toast(crate::tui::underwater::ShellPhase::Idle)
|
||||
.expect("a toast is shown");
|
||||
assert!(shown.text.contains("toast-pr"), "{}", shown.text);
|
||||
assert_eq!(save_errors(&app), 1, "a standing failure does not expire");
|
||||
|
||||
let healed = SaveHealthReading {
|
||||
generation: 2,
|
||||
failing: None,
|
||||
};
|
||||
super::event_loop::surface_session_save_health(&mut app, Some(healed), &mut seen);
|
||||
assert_eq!(save_errors(&app), 0, "a later successful save withdraws it");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn shutdown_reports_only_a_save_that_is_still_failing() {
|
||||
use crate::tui::persistence_actor::SaveHealthReading;
|
||||
let locale = codewhale_localization::Locale::En;
|
||||
let healthy = SaveHealthReading {
|
||||
generation: 4,
|
||||
failing: None,
|
||||
};
|
||||
assert_eq!(
|
||||
super::event_loop::shutdown_persistence_notice(locale, &healthy),
|
||||
None,
|
||||
"failures later replaced by a successful save are not reported"
|
||||
);
|
||||
let failing = SaveHealthReading {
|
||||
generation: 5,
|
||||
failing: Some(("session-a".to_string(), std::io::ErrorKind::StorageFull)),
|
||||
};
|
||||
let notice = super::event_loop::shutdown_persistence_notice(locale, &failing)
|
||||
.expect("a failing save produces an exit notice");
|
||||
assert!(notice.contains("session-a"), "{notice}");
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Startup type-ahead integrity (#5925).
|
||||
//
|
||||
|
||||
@@ -112,15 +112,16 @@ fn reasoning_effort_for_route_selection(
|
||||
}
|
||||
}
|
||||
|
||||
/// Settle the classifier usage of a turn whose route then failed, into the
|
||||
/// cost scope that was current when the classifier call was made. A fresh
|
||||
/// token taken here would bill it to whatever session is current by now:
|
||||
/// after a `/new` or session load during the call, a different one.
|
||||
fn settle_failed_parent_route(
|
||||
cost_scope: crate::cost_status::CostScopeToken,
|
||||
error: String,
|
||||
initial_routed_usage: &crate::cost_status::RuntimeUsageBatch,
|
||||
) -> String {
|
||||
crate::cost_status::report_runtime_usage_batch(
|
||||
crate::cost_status::scope_token(),
|
||||
None,
|
||||
initial_routed_usage,
|
||||
);
|
||||
crate::cost_status::report_runtime_usage_batch(cost_scope, None, initial_routed_usage);
|
||||
error
|
||||
}
|
||||
|
||||
@@ -141,6 +142,9 @@ fn settle_failed_parent_route(
|
||||
pub(crate) async fn plan_turn_route(
|
||||
request: TurnRoutePlanRequest<'_>,
|
||||
) -> Result<PlannedTurnRoute, String> {
|
||||
// Taken before the classifier call so its usage settles into the scope
|
||||
// it was spent in, not the one current when a failure is noticed.
|
||||
let cost_scope = crate::cost_status::scope_token();
|
||||
let mut auto_selection = if request.should_auto_resolve {
|
||||
Some(
|
||||
crate::model_routing::resolve_auto_route_with_inventory_for_session_and_cache_policy(
|
||||
@@ -210,6 +214,7 @@ pub(crate) async fn plan_turn_route(
|
||||
Ok(route) => route,
|
||||
Err(err) => {
|
||||
return Err(settle_failed_parent_route(
|
||||
cost_scope,
|
||||
err.to_string(),
|
||||
&initial_routed_usage,
|
||||
));
|
||||
@@ -219,7 +224,11 @@ pub(crate) async fn plan_turn_route(
|
||||
match turn_route.preflight() {
|
||||
Ok(route) => route,
|
||||
Err(err) => {
|
||||
return Err(settle_failed_parent_route(err, &initial_routed_usage));
|
||||
return Err(settle_failed_parent_route(
|
||||
cost_scope,
|
||||
err,
|
||||
&initial_routed_usage,
|
||||
));
|
||||
}
|
||||
}
|
||||
} else {
|
||||
@@ -357,17 +366,43 @@ mod tests {
|
||||
dropped_records: 1,
|
||||
};
|
||||
|
||||
let scope = crate::cost_status::scope_token();
|
||||
assert_eq!(
|
||||
settle_failed_parent_route("route failed".to_string(), &batch),
|
||||
settle_failed_parent_route(scope, "route failed".to_string(), &batch),
|
||||
"route failed"
|
||||
);
|
||||
settle_failed_parent_route("route failed".to_string(), &batch);
|
||||
settle_failed_parent_route(scope, "route failed".to_string(), &batch);
|
||||
let pending = crate::cost_status::drain();
|
||||
assert_eq!(
|
||||
pending.usage_source_fingerprints.len(),
|
||||
2,
|
||||
"both exact classifier outcomes persist, and replay is idempotent"
|
||||
);
|
||||
|
||||
// The classifier ran in one session; `/new` closed it before the
|
||||
// route failed. Its cost belongs to the closed session, never the new.
|
||||
let spent_in = crate::cost_status::scope_token();
|
||||
let _closed = crate::cost_status::close_current_scope();
|
||||
let batch = crate::cost_status::RuntimeUsageBatch {
|
||||
records: batch
|
||||
.records
|
||||
.iter()
|
||||
.cloned()
|
||||
.map(|mut record| {
|
||||
record.source_id = "auto-router:plan-usage-after-new".to_string();
|
||||
record
|
||||
})
|
||||
.collect(),
|
||||
drop_records: Vec::new(),
|
||||
dropped_records: 0,
|
||||
};
|
||||
settle_failed_parent_route(spent_in, "route failed".to_string(), &batch);
|
||||
assert!(
|
||||
crate::cost_status::drain()
|
||||
.usage_source_fingerprints
|
||||
.is_empty(),
|
||||
"classifier cost leaked into the session opened after it was spent"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
+62
-23
@@ -421,7 +421,16 @@ pub(crate) async fn transcribe_local_whisper(audio_samples: &[i16]) -> Result<St
|
||||
}
|
||||
|
||||
fn transcribe_local_whisper_blocking(wav: &[u8]) -> Result<String, String> {
|
||||
let tmp = std::env::temp_dir().join(format!("cw-voice-{}.wav", std::process::id()));
|
||||
// C01-10: a fresh, exclusively created file per call. The old
|
||||
// `cw-voice-<pid>.wav` name was shared by every transcription in the
|
||||
// process (an interim pass, the final pass, a runtime-API dictation) and
|
||||
// predictable in a shared temp dir. Dropping `audio` removes it.
|
||||
let audio = tempfile::Builder::new()
|
||||
.prefix("cw-voice-")
|
||||
.suffix(".wav")
|
||||
.tempfile()
|
||||
.map_err(|e| e.to_string())?;
|
||||
let tmp = audio.path().to_path_buf();
|
||||
std::fs::write(&tmp, wav).map_err(|e| e.to_string())?;
|
||||
// Try each local binary until one succeeds; whisper.cpp outputs to stdout or file.
|
||||
for bin in LOCAL_WHISPER_BINS {
|
||||
@@ -437,7 +446,6 @@ fn transcribe_local_whisper_blocking(wav: &[u8]) -> Result<String, String> {
|
||||
&& out.status.success()
|
||||
{
|
||||
let txt = String::from_utf8_lossy(&out.stdout).trim().to_string();
|
||||
let _ = std::fs::remove_file(&tmp);
|
||||
if !txt.is_empty() {
|
||||
return Ok(txt);
|
||||
}
|
||||
@@ -445,14 +453,12 @@ fn transcribe_local_whisper_blocking(wav: &[u8]) -> Result<String, String> {
|
||||
let sidecar = tmp.with_extension("txt");
|
||||
if let Ok(s) = std::fs::read_to_string(&sidecar) {
|
||||
let _ = std::fs::remove_file(&sidecar);
|
||||
let _ = std::fs::remove_file(&tmp);
|
||||
if !s.trim().is_empty() {
|
||||
return Ok(s.trim().to_string());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
let _ = std::fs::remove_file(&tmp);
|
||||
Err("local whisper not available".into())
|
||||
}
|
||||
|
||||
@@ -467,28 +473,39 @@ pub(crate) async fn transcribe_groq(audio_samples: &[i16]) -> Result<String, Str
|
||||
}
|
||||
|
||||
/// Resolve ASR model/provider preference.
|
||||
/// Priority: explicit config `voice.asr_model` > env `CODEWHALE_ASR_MODEL` > auto-detect (local-whisper > groq > xiaomi).
|
||||
/// Priority: env `CODEWHALE_ASR_MODEL` > auto-detect (local-whisper > groq >
|
||||
/// provider). Known limitation (C01-10): there is no config-file ASR setting
|
||||
/// yet — `config` is accepted for the day one exists and is not read, so the
|
||||
/// env var is the only explicit override.
|
||||
pub(crate) fn resolve_asr_choice(_config: &Config) -> (String, String) {
|
||||
// Check explicit env override first (free, cross-platform)
|
||||
if let Ok(m) = std::env::var("CODEWHALE_ASR_MODEL") {
|
||||
let m = m.trim().to_ascii_lowercase();
|
||||
if m.contains("groq") || m.contains("whisper") {
|
||||
return ("groq".into(), GROQ_ASR_MODEL.into());
|
||||
}
|
||||
if m.contains("local") || m.contains("whisper.cpp") {
|
||||
return ("local-whisper".into(), "tiny".into());
|
||||
}
|
||||
if m.contains("mimo") || m.contains("xiaomi") {
|
||||
return ("provider".into(), ASR_MODEL.into());
|
||||
}
|
||||
let explicit = std::env::var("CODEWHALE_ASR_MODEL")
|
||||
.ok()
|
||||
.and_then(|value| explicit_asr_choice(&value));
|
||||
// Auto-detect best free: local whisper (offline, no key) > Groq free
|
||||
// tier > provider ASR (needs key).
|
||||
let (kind, model) = explicit.unwrap_or_else(|| match detect_free_asr() {
|
||||
"local-whisper" => ("local-whisper", "tiny"),
|
||||
"groq" => ("groq", GROQ_ASR_MODEL),
|
||||
_ => ("provider", ASR_MODEL),
|
||||
});
|
||||
(kind.to_string(), model.to_string())
|
||||
}
|
||||
|
||||
/// Map an explicit `CODEWHALE_ASR_MODEL` value to (kind, model). Local
|
||||
/// spellings are tested first: `local-whisper` and `whisper.cpp` both contain
|
||||
/// "whisper", which on its own names Groq's hosted Whisper (C01-10).
|
||||
fn explicit_asr_choice(value: &str) -> Option<(&'static str, &'static str)> {
|
||||
let m = value.trim().to_ascii_lowercase();
|
||||
if m.contains("local") || m.contains("whisper.cpp") || m.contains("whisper-cpp") {
|
||||
return Some(("local-whisper", "tiny"));
|
||||
}
|
||||
// Auto-detect best free: local whisper (offline, no key) > Groq free tier > Xiaomi ASR (needs key)
|
||||
let free = detect_free_asr();
|
||||
match free {
|
||||
"local-whisper" => ("local-whisper".into(), "tiny".into()),
|
||||
"groq" => ("groq".into(), GROQ_ASR_MODEL.into()),
|
||||
_ => ("provider".into(), ASR_MODEL.into()),
|
||||
if m.contains("groq") || m.contains("whisper") {
|
||||
return Some(("groq", GROQ_ASR_MODEL));
|
||||
}
|
||||
if m.contains("mimo") || m.contains("xiaomi") {
|
||||
return Some(("provider", ASR_MODEL));
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
// --- Headless capture (HTTP/native-client path) ----------------------------
|
||||
@@ -717,6 +734,28 @@ mod tests {
|
||||
assert!(independent.get("provider").is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn explicit_local_whisper_is_not_routed_to_groq() {
|
||||
// C01-10: every local spelling contains "whisper"; none may reach the
|
||||
// hosted Groq backend (which needs a key and sends audio off-host).
|
||||
for local in ["local-whisper", "whisper.cpp", "whisper-cpp", " LOCAL "] {
|
||||
assert_eq!(
|
||||
explicit_asr_choice(local),
|
||||
Some(("local-whisper", "tiny")),
|
||||
"{local}"
|
||||
);
|
||||
}
|
||||
for hosted in ["groq", "whisper", "whisper-large-v3-turbo"] {
|
||||
assert_eq!(
|
||||
explicit_asr_choice(hosted),
|
||||
Some(("groq", GROQ_ASR_MODEL)),
|
||||
"{hosted}"
|
||||
);
|
||||
}
|
||||
assert_eq!(explicit_asr_choice("mimo"), Some(("provider", ASR_MODEL)));
|
||||
assert_eq!(explicit_asr_choice("something-else"), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn wav_encoding_produces_valid_header() {
|
||||
let samples = vec![0i16; 16000]; // 1 second of silence
|
||||
|
||||
@@ -41,6 +41,7 @@ notes, and relevant issue/PR comments.
|
||||
|
||||
**Merged or adapted contributions**
|
||||
|
||||
- **[zhuowp](https://github.com/zhuowp)** — supplied the process-scoped PowerShell execution-policy repair adapted for Codewhale; machine and user Group Policy still take precedence ([#6745](https://github.com/Hmbown/Codewhale/issues/6745)).
|
||||
- **[gaord](https://github.com/gaord)** — let a client fork a thread at a named turn ([#6580](https://github.com/Hmbown/Codewhale/pull/6580)), let undo roll back files for the turn it is undoing ([#6483](https://github.com/Hmbown/Codewhale/pull/6483)), stopped resume and fork from duplicating threads and sessions ([#6406](https://github.com/Hmbown/Codewhale/pull/6406)), exposed user-defined provider routes to native clients ([#6404](https://github.com/Hmbown/Codewhale/pull/6404)), and kept a fork going when a turn lost its tool call ([#6664](https://github.com/Hmbown/Codewhale/pull/6664)).
|
||||
- **[Lstarsky0](https://github.com/Lstarsky0)** — moved the docs/work, legal, digest and FAQ pages onto the dictionary spine ([#6405](https://github.com/Hmbown/Codewhale/pull/6405), [#6417](https://github.com/Hmbown/Codewhale/pull/6417), [#6499](https://github.com/Hmbown/Codewhale/pull/6499), [#6574](https://github.com/Hmbown/Codewhale/pull/6574)), tightened the Chinese-branching ceiling to 18 ([#6403](https://github.com/Hmbown/Codewhale/pull/6403)), and made Fleet publish without a two-link window ([#6431](https://github.com/Hmbown/Codewhale/pull/6431)). Also moved the constitution page onto the dictionary spine and kept its install link in the selected locale ([#6733](https://github.com/Hmbown/Codewhale/pull/6733)).
|
||||
- **[aboimpinto](https://github.com/aboimpinto)** — restored a green Linux full-workspace test gate without loosening any test, twice ([#6581](https://github.com/Hmbown/Codewhale/pull/6581), [#6666](https://github.com/Hmbown/Codewhale/pull/6666)).
|
||||
|
||||
+1
-1
@@ -54,7 +54,7 @@ For the read-only record of what a thread or turn did, see
|
||||
| `codewhale app-server --mobile` | HTTP/SSE on loopback + `/mobile` | Runtime API + local mobile control page |
|
||||
| `codewhale app-server --stdio` | JSON-RPC 2.0 over stdio | Local SDK / control probe (no listener) |
|
||||
| `codewhale app-server --socket [--socket-path P]` | JSON-RPC 2.0 over a `0600` unix domain socket | Desktop daemon: multi-client, peer-uid checked, `daemon/attach` claim handshake (macOS/Linux; Windows named pipe reserved, not implemented) |
|
||||
| `codewhale app-server` | HTTP on `127.0.0.1:8787` | Legacy in-process app-server (`/healthz`, `/thread`, `/app`, `/prompt`, `/jobs`, `/mcp/startup`); `/prompt` and `/thread` messages execute real turns via the runtime bridge. There is no direct `/tool` route: tools run only inside Engine turns, under the Engine's tool catalog and approval posture. This legacy server does not surface approvals: its bridge forwards only text deltas and the turn's completion, and it has no decision route, so an approval-gated call waits unanswered. Drive approval-gated work through the Runtime API (`/v1/threads/*` events and `POST /v1/approvals/{approval_id}`) |
|
||||
| `codewhale app-server` | HTTP on `127.0.0.1:8787` | Legacy in-process app-server (`/healthz`, `/thread`, `/app`, `/prompt`, `/jobs`); `/prompt` and `/thread` messages execute real turns via the runtime bridge. There is no direct `/tool` route: tools run only inside Engine turns, under the Engine's tool catalog and approval posture. This legacy server does not surface approvals: its bridge forwards only text deltas and the turn's completion, and it has no decision route, so an approval-gated call waits unanswered. Drive approval-gated work through the Runtime API (`/v1/threads/*` events and `POST /v1/approvals/{approval_id}`) |
|
||||
| `codewhale serve --http` / `--mobile` | same server as `app-server --http`/`--mobile` | Compatibility aliases |
|
||||
|
||||
`app-server --http` and `--mobile` launch the same mature runtime API server
|
||||
|
||||
@@ -51,7 +51,7 @@ local supervisor / SDK / automation harness
|
||||
| `codewhale app-server --mobile` | 回环上的 HTTP/SSE + `/mobile` | 运行时 API + 本地移动端控制页 |
|
||||
| `codewhale app-server --stdio` | 基于 stdio 的 JSON-RPC 2.0 | 本地 SDK / 控制探针(不监听端口) |
|
||||
| `codewhale app-server --socket [--socket-path P]` | 基于权限为 `0600` 的 unix 域套接字的 JSON-RPC 2.0 | 桌面守护进程:多客户端、对端 uid 校验、`daemon/attach` 认领握手(macOS/Linux;Windows 命名管道已预留但未实现) |
|
||||
| `codewhale app-server` | 在 `127.0.0.1:8787` 上的 HTTP | 旧式进程内 app-server(`/healthz`、`/thread`、`/app`、`/prompt`、`/jobs`、`/mcp/startup`);`/prompt` 与 `/thread` 消息会通过运行时桥执行真实回合。它没有直接的 `/tool` 路由:工具只在 Engine 回合内部、在 Engine 的工具目录与审批姿态下运行。这个旧式服务器不呈现审批:它的桥只转发文本增量和回合的完成,并且没有决策路由,所以一个受审批门控的调用会一直等不到答复。受审批门控的工作请通过运行时 API 驱动(`/v1/threads/*` 事件与 `POST /v1/approvals/{approval_id}`) |
|
||||
| `codewhale app-server` | 在 `127.0.0.1:8787` 上的 HTTP | 旧式进程内 app-server(`/healthz`、`/thread`、`/app`、`/prompt`、`/jobs`);`/prompt` 与 `/thread` 消息会通过运行时桥执行真实回合。它没有直接的 `/tool` 路由:工具只在 Engine 回合内部、在 Engine 的工具目录与审批姿态下运行。这个旧式服务器不呈现审批:它的桥只转发文本增量和回合的完成,并且没有决策路由,所以一个受审批门控的调用会一直等不到答复。受审批门控的工作请通过运行时 API 驱动(`/v1/threads/*` 事件与 `POST /v1/approvals/{approval_id}`) |
|
||||
| `codewhale serve --http` / `--mobile` | 与 `app-server --http`/`--mobile` 相同的服务器 | 兼容别名 |
|
||||
|
||||
`app-server --http` 与 `--mobile` 启动的是历史上经由 `serve --http` 访问的同一个
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { chmod, mkdir, readFile, rename, writeFile } from "node:fs/promises";
|
||||
import { chmod, mkdir, open, readFile, rename, rm } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { randomBytes } from "node:crypto";
|
||||
|
||||
@@ -20,6 +20,39 @@ async function chmodBestEffort(filePath, mode) {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Replace `filePath` so a crash leaves either the old bytes or the new ones:
|
||||
* a unique temporary name (two processes sharing a state dir never write the
|
||||
* same temp file), fsync before the rename, and a best-effort directory fsync
|
||||
* after it so the rename itself survives power loss where the OS allows.
|
||||
*/
|
||||
export async function writeFileDurable(filePath, contents, { mode = 0o600 } = {}) {
|
||||
const dir = path.dirname(filePath);
|
||||
const tmp = path.join(dir, `.${path.basename(filePath)}.${process.pid}.${randomBytes(6).toString("hex")}.tmp`);
|
||||
let handle = null;
|
||||
try {
|
||||
handle = await open(tmp, "wx", mode);
|
||||
await handle.writeFile(contents);
|
||||
await handle.sync();
|
||||
await handle.close();
|
||||
handle = null;
|
||||
await rename(tmp, filePath);
|
||||
} catch (error) {
|
||||
await handle?.close().catch(() => {});
|
||||
await rm(tmp, { force: true }).catch(() => {});
|
||||
throw error;
|
||||
}
|
||||
let dirHandle = null;
|
||||
try {
|
||||
dirHandle = await open(dir, "r");
|
||||
await dirHandle.sync();
|
||||
} catch {
|
||||
// Windows and some filesystems cannot fsync a directory.
|
||||
} finally {
|
||||
await dirHandle?.close().catch(() => {});
|
||||
}
|
||||
}
|
||||
|
||||
export class ThreadStore {
|
||||
static async open(filePath, options = {}) {
|
||||
const store = new ThreadStore(filePath, options);
|
||||
@@ -48,6 +81,9 @@ export class ThreadStore {
|
||||
if (this.options.messageLimit > 0 && !Array.isArray(this.data.messages)) {
|
||||
this.data.messages = [];
|
||||
}
|
||||
if (this.options.messageLimit > 0 && (!this.data.inflight || typeof this.data.inflight !== "object" || Array.isArray(this.data.inflight))) {
|
||||
this.data.inflight = {};
|
||||
}
|
||||
if (this.options.actions && (!this.data.actions || typeof this.data.actions !== "object")) {
|
||||
this.data.actions = {};
|
||||
}
|
||||
@@ -66,6 +102,39 @@ export class ThreadStore {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Claim one inbound message before acting on it. "new": handle it now (it
|
||||
* is recorded as in flight first, durably). "done": a finished duplicate.
|
||||
* "interrupted": an earlier process claimed it and stopped before
|
||||
* completeMessage, so its effect is unknown; the caller reports that
|
||||
* instead of running it again. Callers handle messages sequentially — a
|
||||
* key still in flight in this process is not a restart.
|
||||
*/
|
||||
async claimMessage(messageKey) {
|
||||
if (!messageKey || this.options.messageLimit <= 0) return "new";
|
||||
this.ensureShape();
|
||||
const inflight = this.data.inflight;
|
||||
if (this.data.messages.includes(messageKey)) {
|
||||
if (!Object.hasOwn(inflight, messageKey)) return "done";
|
||||
delete inflight[messageKey];
|
||||
await this.save();
|
||||
return "interrupted";
|
||||
}
|
||||
this.data.messages.push(messageKey);
|
||||
this.data.messages = this.data.messages.slice(-this.options.messageLimit);
|
||||
for (const key of Object.keys(inflight)) if (!this.data.messages.includes(key)) delete inflight[key];
|
||||
inflight[messageKey] = new Date().toISOString();
|
||||
await this.save();
|
||||
return "new";
|
||||
}
|
||||
|
||||
/** Mark a claimed message handled (whether it succeeded or failed in-process). */
|
||||
async completeMessage(messageKey) {
|
||||
if (!messageKey || !this.data.inflight || !Object.hasOwn(this.data.inflight, messageKey)) return;
|
||||
delete this.data.inflight[messageKey];
|
||||
await this.save();
|
||||
}
|
||||
|
||||
async recordMessage(messageKey) {
|
||||
if (!messageKey || this.options.messageLimit <= 0) return false;
|
||||
this.ensureShape();
|
||||
@@ -180,10 +249,7 @@ export class ThreadStore {
|
||||
const dir = path.dirname(this.filePath);
|
||||
await mkdir(dir, { recursive: true, mode: 0o700 });
|
||||
if (this.options.privateMode) await chmodBestEffort(dir, 0o700);
|
||||
const tmp = `${this.filePath}.tmp`;
|
||||
await writeFile(tmp, `${JSON.stringify(this.data, null, 2)}\n`, { mode: 0o600 });
|
||||
if (this.options.privateMode) await chmodBestEffort(tmp, 0o600);
|
||||
await rename(tmp, this.filePath);
|
||||
await writeFileDurable(this.filePath, `${JSON.stringify(this.data, null, 2)}\n`, { mode: 0o600 });
|
||||
if (this.options.privateMode) await chmodBestEffort(this.filePath, 0o600);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdtemp, rm } from "node:fs/promises";
|
||||
import { mkdtemp, readdir, readFile, rm } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import path from "node:path";
|
||||
|
||||
@@ -19,7 +19,8 @@ import {
|
||||
readSse,
|
||||
splitMessage,
|
||||
stripGroupPrefix,
|
||||
ThreadStore
|
||||
ThreadStore,
|
||||
writeFileDurable
|
||||
} from "../src/lib.mjs";
|
||||
|
||||
test("env and primitive parsers handle bridge env conventions", () => {
|
||||
@@ -255,3 +256,35 @@ test("ThreadStore persists numeric cursors", async () => {
|
||||
await rm(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("writeFileDurable replaces the file through unique temp names and leaves none behind", async () => {
|
||||
const dir = await mkdtemp(path.join(tmpdir(), "codewhale-bridge-core-"));
|
||||
try {
|
||||
const target = path.join(dir, "sync-buf.txt");
|
||||
// Concurrent writers used to share one fixed `.tmp` path and race on it.
|
||||
await Promise.all(Array.from({ length: 8 }, (_, index) => writeFileDurable(target, `cursor-${index}`)));
|
||||
assert.match(await readFile(target, "utf8"), /^cursor-[0-7]$/);
|
||||
assert.deepEqual(await readdir(dir), ["sync-buf.txt"]);
|
||||
} finally {
|
||||
await rm(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("ThreadStore claims survive a restart: finished messages skip, an in-flight one is reported", async () => {
|
||||
const dir = await mkdtemp(path.join(tmpdir(), "codewhale-bridge-core-"));
|
||||
try {
|
||||
const statePath = path.join(dir, "thread-map.json");
|
||||
const store = await ThreadStore.open(statePath, { messageLimit: 10 });
|
||||
assert.equal(await store.claimMessage("u:1"), "new");
|
||||
await store.completeMessage("u:1");
|
||||
assert.equal(await store.claimMessage("u:2"), "new");
|
||||
// The process dies here, before completeMessage("u:2").
|
||||
const restarted = await ThreadStore.open(statePath, { messageLimit: 10 });
|
||||
assert.equal(await restarted.claimMessage("u:1"), "done");
|
||||
assert.equal(await restarted.claimMessage("u:2"), "interrupted");
|
||||
assert.equal(await restarted.claimMessage("u:2"), "done", "reported once, then an ordinary duplicate");
|
||||
assert.equal(await restarted.claimMessage("u:3"), "new");
|
||||
} finally {
|
||||
await rm(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
@@ -23,9 +23,10 @@ import {
|
||||
latestRunningTurn,
|
||||
activeTurnBlock,
|
||||
helpText,
|
||||
processUpdateBatch,
|
||||
} from "./lib.mjs";
|
||||
import { renderQrToText } from "./qr.mjs";
|
||||
import { ThreadStore as CoreThreadStore } from "../../bridge-core/src/lib.mjs";
|
||||
import { ThreadStore as CoreThreadStore, writeFileDurable } from "../../bridge-core/src/lib.mjs";
|
||||
|
||||
// ============================================================================
|
||||
// ThreadStore — JSON 文件持久化(与 feishu/telegram/wechat bridge 一致)
|
||||
@@ -59,11 +60,7 @@ async function loadAccount(stateDir) {
|
||||
async function saveAccount(stateDir, account) {
|
||||
const p = resolveAccountPath(stateDir);
|
||||
await fs.mkdir(path.dirname(p), { recursive: true, mode: 0o700 });
|
||||
const tmp = `${p}.tmp`;
|
||||
await fs.writeFile(tmp, `${JSON.stringify(account, null, 2)}\n`, {
|
||||
mode: 0o600,
|
||||
});
|
||||
await fs.rename(tmp, p);
|
||||
await writeFileDurable(p, `${JSON.stringify(account, null, 2)}\n`, { mode: 0o600 });
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
@@ -718,9 +715,57 @@ async function saveSyncBuf(stateDir, buf) {
|
||||
// The state dir may not exist yet on a first run whose first persisted write
|
||||
// is the poll cursor rather than account.json.
|
||||
await fs.mkdir(path.dirname(p), { recursive: true, mode: 0o700 });
|
||||
const tmp = `${p}.tmp`;
|
||||
await fs.writeFile(tmp, buf, { mode: 0o600 });
|
||||
await fs.rename(tmp, p);
|
||||
await writeFileDurable(p, buf, { mode: 0o600 });
|
||||
}
|
||||
|
||||
/** One claimed inbound message: context token, text, allowlist, command. */
|
||||
async function handleInbound(msg) {
|
||||
const fromUser = msg.from_user_id || "";
|
||||
// 保存 context_token
|
||||
if (msg.context_token) {
|
||||
await threadStore.patchChat(fromUser, {
|
||||
contextToken: msg.context_token,
|
||||
updatedAt: new Date().toISOString(),
|
||||
});
|
||||
}
|
||||
|
||||
// 提取文本
|
||||
const text = extractText(msg.item_list);
|
||||
|
||||
if (!text) {
|
||||
await sendText(
|
||||
fromUser,
|
||||
"仅支持文本消息。图片/语音/视频/文件暂不支持。"
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
console.log(
|
||||
`[inbound] from=${fromUser} text=${text.slice(0, 100)}`
|
||||
);
|
||||
|
||||
// 白名单检查
|
||||
if (!isAllowed(fromUser)) {
|
||||
await sendText(
|
||||
fromUser,
|
||||
[
|
||||
"This WeChat user is not in WEIXIN_CHAT_ALLOWLIST.",
|
||||
`user_id=${fromUser}`,
|
||||
"",
|
||||
"For first pairing, add this user_id to WEIXIN_CHAT_ALLOWLIST, or temporarily set WEIXIN_ALLOW_UNLISTED=true.",
|
||||
].join("\n")
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
// 命令路由
|
||||
const command = parseCommand(text);
|
||||
await handleCommand(fromUser, command).catch((error) => {
|
||||
console.error(
|
||||
`failed to handle command from=${fromUser} text=${text.slice(0, 100)}`,
|
||||
error
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
async function monitorLoop() {
|
||||
@@ -775,69 +820,26 @@ async function monitorLoop() {
|
||||
|
||||
consecutiveFailures = 0;
|
||||
|
||||
// 保存游标
|
||||
if (resp.get_updates_buf) {
|
||||
getUpdatesBuf = resp.get_updates_buf;
|
||||
await saveSyncBuf(config.stateDir, getUpdatesBuf);
|
||||
}
|
||||
|
||||
// 处理消息
|
||||
const msgs = resp.msgs || [];
|
||||
for (const msg of msgs) {
|
||||
const fromUser = msg.from_user_id || "";
|
||||
const messageId = String(msg.message_id || "");
|
||||
|
||||
if (!fromUser) continue;
|
||||
|
||||
const msgKey = `${fromUser}:${messageId}`;
|
||||
if (await threadStore.recordMessage(msgKey)) continue;
|
||||
|
||||
// 保存 context_token
|
||||
if (msg.context_token) {
|
||||
await threadStore.patchChat(fromUser, {
|
||||
contextToken: msg.context_token,
|
||||
updatedAt: new Date().toISOString(),
|
||||
});
|
||||
}
|
||||
|
||||
// 提取文本
|
||||
const text = extractText(msg.item_list);
|
||||
|
||||
if (!text) {
|
||||
// 先处理消息,再保存游标:进程崩溃时整批从 iLink 重放,
|
||||
// 已完成的消息按 key 跳过,崩溃时正在处理的消息只提示、不重跑。
|
||||
await processUpdateBatch({
|
||||
messages: resp.msgs || [],
|
||||
nextCursor: resp.get_updates_buf,
|
||||
store: threadStore,
|
||||
keyOf: (msg) => (msg.from_user_id ? `${msg.from_user_id}:${String(msg.message_id || "")}` : null),
|
||||
interrupted: async (msg) => {
|
||||
if (!isAllowed(msg.from_user_id)) return;
|
||||
await sendText(
|
||||
fromUser,
|
||||
"仅支持文本消息。图片/语音/视频/文件暂不支持。"
|
||||
msg.from_user_id,
|
||||
"桥接进程在处理这条消息时重启,未重复执行;如仍需要请重新发送。\nThe bridge restarted while handling this message and did not run it again; send it again if you still need it."
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
console.log(
|
||||
`[inbound] from=${fromUser} text=${text.slice(0, 100)}`
|
||||
);
|
||||
|
||||
// 白名单检查
|
||||
if (!isAllowed(fromUser)) {
|
||||
await sendText(
|
||||
fromUser,
|
||||
[
|
||||
"This WeChat user is not in WEIXIN_CHAT_ALLOWLIST.",
|
||||
`user_id=${fromUser}`,
|
||||
"",
|
||||
"For first pairing, add this user_id to WEIXIN_CHAT_ALLOWLIST, or temporarily set WEIXIN_ALLOW_UNLISTED=true.",
|
||||
].join("\n")
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
// 命令路由
|
||||
const command = parseCommand(text);
|
||||
await handleCommand(fromUser, command).catch((error) => {
|
||||
console.error(
|
||||
`failed to handle command from=${fromUser} text=${text.slice(0, 100)}`,
|
||||
error
|
||||
);
|
||||
});
|
||||
}
|
||||
},
|
||||
handle: handleInbound,
|
||||
commitCursor: async (buf) => {
|
||||
getUpdatesBuf = buf;
|
||||
await saveSyncBuf(config.stateDir, buf);
|
||||
},
|
||||
});
|
||||
} catch (error) {
|
||||
if (error.name === "AbortError" || error.message?.includes("abort")) {
|
||||
// 长轮询超时是正常的,立即重试
|
||||
|
||||
@@ -35,6 +35,33 @@ export {
|
||||
splitMessage,
|
||||
};
|
||||
|
||||
/**
|
||||
* Handle one getUpdates batch, then advance the poll cursor. The cursor moves
|
||||
* only after every message in the batch was handled, so a crash replays the
|
||||
* batch from iLink instead of dropping a prompt. Each message is claimed
|
||||
* durably first: on replay, finished messages are skipped, and one a crashed
|
||||
* process was still handling is reported through `interrupted` rather than
|
||||
* run twice (its effect is unknown).
|
||||
*/
|
||||
export async function processUpdateBatch({ messages, nextCursor, store, keyOf, handle, interrupted, commitCursor }) {
|
||||
for (const msg of messages || []) {
|
||||
const key = keyOf(msg);
|
||||
if (!key) continue;
|
||||
const claim = await store.claimMessage(key);
|
||||
if (claim === "done") continue;
|
||||
if (claim === "interrupted") {
|
||||
await interrupted(msg);
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
await handle(msg);
|
||||
} finally {
|
||||
await store.completeMessage(key);
|
||||
}
|
||||
}
|
||||
if (nextCursor) await commitCursor(nextCursor);
|
||||
}
|
||||
|
||||
export function randomUin() {
|
||||
const uint32 = crypto.randomBytes(4).readUInt32BE(0);
|
||||
return Buffer.from(String(uint32), "utf-8").toString("base64");
|
||||
|
||||
@@ -1,5 +1,10 @@
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdtemp, rm } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import path from "node:path";
|
||||
|
||||
import { ThreadStore } from "../../bridge-core/src/lib.mjs";
|
||||
|
||||
import {
|
||||
activeTurnBlock,
|
||||
@@ -10,6 +15,7 @@ import {
|
||||
parseCommand,
|
||||
parseList,
|
||||
preservedChatStateFields,
|
||||
processUpdateBatch,
|
||||
splitMessage
|
||||
} from "../src/lib.mjs";
|
||||
|
||||
@@ -51,3 +57,44 @@ test("shared state and runtime helpers preserve Weixin bridge behavior", () => {
|
||||
message: "Thread already has active turn turn-1. Wait for it to finish or send /interrupt."
|
||||
});
|
||||
});
|
||||
|
||||
test("a crash mid-batch replays the batch without losing or re-running prompts", async () => {
|
||||
const dir = await mkdtemp(path.join(tmpdir(), "codewhale-weixin-bridge-"));
|
||||
try {
|
||||
const statePath = path.join(dir, "thread-map.json");
|
||||
const batch = [1, 2, 3].map((id) => ({ from_user_id: "u", message_id: id }));
|
||||
const keyOf = (msg) => `${msg.from_user_id}:${msg.message_id}`;
|
||||
const handled = [];
|
||||
const cursors = [];
|
||||
const crash = new Error("process killed");
|
||||
const first = await ThreadStore.open(statePath, { messageLimit: 50 });
|
||||
let afterCrash = null;
|
||||
await assert.rejects(processUpdateBatch({
|
||||
messages: batch, nextCursor: "buf-2", store: first, keyOf,
|
||||
handle: async (msg) => {
|
||||
if (msg.message_id === 2) {
|
||||
// What a restarted process finds on disk while message 2 runs.
|
||||
afterCrash = await ThreadStore.open(statePath, { messageLimit: 50 });
|
||||
throw crash;
|
||||
}
|
||||
handled.push(msg.message_id);
|
||||
},
|
||||
interrupted: async () => assert.fail("nothing is interrupted on the first pass"),
|
||||
commitCursor: async (buf) => cursors.push(buf),
|
||||
}), crash);
|
||||
assert.deepEqual(cursors, [], "the cursor does not move past unhandled messages");
|
||||
|
||||
const reported = [];
|
||||
await processUpdateBatch({
|
||||
messages: batch, nextCursor: "buf-2", store: afterCrash, keyOf,
|
||||
handle: async (msg) => handled.push(msg.message_id),
|
||||
interrupted: async (msg) => reported.push(msg.message_id),
|
||||
commitCursor: async (buf) => cursors.push(buf),
|
||||
});
|
||||
assert.deepEqual(handled, [1, 3], "message 1 is not re-run; message 3 is not lost");
|
||||
assert.deepEqual(reported, [2], "the in-flight message is reported, not re-run");
|
||||
assert.deepEqual(cursors, ["buf-2"]);
|
||||
} finally {
|
||||
await rm(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
+5
-3
@@ -88,8 +88,8 @@ The Rust particle implementation lives in the product's
|
||||
`crates/tui/src/tui/ambient_life/pet_sim.rs`; this package's runner imports it.
|
||||
Swift and Kotlin particle ports must match its conformance digests. Generated
|
||||
native bundles are committed so the product Rust build needs no Node compiler.
|
||||
Run `npm run sync` after changing core source; the generated-byte check in CI
|
||||
rejects a stale bundle. Local Whalesong consumers use aliases to these same
|
||||
Run `npm run sync` after changing core source; `npm run check` (run by CI)
|
||||
rejects a stale bundle or `shared.html` copy. Local Whalesong consumers use aliases to these same
|
||||
canonical files, not separately maintained source copies.
|
||||
|
||||
The original Whalesong importer, signal model, schema and browser storage code
|
||||
@@ -129,7 +129,9 @@ Live recording continues in segments. At 216,000 buckets (24 hours) or 64 MiB,
|
||||
the recorder syncs the completed file, preserves it as
|
||||
`OUTPUT.segment-000001.jsonl` (then `000002`, etc.), and atomically replaces the
|
||||
same live pathname. Each segment starts at sequence zero and replays independently.
|
||||
Use `--segment-buckets=N` to rotate sooner. Followers establish a new baseline
|
||||
Use `--segment-buckets=N` to rotate sooner. A stall or suspension shorter than
|
||||
one segment is recorded as unknown buckets; a longer one starts a new segment
|
||||
whose first bucket is unknown, as `--resume` does, and only reports its length. Followers establish a new baseline
|
||||
after replacement, then accept subsequent appends as current observations.
|
||||
|
||||
The live importer retains unfinished lifetimes and 16 seconds of completed
|
||||
|
||||
Loaded 100 of 158 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user