diff --git a/CHANGELOG.md b/CHANGELOG.md index 063a49721..f53cde614 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -160,6 +160,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- Worker coordination serializes durable admissions across processes, preserves foreign workers and winning completion receipts, and fences stale checkpoints. Fleet restart reservations retain uncertain prior processes until exact teardown is confirmed; final output, usage and the receipt publish together without duplicate accounting on retry. Retained transcripts follow committed ownership. + - JSONPath projections bound path length, traversal steps and intermediate matches before cloning results. Direct fetches share the same parser and enforce one aggregate serialized-field budget; oversized selections refuse with narrower-selection guidance. - Long tool output and diffs use bounded display windows and a byte-limited row cache. Retained details load verified byte ranges and label what search and copy include; explicit source copy keeps stored text with control-sequence cleanup. Oversized Unicode is shown safely without changing question answers; long answers retain their exact bytes in the existing owned output artifact before bounded model-context projection. New pager notices are translated across all 15 complete language packs. diff --git a/crates/engine/src/core/engine.rs b/crates/engine/src/core/engine.rs index 7b382e228..8b5cca925 100644 --- a/crates/engine/src/core/engine.rs +++ b/crates/engine/src/core/engine.rs @@ -198,8 +198,7 @@ impl StructuredState { working_set.summary_block(&workspace, &crate::tools::file::working_set_read_allowed); let subagent_snapshots = if let Some(handle) = subagents { - let mut guard = handle.write().await; - guard.cleanup_for_session(active_session_id, Duration::from_secs(60 * 60)); + let guard = handle.read().await; guard .list_for_session(active_session_id) .into_iter() @@ -4051,12 +4050,28 @@ impl Engine { ) }; let event = if due { - let mut manager = self.subagent_manager.write().await; - manager.cleanup_for_session( - &active_session_id, - Duration::from_secs(60 * 60), - ); - agent_list_event(&manager, &active_session_id) + let manager = self.subagent_manager.clone(); + let session_id = active_session_id.clone(); + match codewhale_runtime::runtime_control::owner_work(move || { + let mut manager = manager.blocking_write(); + manager.cleanup_for_session( + &session_id, + Duration::from_secs(60 * 60), + )?; + Ok(agent_list_event(&manager, &session_id)) + }) + .await + { + Ok(event) => event, + Err(error) => { + let _ = self + .send_event(Event::error(ErrorEnvelope::transient( + format!("Failed to clean up sub-agents: {error}"), + ))) + .await; + continue; + } + } } else { let manager = self.subagent_manager.read().await; agent_list_event(&manager, &active_session_id) @@ -4082,11 +4097,16 @@ impl Engine { } Op::CancelSubAgent { agent_id } => { let active_session_id = self.session.id.clone(); - let cancelled = self - .subagent_manager - .write() - .await - .cancel_agent_for_session(&active_session_id, &agent_id); + let manager = self.subagent_manager.clone(); + let session_id = active_session_id.clone(); + let target = agent_id.clone(); + let cancelled = + codewhale_runtime::runtime_control::owner_work(move || { + manager + .blocking_write() + .cancel_agent_for_session(&session_id, &target) + }) + .await; let result = match cancelled { Ok(snapshot) => { let snapshot = crate::tools::subagent::settle_requested_child( @@ -4159,18 +4179,27 @@ impl Engine { let active_session_id = self.session.id.clone(); let runtime = self.off_turn_subagent_runtime(); let manager_handle = Arc::clone(&self.subagent_manager); - let (outcome, refresh) = { - let mut manager = self.subagent_manager.write().await; - let outcome = manager - .continue_child_from_user_for_session( - &active_session_id, - manager_handle, - runtime, - &agent_id, - &text, - ) - .map_err(|err| err.to_string()); - (outcome, agent_list_event(&manager, &active_session_id)) + let manager = Arc::clone(&manager_handle); + let session = active_session_id.clone(); + let target = agent_id.clone(); + let result = + codewhale_runtime::runtime_control::owner_work(move || { + let mut manager = manager.blocking_write(); + let outcome = manager + .continue_child_from_user_for_session( + &session, + manager_handle, + runtime, + &target, + &text, + ) + .map_err(|err| err.to_string()); + Ok((outcome, agent_list_event(&manager, &session))) + }) + .await; + let (outcome, refresh) = match result { + Ok((outcome, refresh)) => (outcome, Some(refresh)), + Err(error) => (Err(error.to_string()), None), }; let _ = self .send_event(Event::SubAgentFollowUp { @@ -4179,7 +4208,9 @@ impl Engine { outcome, }) .await; - if let Err(_e) = self.tx_event.try_send(refresh) { + if let Some(refresh) = refresh + && self.tx_event.try_send(refresh).is_err() + { tracing::debug!( "Event channel full; dropping FollowUpSubAgent refresh" ); @@ -4322,12 +4353,6 @@ impl Engine { workspace, mode, } => { - // A fresh conversation arrives with its own - // sources: re-arm the guard and rebuild the notice - // from the assembly that installs its header. - self.instruction_guard_key = None; - self.pending_instruction_notices.clear(); - self.session.instruction_assembly = None; let plugin_workspace_changed = self.plugin_registry.workspace() != workspace.as_path(); let previous_session_id = self.session.id.clone(); @@ -4338,7 +4363,8 @@ impl Engine { } else { previous_session_id.clone() }; - let closed_session_id = self.install_synced_session_id(next_session_id); + let closed_session_id = (next_session_id != previous_session_id) + .then_some(previous_session_id); // SyncSession installs a conversation's identity; an id // change IS a conversation boundary in this runtime — // callers must pass their own conversation id for @@ -4349,11 +4375,28 @@ impl Engine { // the new conversation (#5372). Same-session reloads // keep their id and are deliberately left untouched. if let Some(closed_session_id) = closed_session_id { - let finalized = self - .subagent_manager - .write() - .await - .finalize_session_close_for_session(&closed_session_id); + let manager = self.subagent_manager.clone(); + let session_id = closed_session_id.clone(); + let finalized = + codewhale_runtime::runtime_control::owner_work(move || { + manager + .blocking_write() + .finalize_session_close_for_session(&session_id) + }) + .await; + let finalized = match finalized { + Ok(finalized) => finalized, + Err(error) => { + let _ = self + .send_event(Event::error(ErrorEnvelope::transient( + format!( + "Failed to close sub-agent session {closed_session_id}; session was not switched: {error}" + ), + ))) + .await; + continue; + } + }; crate::tools::subagent::settle_requested_children_for_session( &self.subagent_manager, &closed_session_id, @@ -4368,6 +4411,13 @@ impl Engine { ); } } + // A fresh conversation arrives with its own + // sources: re-arm the guard and rebuild the notice + // from the assembly that installs its header. + self.instruction_guard_key = None; + self.pending_instruction_notices.clear(); + self.session.instruction_assembly = None; + let _ = self.install_synced_session_id(next_session_id); self.restore_session_history( messages, system_prompt, @@ -4705,19 +4755,41 @@ impl Engine { // #freeze: flush any sub-agent checkpoint that the hot-path debounce // coalesced away, so a graceful shutdown keeps the latest progress. - { - let mut manager = self.subagent_manager.write().await; - let children = manager.list_for_session(&self.session.id); + let manager = self.subagent_manager.clone(); + let session_id = self.session.id.clone(); + let owner_agent_id = self + .child_host + .as_ref() + .map(|owner| owner.authority.owner_agent_id.clone()); + let shutdown = codewhale_runtime::runtime_control::owner_work(move || { + let mut manager = manager.blocking_write(); + let children = manager.list_for_session(&session_id); + let mut failure = None; for child in children { - let owned = self.child_host.as_ref().is_none_or(|owner| { - child.parent_run_id.as_deref() - == Some(owner.authority.owner_agent_id.as_str()) - }); - if owned && child.status == SubAgentStatus::Running { - let _ = manager.cancel_agent_for_session(&self.session.id, &child.agent_id); + let owned = owner_agent_id + .as_ref() + .is_none_or(|owner| child.parent_run_id.as_deref() == Some(owner.as_str())); + if owned + && child.status == SubAgentStatus::Running + && let Err(error) = + manager.cancel_agent_for_session(&session_id, &child.agent_id) + { + failure.get_or_insert(error); } } - manager.flush_pending_persist(); + if let Err(error) = manager.flush_pending_persist() { + failure.get_or_insert(error); + } + failure.map_or(Ok(()), Err) + }) + .await; + if let Err(error) = shutdown { + let _ = self + .send_event(Event::error(ErrorEnvelope::transient(format!( + "Sub-agent shutdown is uncertain: {error}" + )))) + .await; + child_result = Some(Err(error)); } crate::tools::subagent::settle_requested_children_for_session( &self.subagent_manager, @@ -4727,7 +4799,19 @@ impl Engine { .map(|child| child.authority.owner_agent_id.as_str()), ) .await; - self.subagent_manager.write().await.flush_pending_persist(); + let manager = self.subagent_manager.clone(); + if let Err(error) = codewhale_runtime::runtime_control::owner_work(move || { + manager.blocking_write().flush_pending_persist() + }) + .await + { + let _ = self + .send_event(Event::error(ErrorEnvelope::transient(format!( + "Sub-agent shutdown checkpoint is uncertain: {error}" + )))) + .await; + child_result = Some(Err(error)); + } // #420: graceful MCP shutdown — send SIGTERM and give stdio servers // a brief window to exit before drop fires SIGKILL via kill_on_drop. @@ -10393,27 +10477,32 @@ fn apply_patch_permission_paths(input: &Value) -> Vec { } /// Spawn the engine in a background task -pub fn spawn_engine( +pub async fn spawn_engine( config: EngineConfig, api_config: &Config, mcp_handoff: Option, ) -> EngineHandle { - let (engine, handle) = Engine::new(config, api_config); - engine - .mcp_pool_slot - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner) - .handoff = mcp_handoff; + let api_config = api_config.clone(); + let (engine, handle) = codewhale_runtime::runtime_control::owner_work(move || { + let (engine, handle) = Engine::new(config, &api_config); + engine + .mcp_pool_slot + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .handoff = mcp_handoff; + Ok((Box::new(engine), handle)) + }) + .await + .expect("Engine initialization owner failed"); // Box the run future before supervision. An extra async wrapper embeds // the large engine state again in both its own and the supervisor's poll // frames, which can overflow an ordinary worker-thread stack. spawn_supervised( "engine-event-loop", std::panic::Location::caller(), - Box::pin(engine.run()), + Box::pin((*engine).run()), ); - handle } diff --git a/crates/engine/src/fleet/executor.rs b/crates/engine/src/fleet/executor.rs index 91a61f926..c08afb3be 100644 --- a/crates/engine/src/fleet/executor.rs +++ b/crates/engine/src/fleet/executor.rs @@ -22,7 +22,9 @@ use anyhow::Result; use codewhale_config::FleetExecConfig; -use codewhale_protocol::fleet::{FleetHostSpec, FleetTaskSpec, FleetWorkerEventPayload}; +use codewhale_protocol::fleet::{ + FleetHostSpec, FleetReceipt, FleetTaskSpec, FleetWorkerEventPayload, +}; use super::host::{FleetHostAdapter, FleetWorkerCommand}; use super::profile::AgentProfile; @@ -595,6 +597,8 @@ struct WorkerStream { // corrupts valid UTF-8 when a multibyte code point crosses a read boundary. pending: Vec, terminal: bool, + confirmed_stopped: bool, + pending_terminal: Option, terminal_route: TerminalRouteEvidence, /// When this worker process was started, for per-task wall-clock limits (R5). started_at: std::time::Instant, @@ -720,20 +724,25 @@ pub struct FleetWorkerTerminalEvent { pub requires_reported_route: bool, } +#[derive(Debug, Clone)] +pub(crate) struct FleetPreparedTaskOutcome { + pub payload: FleetWorkerEventPayload, + pub tail_payloads: Vec, + pub final_status: Option, + pub receipt: FleetReceipt, +} + +#[derive(Debug, Clone)] +enum PendingFleetTerminal { + Observed(FleetWorkerTerminalEvent), + Prepared(FleetPreparedTaskOutcome), +} + impl FleetExecutor { - pub async fn refresh_tailnet_inventory(&mut self) { - let snapshot = tokio::task::spawn_blocking(|| { - super::tailnet::TailnetSnapshot::new( - super::tailnet::inventory().map_err(|error| error.to_string()), - ) - }) - .await - .unwrap_or_else(|error| { - super::tailnet::TailnetSnapshot::new(Err(format!( - "Tailnet inventory task failed: {error}" - ))) - }); - self.tailnet_snapshot = Some(std::sync::Arc::new(snapshot)); + pub fn refresh_tailnet_inventory(&mut self) { + self.tailnet_snapshot = Some(std::sync::Arc::new(super::tailnet::TailnetSnapshot::new( + super::tailnet::inventory().map_err(|error| error.to_string()), + ))); } pub fn new(workspace: impl AsRef) -> Self { @@ -855,6 +864,8 @@ impl FleetExecutor { offset: 0, pending: Vec::new(), terminal: false, + confirmed_stopped: false, + pending_terminal: None, terminal_route: TerminalRouteEvidence::default(), started_at: std::time::Instant::now(), final_answer: None, @@ -879,6 +890,79 @@ impl FleetExecutor { .and_then(|stream| stream.attempt.clone()) } + pub(crate) fn confirmed_stopped_attempts( + &self, + ) -> std::collections::BTreeMap { + self.streams + .iter() + .filter_map(|(id, stream)| { + stream + .confirmed_stopped + .then(|| stream.attempt.clone()) + .flatten() + .map(|attempt| (id.clone(), attempt)) + }) + .collect() + } + + pub(crate) fn is_confirmed_stopped(&self, worker_id: &str) -> bool { + self.streams + .get(worker_id) + .is_some_and(|stream| stream.confirmed_stopped) + } + + pub(crate) fn prepared_terminal_outcome( + &self, + worker_id: &str, + ) -> Option { + match self.streams.get(worker_id)?.pending_terminal.as_ref()? { + PendingFleetTerminal::Prepared(outcome) => Some(outcome.clone()), + PendingFleetTerminal::Observed(_) => None, + } + } + + pub(crate) fn retain_prepared_terminal_outcome( + &mut self, + worker_id: &str, + outcome: FleetPreparedTaskOutcome, + ) -> Result<()> { + let stream = self + .streams + .get_mut(worker_id) + .ok_or_else(|| anyhow::anyhow!("Fleet terminal stream is absent"))?; + anyhow::ensure!( + stream.confirmed_stopped, + "Fleet terminal process is not confirmed stopped" + ); + if let Some(attempt) = &stream.attempt { + anyhow::ensure!( + outcome.receipt.run_id == attempt.run_id + && outcome.receipt.task_id == attempt.task_id + && outcome.receipt.attempt == Some(attempt.attempt) + && outcome.receipt.worker_id == worker_id, + "Fleet terminal candidate does not match its tracked attempt" + ); + } + stream.pending_terminal = Some(PendingFleetTerminal::Prepared(outcome)); + Ok(()) + } + + pub(crate) fn poll_terminal_for_commit( + &mut self, + worker_id: &str, + ) -> Option { + if let Some(pending) = self.streams.get(worker_id)?.pending_terminal.as_ref() { + return match pending { + PendingFleetTerminal::Observed(event) => Some(event.clone()), + PendingFleetTerminal::Prepared(_) => None, + }; + } + let event = self.poll_terminal_with_status(worker_id)?; + self.streams.get_mut(worker_id)?.pending_terminal = + Some(PendingFleetTerminal::Observed(event.clone())); + Some(event) + } + /// Wall-clock time this worker process has been running (R5). `None` when /// the worker is not tracked or its exit was already observed: a process /// that has exited is not running, so a deadline must not rewrite the @@ -912,6 +996,9 @@ impl FleetExecutor { } else { self.adapter.stop_worker(worker_id)?; } + if let Some(stream) = self.streams.get_mut(worker_id) { + stream.confirmed_stopped = true; + } Ok(()) } @@ -1027,6 +1114,7 @@ impl FleetExecutor { tail_payloads.push(payload); } stream.terminal = true; + stream.confirmed_stopped = true; let final_answer = stream.final_answer.take(); // Surface the visible final answer on a successful completion so // report/summary tasks (no scorer, no file artifact) show their @@ -1198,6 +1286,8 @@ mod tests { offset: 0, pending: Vec::new(), terminal: false, + confirmed_stopped: false, + pending_terminal: None, terminal_route: TerminalRouteEvidence::default(), started_at: std::time::Instant::now(), final_answer: None, diff --git a/crates/engine/src/fleet/ledger.rs b/crates/engine/src/fleet/ledger.rs index ec194df3f..68107d800 100644 --- a/crates/engine/src/fleet/ledger.rs +++ b/crates/engine/src/fleet/ledger.rs @@ -153,6 +153,8 @@ pub enum FleetLedgerRecord { /// publishing a terminal task without the receipt that explains it. TaskAttemptFinalized { event: FleetWorkerEvent, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + tail_events: Vec, #[serde(default, skip_serializing_if = "Option::is_none")] final_status: Option, receipt: Box, @@ -340,6 +342,7 @@ pub struct FleetLedger { ledger_file: WorkspaceFile, lock_file: WorkspaceFile, original_lock: std::fs::File, + durable_barrier: parking_lot::Mutex>, /// Stable advisory-lock inode shared by every manager for this workspace. /// /// The ledger itself is replaced during compaction, so locking @@ -369,6 +372,7 @@ impl FleetLedger { ledger_file, lock_file, original_lock, + durable_barrier: parking_lot::Mutex::new(None), ledger_path, lock_path, #[cfg(any(test, feature = "test-support"))] @@ -444,6 +448,7 @@ impl FleetLedger { .ledger_file .open_update(true, true) .with_context(|| format!("opening fleet ledger {}", self.ledger_path.display()))?; + self.durable_barrier.lock().take(); // A process can die after writing only part of its final JSON record. // O_APPEND would otherwise concatenate the next valid record directly // onto that unterminated tail, causing replay to discard both. Preserve @@ -480,6 +485,8 @@ impl FleetLedger { .with_context(|| format!("flushing fleet ledger {}", self.ledger_path.display()))?; file.sync_data() .with_context(|| format!("syncing fleet ledger {}", self.ledger_path.display()))?; + let length = file.metadata()?.len(); + *self.durable_barrier.lock() = Some((file, length)); // Wake SSE subscribers after the bytes are durable (#6211 R7b). Every // record kind funnels through here, so a wake only promises "something // changed" — subscribers re-poll with their cursor. @@ -954,16 +961,40 @@ impl FleetLedger { timestamp: &str, payload: FleetWorkerEventPayload, final_status: Option, + receipt: FleetReceipt, + ) -> Result> { + self.finalize_task_attempt_with_tail_if_leased( + run_id, + worker_id, + task_id, + expected_attempts, + timestamp, + payload, + Vec::new(), + final_status, + receipt, + ) + } + + #[allow(clippy::too_many_arguments)] + pub(crate) fn finalize_task_attempt_with_tail_if_leased( + &self, + run_id: &FleetRunId, + worker_id: &str, + task_id: &str, + expected_attempts: u32, + timestamp: &str, + payload: FleetWorkerEventPayload, + tail_payloads: Vec, + final_status: Option, mut receipt: FleetReceipt, ) -> Result> { - if !matches!( - &payload, - FleetWorkerEventPayload::Completed { .. } - | FleetWorkerEventPayload::Failed { .. } - | FleetWorkerEventPayload::Cancelled { .. } - ) { - bail!("attempt finalization requires a terminal worker event"); - } + let payload_status = match &payload { + FleetWorkerEventPayload::Completed { .. } => FleetTaskLedgerStatus::Completed, + FleetWorkerEventPayload::Failed { .. } => FleetTaskLedgerStatus::Failed, + FleetWorkerEventPayload::Cancelled { .. } => FleetTaskLedgerStatus::Cancelled, + _ => bail!("attempt finalization requires a terminal worker event"), + }; if final_status.is_some_and(|status| { !matches!( status, @@ -984,28 +1015,126 @@ impl FleetLedger { { bail!("attempt receipt generation does not match its lease"); } - self.with_write_lock(|| { + for tail in &tail_payloads { + if matches!( + tail, + FleetWorkerEventPayload::Completed { .. } + | FleetWorkerEventPayload::Failed { .. } + | FleetWorkerEventPayload::Cancelled { .. } + ) { + bail!("terminal tail contains another terminal outcome"); + } + } + let had_usage = tail_payloads + .iter() + .any(|payload| matches!(payload, FleetWorkerEventPayload::UsageReport { .. })); + let tail_digest = (!tail_payloads.is_empty()) + .then(|| serde_json::to_vec(&tail_payloads)) + .transpose()? + .map(|bytes| codewhale_runtime::hashing::sha256_hex(&bytes)); + let appended = self.with_write_lock(|| { let state = self.rebuild_state_unlocked()?; let key = task_key(&run_id.0, task_id); let Some(task) = state.tasks.get(&key) else { return Ok(None); }; - if task.status != FleetTaskLedgerStatus::Leased - || task.leased_to.as_deref() != Some(worker_id) + if task.leased_to.as_deref() != Some(worker_id) || task.entry.attempts != expected_attempts { return Ok(None); } - let event = next_worker_event(&state, run_id, worker_id, task_id, timestamp, payload); + let lifecycle_key = event_key(worker_id, &run_id.0, task_id); + if task.status != FleetTaskLedgerStatus::Leased { + let Some(existing_receipt) = state.receipts.get(&key) else { + return Ok(None); + }; + let Some(existing_event) = state.latest_events.get(&lifecycle_key) else { + return Ok(None); + }; + receipt.attempt = Some(expected_attempts); + receipt.terminal_seq = Some(existing_event.seq); + let mut expected_event = next_worker_event( + &state, + run_id, + worker_id, + task_id, + timestamp, + payload.clone(), + ); + expected_event.seq = existing_event.seq; + if let Some(digest) = &tail_digest { + expected_event + .extra + .insert("terminal_tail_sha256".into(), Value::String(digest.clone())); + } + if task.status != final_status.unwrap_or(payload_status) + || serde_json::to_value(existing_receipt)? != serde_json::to_value(&receipt)? + || serde_json::to_value(existing_event)? + != serde_json::to_value(&expected_event)? + { + return Ok(None); + } + self.sync_durable_unlocked() + .context("retrying exact Fleet terminal durability")?; + return Ok(Some(existing_event.clone())); + } + let mut sequence = state.latest_seq.get(&lifecycle_key).copied().unwrap_or(0); + let mut tail_events = Vec::with_capacity(tail_payloads.len()); + for tail in tail_payloads { + let mut event = + next_worker_event(&state, run_id, worker_id, task_id, timestamp, tail); + sequence = sequence + .checked_add(1) + .ok_or_else(|| anyhow::anyhow!("Fleet event sequence exhausted"))?; + event.seq = sequence; + tail_events.push(event); + } + let mut event = + next_worker_event(&state, run_id, worker_id, task_id, timestamp, payload); + event.seq = sequence + .checked_add(1) + .ok_or_else(|| anyhow::anyhow!("Fleet event sequence exhausted"))?; + if let Some(digest) = tail_digest { + event + .extra + .insert("terminal_tail_sha256".into(), Value::String(digest)); + } receipt.attempt = Some(expected_attempts); receipt.terminal_seq = Some(event.seq); self.append_record_unlocked(&FleetLedgerRecord::TaskAttemptFinalized { event: event.clone(), + tail_events, final_status, receipt: Box::new(receipt), })?; Ok(Some(event)) - }) + }); + if matches!(&appended, Ok(Some(_))) && had_usage { + self.record_budget_exceeded_alert_once(run_id, timestamp)?; + } + appended + } + + pub(crate) fn ensure_durable(&self) -> Result<()> { + self.with_write_lock(|| self.sync_durable_unlocked()) + } + + fn sync_durable_unlocked(&self) -> Result<()> { + let file = self.ledger_file.open_update(false, false)?; + let length = file.metadata()?.len(); + let mut cached = self.durable_barrier.lock(); + if let Some((previous, durable_length)) = cached.take() + && durable_length == length + && same_file(&previous, &file)? + { + *cached = Some((previous, durable_length)); + return Ok(()); + } + file.sync_data() + .context("acknowledging Fleet ledger durability")?; + let length = file.metadata()?.len(); + *cached = Some((file, length)); + Ok(()) } /// Append a scheduler-owned terminal event only if the stale lease version @@ -1770,6 +1899,7 @@ impl FleetLedger { if !contents.is_empty() { contents.push('\n'); } + self.durable_barrier.lock().take(); self.ledger_file .replace(contents.as_bytes()) .with_context(|| { @@ -1871,10 +2001,38 @@ fn runtime_events_from_record( worker_seq: None, payload: json!({ "status": status }), }], - FleetLedgerRecord::TaskAttemptFinalized { event, receipt, .. } => vec![ - runtime_worker_event("worker", ledger_ordinal, replay_epoch, event), - runtime_receipt_event("receipt", ledger_ordinal, replay_epoch, receipt), - ], + FleetLedgerRecord::TaskAttemptFinalized { + event, + tail_events, + receipt, + .. + } => { + let mut events = tail_events + .iter() + .enumerate() + .map(|(index, event)| { + runtime_worker_event( + &format!("tail-{index}"), + ledger_ordinal, + replay_epoch, + event, + ) + }) + .collect::>(); + events.push(runtime_worker_event( + "worker", + ledger_ordinal, + replay_epoch, + event, + )); + events.push(runtime_receipt_event( + "receipt", + ledger_ordinal, + replay_epoch, + receipt, + )); + events + } FleetLedgerRecord::EventAppended { event } => { vec![runtime_worker_event( "worker", @@ -2287,9 +2445,13 @@ fn apply_record(state: &mut FleetLedgerState, record: FleetLedgerRecord) { } FleetLedgerRecord::TaskAttemptFinalized { event, + tail_events, final_status, receipt, } => { + for tail in tail_events { + apply_record(state, FleetLedgerRecord::EventAppended { event: tail }); + } let run_id = event.run_id.clone(); let task_id = event.task_id.clone(); let worker_id = event.worker_id.clone(); diff --git a/crates/engine/src/fleet/manager.rs b/crates/engine/src/fleet/manager.rs index 20ebc02a0..1d2123e20 100644 --- a/crates/engine/src/fleet/manager.rs +++ b/crates/engine/src/fleet/manager.rs @@ -7,7 +7,6 @@ #![allow(dead_code)] use std::collections::{BTreeMap, BTreeSet}; -use std::io::ErrorKind; use std::path::{Path, PathBuf}; use std::time::Duration; @@ -18,8 +17,9 @@ use serde_json::Value; use uuid::Uuid; use super::executor::{ - FleetExecutor, FleetExecutorAttempt, FleetWorkerReportedRoute, FleetWorkerTerminalEvent, - authority_envelope_for_worker, build_worker_exec_command_with_launch_spec, + FleetExecutor, FleetExecutorAttempt, FleetPreparedTaskOutcome, FleetWorkerReportedRoute, + FleetWorkerTerminalEvent, authority_envelope_for_worker, + build_worker_exec_command_with_launch_spec, }; use super::host::FleetHostErrorKind; use super::ledger::{ @@ -36,9 +36,16 @@ use codewhale_runtime::config::Config; const DEFAULT_STALE_AFTER_SECONDS: u64 = 300; +#[derive(Debug, thiserror::Error)] +#[error("worker {worker_id} task changed before it could be restarted")] +struct FleetRestartChanged { + worker_id: String, +} + +#[derive(Clone)] pub struct FleetManager { workspace: PathBuf, - ledger: FleetLedger, + ledger: std::sync::Arc, stale_after: Duration, exec_config: codewhale_config::FleetExecConfig, /// `[fleet]` table used to build the agent roster for dispatch @@ -300,7 +307,7 @@ struct FleetExecutorTaskContext { impl FleetManager { pub fn open(workspace: impl AsRef) -> Result { let workspace = workspace.as_ref().to_path_buf(); - let ledger = FleetLedger::open(&workspace)?; + let ledger = std::sync::Arc::new(FleetLedger::open(&workspace)?); Ok(Self { workspace, ledger, @@ -628,7 +635,9 @@ impl FleetManager { max_workers: usize, unavailable_workers: &BTreeSet, ) -> Result { - self.reconcile_coordination_worker_statuses()?; + if !self.reconcile_coordination_worker_statuses(run_id, &BTreeMap::new())? { + return Ok(FleetTickReport::default()); + } let max_workers = max_workers.clamp(1, 128); let mut report = FleetTickReport::default(); let state = self.ledger.rebuild_state()?; @@ -694,58 +703,167 @@ impl FleetManager { Ok(report) } - fn reconcile_coordination_worker_statuses(&self) -> Result<()> { + fn reconcile_coordination_worker_statuses( + &self, + run_id: &FleetRunId, + stopped_workers: &BTreeMap, + ) -> Result { let Some(manager) = self.sub_agent_manager.as_ref() else { - return Ok(()); + return Ok(true); }; let Ok(mut guard) = manager.try_write() else { - // The next scheduler tick retries before leasing more work. - return Ok(()); + return Ok(false); }; let state = self.ledger.rebuild_state()?; - for record in guard - .fleet_worker_records_for_workspace(&self.workspace) - .map_err(anyhow::Error::msg)? + if state.tasks.values().any(|task| { + matches!( + task.status, + FleetTaskLedgerStatus::Completed + | FleetTaskLedgerStatus::Failed + | FleetTaskLedgerStatus::Cancelled + ) + }) || state + .receipts + .values() + .any(|receipt| receipt.attempt.is_some() && receipt.terminal_seq.is_some()) { - let current = state - .tasks - .values() - .filter(|task| task.entry.run_id.0 == record.spec.run_id) - .filter(|task| task.leased_to.as_deref() == Some(record.spec.worker_id.as_str())) - .max_by_key(|task| task.lifecycle_seq); - let Some(current) = current else { - continue; - }; - let (status, status_label) = match current.status { - FleetTaskLedgerStatus::Enqueued => { - (crate::tools::subagent::AgentWorkerStatus::Queued, "queued") - } - FleetTaskLedgerStatus::Leased => ( - crate::tools::subagent::AgentWorkerStatus::Running, - "running", - ), - FleetTaskLedgerStatus::Completed => ( - crate::tools::subagent::AgentWorkerStatus::Completed, - "completed", - ), - FleetTaskLedgerStatus::Failed => { - (crate::tools::subagent::AgentWorkerStatus::Failed, "failed") - } - FleetTaskLedgerStatus::Cancelled => ( - crate::tools::subagent::AgentWorkerStatus::Cancelled, - "cancelled", - ), - }; - guard.project_external_worker_status( - &record.spec.worker_id, - status, - Some(format!( - "Fleet task {} is {}", - current.entry.task_id, status_label - )), - ); + self.ledger.ensure_durable()?; } - Ok(()) + let roster = self.agent_roster(); + guard.with_store_transaction(|held| { + let mut reconciled = true; + for mut record in held + .fleet_worker_records_for_workspace(&self.workspace) + .map_err(anyhow::Error::msg)? + { + let current = state + .tasks + .values() + .filter(|task| task.entry.run_id.0 == record.spec.run_id) + .filter(|task| { + task.leased_to.as_deref() == Some(record.spec.worker_id.as_str()) + }) + .max_by_key(|task| task.lifecycle_seq); + let Some(current) = current else { + continue; + }; + if record.spec.run_id != run_id.0 + && !(held.owns_store_record(&record.spec.worker_id) + && held.get_result(&record.spec.worker_id).is_err() + && record.started_at_ms.is_none() + && record.prepared_generation.is_none() + && record.launch_admission_version == 1 + && !record.recovery_interrupted + && record.spec.launch_manifest.as_ref().is_some_and(|manifest| manifest.generation == 1) + && matches!(current.status, FleetTaskLedgerStatus::Completed | FleetTaskLedgerStatus::Failed | FleetTaskLedgerStatus::Cancelled)) + { + continue; + } + let run = state + .runs + .get(¤t.entry.run_id.0) + .ok_or_else(|| anyhow!("Fleet lifecycle run is absent"))?; + let task_spec = run + .task_specs + .iter() + .find(|task| task.id == current.entry.task_id) + .ok_or_else(|| anyhow!("Fleet lifecycle task spec is absent"))?; + let worker_spec = run + .worker_specs + .iter() + .find(|worker| worker.id == record.spec.worker_id) + .cloned() + .unwrap_or_else(|| default_local_worker(&record.spec.worker_id)); + let cwd = resolve_task_cwd(&self.workspace, task_spec)?; + let expected = bind_fleet_launch_attempt( + worker_runtime::apply_exec_hardening( + worker_runtime::fleet_task_to_worker_spec_with_profiles( + &record.spec.worker_id, + ¤t.entry.run_id.0, + task_spec, + &worker_spec, + self.run_model(), + &cwd, + &self.workspace, + roster.members(), + None, + )?, + &self.exec_config, + ), + current.entry.attempts, + ); + let mut registered = record.spec.clone(); + if record.prepared_generation.is_some() + && let (Some(manifest), Some(expected_manifest)) = ( + registered.launch_manifest.as_mut(), + expected.launch_manifest.as_ref(), + ) + && expected_manifest.generation.checked_add(1) == Some(manifest.generation) + { + manifest.generation = expected_manifest.generation; + } + validate_registered_launch_spec(®istered, &expected)?; + let (status, status_label) = match current.status { + FleetTaskLedgerStatus::Enqueued => { + (crate::tools::subagent::AgentWorkerStatus::Queued, "queued") + } + FleetTaskLedgerStatus::Leased => ( + crate::tools::subagent::AgentWorkerStatus::Running, + "running", + ), + FleetTaskLedgerStatus::Completed => ( + crate::tools::subagent::AgentWorkerStatus::Completed, + "completed", + ), + FleetTaskLedgerStatus::Failed => { + (crate::tools::subagent::AgentWorkerStatus::Failed, "failed") + } + FleetTaskLedgerStatus::Cancelled => ( + crate::tools::subagent::AgentWorkerStatus::Cancelled, + "cancelled", + ), + }; + let receipt = state.receipts.get(&task_key(¤t.entry.run_id.0, ¤t.entry.task_id)); + let receipt_worker_id = record.spec.worker_id.clone(); + let receipt_matches = |attempt: u32| receipt.is_some_and(|receipt| { + receipt.run_id == current.entry.run_id && receipt.task_id == current.entry.task_id + && receipt.worker_id == receipt_worker_id && receipt.attempt == Some(attempt) + && receipt.terminal_seq.is_some() + }); + let generation = record.spec.launch_manifest.as_ref().map_or(0, |manifest| manifest.generation); + if record.prepared_previous_attempt_unsettled { + let previous_generation = generation.checked_sub(1).ok_or_else(|| anyhow!("invalid prepared Fleet generation"))?; + let stopped = stopped_workers.get(&record.spec.worker_id).is_some_and(|attempt| attempt.run_id == current.entry.run_id && attempt.task_id == current.entry.task_id && attempt.attempt == previous_generation); + if stopped || receipt_matches(previous_generation) { + let mut previous = record.spec.clone(); + previous.launch_manifest.as_mut().expect("prepared manifest checked").generation = previous_generation; + held.confirm_previous_worker_attempt_settled(&previous)?; + record = held.get_worker_record(&record.spec.worker_id).expect("settled prepared worker exists"); + } else { + if status.is_terminal() { reconciled = false; } + continue; + } + } + if status.is_terminal() && !(record.status.is_terminal() && !record.recovery_interrupted) { + let unstarted = record.started_at_ms.is_none() && (record.launch_admission_version == 1 && generation == 1 || record.prepared_generation == Some(generation)); + let stopped = stopped_workers.get(&record.spec.worker_id).is_some_and(|attempt| attempt.run_id == current.entry.run_id && attempt.task_id == current.entry.task_id && attempt.attempt == current.entry.attempts); + if !unstarted && !stopped && !receipt_matches(current.entry.attempts) { + anyhow::ensure!(!record.recovery_interrupted, "Fleet worker {} has no acknowledged actual process outcome; retain its uncertain attempt", record.spec.worker_id); + reconciled = false; + continue; + } + } + held.project_external_worker_status( + ®istered, + status, + Some(format!( + "Fleet task {} is {}", + current.entry.task_id, status_label + )), + )?; + } + Ok(reconciled) + }) } pub fn status(&self) -> Result { @@ -832,11 +950,11 @@ impl FleetManager { &self, run_id: &FleetRunId, max_workers: usize, - executor: &mut FleetExecutor, + mut executor: FleetExecutor, codewhale_binary: &str, model: Option<&str>, tick_interval: Duration, - ) -> Result { + ) -> Result<(FleetStatusSnapshot, FleetExecutor)> { let max_workers = max_workers.clamp(1, 128); let manager_lock_path = self.manager_lock_path(run_id); // Directory creation and the lock-file open are blocking filesystem @@ -846,7 +964,7 @@ impl FleetManager { let path = manager_lock_path.clone(); let workspace = self.workspace.clone(); let relative = Self::manager_lock_relative_path(run_id); - tokio::task::spawn_blocking(move || -> Result { + codewhale_runtime::runtime_control::owner_work(move || -> Result { // Created and opened through the pinned no-follow handle, so a // linked `.codewhale` or lock name is refused, not followed. codewhale_util::files::WorkspaceFile::open(&workspace, &relative, true) @@ -854,101 +972,103 @@ impl FleetManager { .with_context(|| format!("opening Fleet manager lock {}", path.display())) }) .await - .context("Fleet manager lock setup task failed to join")?? + .context("Fleet manager lock setup task failed")? }; - let mut manager_lock = fd_lock::RwLock::new(lock_file); + let manager_guard = std::sync::Arc::new(lock_file); let standby_interval = tick_interval .min(Duration::from_millis(100)) .max(Duration::from_millis(10)); let mut observed_owner = false; - let _manager_guard = loop { - match manager_lock.try_write() { - Ok(guard) => { - if observed_owner { - if self.run_has_open_work(run_id)? { - bail!( - "Fleet manager for run {} exited with open work; wait for stale reconciliation before resuming", - run_id.0 - ); - } - return self.run_status(run_id); - } - break guard; - } - Err(err) if err.kind() == ErrorKind::WouldBlock => { - // Another process owns this run. Wait for it to finish, - // but never treat lock release as permission to relaunch - // its unchanged leased attempts: an orphan child may still - // be alive after a crash. Stale reconciliation owns that - // recovery/generation transition. - observed_owner = true; - if !self.run_has_open_work(run_id)? { - return self.run_status(run_id); - } - tokio::time::sleep(standby_interval).await; - } - Err(err) => { - return Err(err).with_context(|| { - format!( - "locking Fleet manager ownership {}", - manager_lock_path.display() - ) - }); - } - } - }; - let tailnet_workers: BTreeSet<_> = self - .ledger - .rebuild_state()? - .runs - .get(&run_id.0) - .into_iter() - .flat_map(|run| &run.worker_specs) - .filter(|worker| matches!(worker.host, FleetHostSpec::Tailscale { .. })) - .map(|worker| worker.id.clone()) - .collect(); loop { - // A terminal ledger update can race the foreground host process. - // Do not lease new work onto a logical worker until its executor - // handle has been observed and forgotten below. - let unavailable_workers = executor.worker_ids().into_iter().collect(); - let scheduling_error = self - .schedule_run_excluding(run_id, max_workers, &unavailable_workers) - .err(); - if !tailnet_workers.is_empty() { - let state = self.ledger.rebuild_state()?; - let needs_inventory = active_tasks_for_run(&state, run_id).any(|task| { - task.leased_to.as_deref().is_some_and(|worker| { - tailnet_workers.contains(worker) && !executor.is_tracking(worker) - }) - }); - if needs_inventory { - executor.refresh_tailnet_inventory().await; - } + let acquired = codewhale_util::lock::try_lock_file_exclusive(&manager_guard) + .with_context(|| { + format!( + "locking Fleet manager ownership {}", + manager_lock_path.display() + ) + })?; + if acquired && !observed_owner { + break; } - self.drive_executor_tick(run_id, executor, codewhale_binary, model)?; - self.refresh_run_status(run_id)?; - if let Some(error) = scheduling_error { - if executor.worker_ids().is_empty() { - return Err(error).with_context(|| { - format!( - "scheduling Fleet run {} after draining owned workers", - run_id.0 - ) - }); - } - tracing::warn!( - run_id = %run_id.0, - error = %error, - "Fleet scheduling paused while already-leased workers continue" + observed_owner = true; + let manager = self.clone(); + let owned_run_id = run_id.clone(); + let guard = std::sync::Arc::clone(&manager_guard); + let (open, status) = codewhale_runtime::runtime_control::owner_work(move || { + let _guard = guard; + let open = manager.run_has_open_work(&owned_run_id)?; + let status = if open { + None + } else { + Some(manager.run_status(&owned_run_id)?) + }; + Ok((open, status)) + }) + .await?; + if acquired && open { + bail!( + "Fleet manager for run {} exited with open work; wait for stale reconciliation before resuming", + run_id.0 ); } - // A separate `fleet interrupt` process can make the ledger - // terminal while this manager still owns a live host child. Keep - // driving until the executor has observed that cancellation and - // stopped every tracked process. - if !self.run_has_open_work(run_id)? && executor.worker_ids().is_empty() { - return self.run_status(run_id); + if let Some(status) = status { + return Ok((status, executor)); + } + tokio::time::sleep(standby_interval).await; + } + let manager = self.clone(); + let owned_run_id = run_id.clone(); + let guard = std::sync::Arc::clone(&manager_guard); + let tailnet_workers = codewhale_runtime::runtime_control::owner_work(move || { + let _guard = guard; + Ok(manager + .ledger + .rebuild_state()? + .runs + .get(&owned_run_id.0) + .into_iter() + .flat_map(|run| &run.worker_specs) + .filter(|worker| matches!(worker.host, FleetHostSpec::Tailscale { .. })) + .map(|worker| worker.id.clone()) + .collect::>()) + }) + .await?; + loop { + let manager = self.clone(); + let run_id = run_id.clone(); + let guard = std::sync::Arc::clone(&manager_guard); + let codewhale_binary = codewhale_binary.to_string(); + let model = model.map(str::to_string); + let tailnet_workers = tailnet_workers.clone(); + let (returned_executor, returned_guard, outcome) = codewhale_runtime::runtime_control::owner_work(move || { + let outcome = (|| -> Result> { + let unavailable_workers = executor.worker_ids().into_iter().collect(); + let scheduling_error = manager.schedule_run_excluding(&run_id, max_workers, &unavailable_workers).err(); + if !tailnet_workers.is_empty() { + let state = manager.ledger.rebuild_state()?; + if active_tasks_for_run(&state, &run_id).any(|task| task.leased_to.as_deref().is_some_and(|worker| tailnet_workers.contains(worker) && !executor.is_tracking(worker))) { + executor.refresh_tailnet_inventory(); + } + } + let execution_error = manager.drive_executor_tick(&run_id, &mut executor, &codewhale_binary, model.as_deref()).err(); + if let Some(error) = execution_error.or(scheduling_error) { + if executor.worker_ids().is_empty() { + return Err(error).with_context(|| format!("scheduling Fleet run {} after draining owned workers", run_id.0)); + } + tracing::warn!(run_id = %run_id.0, error = %error, "Fleet scheduling paused while already-leased workers continue"); + } + if !manager.run_has_open_work(&run_id)? && executor.worker_ids().is_empty() + && manager.reconcile_coordination_worker_statuses(&run_id, &BTreeMap::new())? { + return manager.run_status(&run_id).map(Some); + } + Ok(None) + })(); + Ok((executor, guard, outcome)) + }).await?; + executor = returned_executor; + drop(returned_guard); + if let Some(status) = outcome? { + return Ok((status, executor)); } tokio::time::sleep(tick_interval).await; } @@ -962,119 +1082,156 @@ impl FleetManager { model: Option<&str>, ) -> Result { let mut report = FleetExecutorTickReport::default(); - report.started += self.start_leased_workers(run_id, executor, codewhale_binary, model)?; - + let stopped = executor.confirmed_stopped_attempts(); + let reconciliation = self.reconcile_coordination_worker_statuses(run_id, &stopped); + let reconciled = reconciliation.as_ref().is_ok_and(|ready| *ready); + let reconciliation_error = reconciliation.err(); + if let Some(manager) = self.sub_agent_manager.as_ref() + && let Ok(guard) = manager.try_read() + { + let state = self.ledger.rebuild_state()?; + for record in guard + .fleet_worker_records_for_workspace(&self.workspace) + .map_err(anyhow::Error::msg)? + { + if record.spec.run_id == run_id.0 && record.prepared_previous_attempt_unsettled { + let generation = record + .spec + .launch_manifest + .as_ref() + .map_or(0, |manifest| manifest.generation); + let task = state + .tasks + .values() + .filter(|task| { + task.entry.run_id == *run_id + && task.leased_to.as_deref() == Some(record.spec.worker_id.as_str()) + }) + .max_by_key(|task| task.lifecycle_seq); + anyhow::ensure!( + executor + .tracked_attempt(&record.spec.worker_id) + .is_some_and(|attempt| attempt.run_id == *run_id + && task.is_some_and(|task| attempt.task_id == task.entry.task_id) + && attempt.attempt.checked_add(1) == Some(generation)), + "Fleet worker {} previous process teardown is unacknowledged; retain its prepared attempt", + record.spec.worker_id + ); + } + } + } + if reconciled { + report.started += + self.start_leased_workers(run_id, executor, codewhale_binary, model)?; + } for worker_id in executor.worker_ids() { + if let Some(outcome) = executor.prepared_terminal_outcome(&worker_id) { + if self.publish_task_outcome(&outcome)? { + report.terminals += 1; + } + continue; + } let tracked_attempt = executor.tracked_attempt(&worker_id); let tracked_task = match tracked_attempt.as_ref() { Some(attempt) => { - let Some(task) = self.executor_task_context_for_attempt(&worker_id, attempt)? - else { - // The ledger advanced to another attempt (restart), or - // made this attempt terminal (cancel/stop), while this - // process was still alive. The executor owns the host - // handle, so fence and reap the old process without - // publishing any event against the replacement - // generation. - executor.stop_worker(&worker_id)?; - executor.forget_worker(&worker_id); - report.terminals += 1; - continue; - }; - Some(task) + match self.executor_task_context_for_attempt(&worker_id, attempt)? { + Some(task) => Some(task), + None => { + executor.stop_worker(&worker_id)?; + report.terminals += 1; + continue; + } + } } None => None, }; if tracked_attempt.is_none() - && let Some(_task) = self.cancelled_executor_task_context(&worker_id)? + && self.cancelled_executor_task_context(&worker_id)?.is_some() { - // Cancellation is ledgered by an out-of-process control - // command. Only this executor owns the host process handle, - // so it must enforce the terminal state before returning from - // the foreground manager loop. Do not ingest output produced - // after cancellation; publish one final authoritative event - // after the process is actually stopped instead. executor.stop_worker(&worker_id)?; - executor.forget_worker(&worker_id); report.terminals += 1; continue; } - - for payload in executor.drain_events(&worker_id) { - // The subprocess exit is the task-completion authority. Stream - // `done` / `error` lines are useful progress signals, but - // appending them as terminal ledger events before the process - // exits would free the logical worker too early. - if is_terminal_payload(&payload) { - continue; + let mut observed = executor.poll_terminal_for_commit(&worker_id); + if observed.is_none() && !executor.is_confirmed_stopped(&worker_id) { + for payload in executor.drain_events(&worker_id) { + if is_terminal_payload(&payload) { + continue; + } + let task = match tracked_attempt.as_ref() { + Some(attempt) => { + self.executor_task_context_for_attempt(&worker_id, attempt)? + } + None => self.executor_task_context(&worker_id)?, + }; + let Some(task) = task else { + continue; + }; + if self + .ledger + .append_event_if_leased( + &task.entry.run_id, + &worker_id, + &task.entry.task_id, + task.entry.attempts, + ×tamp(), + payload, + )? + .is_some() + { + self.ledger + .heartbeat(&worker_id, ×tamp(), None, None)?; + report.events += 1; + } } - let task = if let Some(attempt) = tracked_attempt.as_ref() { - self.executor_task_context_for_attempt(&worker_id, attempt)? - } else { - self.executor_task_context(&worker_id)? - }; - let Some(task) = task else { - continue; - }; - if self - .ledger - .append_event_if_leased( - &task.entry.run_id, - &worker_id, - &task.entry.task_id, - task.entry.attempts, - ×tamp(), - payload, - )? - .is_none() - { - continue; - } - self.ledger - .heartbeat(&worker_id, ×tamp(), None, None)?; - report.events += 1; + observed = executor.poll_terminal_for_commit(&worker_id); } - - let Some(terminal) = executor.poll_terminal_with_status(&worker_id) else { - // Per-task wall-clock limit (R5), enforced on the live - // process: a worker that never exits must not hold its lease - // (and spend) forever. Stop it first, then finalize with an - // honest Timeout receipt. - let task = match tracked_task { - Some(task) => Some(task), + if let Some(terminal) = observed { + let task = match tracked_attempt.as_ref() { + Some(attempt) => self.executor_task_context_for_attempt(&worker_id, attempt)?, None => self.executor_task_context(&worker_id)?, }; if let Some(task) = task - && let Some(limit) = task_wall_clock_limit(&task.task_spec) - && executor - .worker_running_for(&worker_id) - .is_some_and(|running| running >= limit) + && let Some(outcome) = self.prepare_task_outcome(&task, terminal)? { - executor.stop_worker(&worker_id)?; - executor.forget_worker(&worker_id); - self.record_task_timeout(&task, limit)?; - report.terminals += 1; + executor.retain_prepared_terminal_outcome(&worker_id, outcome.clone())?; + if self.publish_task_outcome(&outcome)? { + report.terminals += 1; + } } continue; - }; - // The process already exited: its exit is the completion - // authority, even when this tick observed it after the deadline. - let task = if let Some(attempt) = tracked_attempt.as_ref() { - self.executor_task_context_for_attempt(&worker_id, attempt)? - } else { - self.executor_task_context(&worker_id)? - }; - let Some(task) = task else { - executor.forget_worker(&worker_id); - continue; - }; - if self.record_task_outcome(&task, terminal)? { - report.terminals += 1; } - executor.forget_worker(&worker_id); + let task = match tracked_task { + Some(task) => Some(task), + None => self.executor_task_context(&worker_id)?, + }; + if let Some(task) = task + && let Some(limit) = task_wall_clock_limit(&task.task_spec) + && executor + .worker_running_for(&worker_id) + .is_some_and(|running| running >= limit) + { + executor.stop_worker(&worker_id)?; + if let Some(outcome) = self.prepare_task_timeout(&task, limit)? { + executor.retain_prepared_terminal_outcome(&worker_id, outcome.clone())?; + if self.publish_task_outcome(&outcome)? { + report.terminals += 1; + } + } + } + } + let stopped = executor.confirmed_stopped_attempts(); + if self.reconcile_coordination_worker_statuses(run_id, &stopped)? { + for worker_id in executor.worker_ids() { + if executor.is_confirmed_stopped(&worker_id) { + executor.forget_worker(&worker_id); + } + } } - self.refresh_run_status(run_id)?; + if let Some(error) = reconciliation_error { + return Err(error); + } Ok(report) } @@ -1262,15 +1419,20 @@ impl FleetManager { .ok_or_else(|| { anyhow!("Fleet task {} does not exist", task.entry.task_id) })?; - self.prepare_registered_restart_generation( - guard, &state, task, task_spec, worker_id, - )?; + guard.with_store_transaction(|held| { + self.prepare_registered_restart_generation( + held, &state, task, task_spec, worker_id, + ) + })?; } Ok(()) }, )?; if !restarted { - bail!("worker {worker_id} task changed before it could be restarted"); + return Err(FleetRestartChanged { + worker_id: worker_id.to_string(), + } + .into()); } self.ledger .update_run_status(&task.entry.run_id, FleetRunStatus::Running, ×tamp())?; @@ -1528,9 +1690,7 @@ impl FleetManager { } None => None, }; - let registration_snapshot = coordination_guard - .as_ref() - .map(|guard| guard.coordination_registration_snapshot()); + let mut registration_snapshot = None; let worker_origin = coordination_guard .as_ref() .map(|guard| guard.capture_fleet_worker_origin(&origin_workspace)) @@ -1558,15 +1718,18 @@ impl FleetManager { // Registration shares the durable transition lock, so a // cancellation cannot win between claim and projection setup. if let Some(guard) = coordination_guard.as_mut() { - guard - .register_worker_with_coordination_in_origin( + guard.with_store_transaction(|held| { + registration_snapshot = Some(held.coordination_registration_snapshot()); + held.register_worker_with_coordination_in_origin( sub_agent_worker, worker_origin .clone() .expect("coordination guard captured worker origin"), ) .map_err(anyhow::Error::msg)?; - registration_succeeded = true; + registration_succeeded = true; + Ok(()) + })?; } Ok(()) }, @@ -1629,10 +1792,15 @@ impl FleetManager { .find(|worker| worker.id == worker_id) .cloned() .unwrap_or_else(|| default_local_worker(worker_id)); - let coordination_record = if let Some(manager) = self.sub_agent_manager.as_ref() { - let Ok(mut guard) = manager.try_write() else { + let mut coordination_guard = if let Some(manager) = self.sub_agent_manager.as_ref() { + let Ok(guard) = manager.try_write() else { continue; }; + Some(guard) + } else { + None + }; + let coordination_record = if let Some(guard) = coordination_guard.as_mut() { let record = guard .fleet_worker_records_for_workspace(&self.workspace) .map_err(anyhow::Error::msg)? @@ -1650,6 +1818,15 @@ impl FleetManager { } else { None }; + if coordination_record + .as_ref() + .and_then(Option::as_ref) + .is_some_and(|record| record.prepared_previous_attempt_unsettled) + { + bail!( + "Fleet worker {worker_id} previous process teardown is unacknowledged; retain its prepared attempt" + ); + } let preparation = (|| -> Result<_> { let cwd = resolve_task_cwd(&self.workspace, &task_spec)?; validate_task_cwd_for_host(&self.workspace, &worker_spec.host, &cwd)?; @@ -1688,16 +1865,17 @@ impl FleetManager { model, roster.members(), )?; - Ok((cwd, command)) + Ok((cwd, command, launch_spec)) })(); let attempt = FleetExecutorAttempt { run_id: task.entry.run_id.clone(), task_id: task.entry.task_id.clone(), attempt: task.entry.attempts, }; - let (cwd, command) = match preparation { + let (cwd, command, launch_spec) = match preparation { Ok(prepared) => prepared, Err(err) => { + drop(coordination_guard); let task = FleetExecutorTaskContext { entry: task.entry.clone(), task_spec, @@ -1719,6 +1897,10 @@ impl FleetManager { continue; } }; + if let Some(guard) = coordination_guard.as_mut() { + guard.activate_registered_worker_attempt(&launch_spec)?; + } + drop(coordination_guard); match executor.start_worker_attempt_on_host( worker_id, &worker_spec.host, @@ -1867,21 +2049,21 @@ impl FleetManager { })) } - fn record_task_outcome( + fn prepare_task_outcome( &self, task: &FleetExecutorTaskContext, terminal: FleetWorkerTerminalEvent, - ) -> Result { + ) -> Result> { let state = self.ledger.rebuild_state()?; let key = task_key(&task.entry.run_id.0, &task.entry.task_id); let Some(current) = state.tasks.get(&key) else { - return Ok(false); + return Ok(None); }; if current.status != FleetTaskLedgerStatus::Leased || current.leased_to.as_deref() != Some(task.worker_id.as_str()) || current.entry.attempts != task.entry.attempts { - return Ok(false); + return Ok(None); } let FleetWorkerTerminalEvent { @@ -1901,25 +2083,6 @@ impl FleetManager { FleetWorkerEventPayload::Cancelled { .. } => FleetTaskLedgerStatus::Cancelled, _ => bail!("Fleet executor outcome must contain a terminal worker event"), }; - for tail_payload in tail_payloads { - if is_terminal_payload(&tail_payload) { - continue; - } - if self - .ledger - .append_event_if_leased( - &task.entry.run_id, - &task.worker_id, - &task.entry.task_id, - task.entry.attempts, - ×tamp(), - tail_payload, - )? - .is_none() - { - return Ok(false); - } - } let artifacts = self.task_artifacts_for_receipt( &task.entry.run_id, &task.entry.task_id, @@ -1988,39 +2151,63 @@ impl FleetManager { FleetTaskResult::Fail | FleetTaskResult::Timeout ) && expected_terminal_status != FleetTaskLedgerStatus::Failed) .then_some(FleetTaskLedgerStatus::Failed); + Ok(Some(FleetPreparedTaskOutcome { + payload, + tail_payloads: tail_payloads + .into_iter() + .filter(|payload| !is_terminal_payload(payload)) + .collect(), + final_status, + receipt, + })) + } + + fn publish_task_outcome(&self, outcome: &FleetPreparedTaskOutcome) -> Result { Ok(self .ledger - .finalize_task_attempt_if_leased( - &task.entry.run_id, - &task.worker_id, - &task.entry.task_id, - task.entry.attempts, - ×tamp(), - payload, - final_status, - receipt, + .finalize_task_attempt_with_tail_if_leased( + &outcome.receipt.run_id, + &outcome.receipt.worker_id, + &outcome.receipt.task_id, + outcome + .receipt + .attempt + .ok_or_else(|| anyhow!("Fleet terminal candidate has no attempt"))?, + &outcome.receipt.completed_at, + outcome.payload.clone(), + outcome.tail_payloads.clone(), + outcome.final_status, + outcome.receipt.clone(), )? .is_some()) } - /// Mark a leased task timed out after its wall-clock limit (R5). The - /// caller stops and forgets the worker before this runs; the ledger's - /// implicit rule maps a Timeout receipt onto the Failed terminal status. - fn record_task_timeout( + fn record_task_outcome( + &self, + task: &FleetExecutorTaskContext, + terminal: FleetWorkerTerminalEvent, + ) -> Result { + match self.prepare_task_outcome(task, terminal)? { + Some(outcome) => self.publish_task_outcome(&outcome), + None => Ok(false), + } + } + + fn prepare_task_timeout( &self, task: &FleetExecutorTaskContext, limit: std::time::Duration, - ) -> Result { + ) -> Result> { let state = self.ledger.rebuild_state()?; let key = task_key(&task.entry.run_id.0, &task.entry.task_id); let Some(current) = state.tasks.get(&key) else { - return Ok(false); + return Ok(None); }; if current.status != FleetTaskLedgerStatus::Leased || current.leased_to.as_deref() != Some(task.worker_id.as_str()) || current.entry.attempts != task.entry.attempts { - return Ok(false); + return Ok(None); } let artifacts = self.task_artifacts_for_receipt( &task.entry.run_id, @@ -2045,19 +2232,23 @@ impl FleetManager { let payload = FleetWorkerEventPayload::Cancelled { cancelled_by: Some(format!("driver-timeout-{}-seconds", limit.as_secs())), }; - Ok(self - .ledger - .finalize_task_attempt_if_leased( - &task.entry.run_id, - &task.worker_id, - &task.entry.task_id, - task.entry.attempts, - ×tamp(), - payload, - Some(FleetTaskLedgerStatus::Failed), - receipt, - )? - .is_some()) + Ok(Some(FleetPreparedTaskOutcome { + payload, + tail_payloads: Vec::new(), + final_status: Some(FleetTaskLedgerStatus::Failed), + receipt, + })) + } + + fn record_task_timeout( + &self, + task: &FleetExecutorTaskContext, + limit: Duration, + ) -> Result { + match self.prepare_task_timeout(task, limit)? { + Some(outcome) => self.publish_task_outcome(&outcome), + None => Ok(false), + } } /// Resolve the route snapshot to persist on a task's receipt (#3154). @@ -3628,19 +3819,20 @@ mod tests { binary: &Path, ) -> FleetStatusSnapshot { let rt = tokio::runtime::Runtime::new().unwrap(); - let mut executor = FleetExecutor::new(&manager.workspace); + let executor = FleetExecutor::new(&manager.workspace); rt.block_on(async { manager .run_to_completion( run_id, max_workers, - &mut executor, + executor, &binary.display().to_string(), None, Duration::from_millis(10), ) .await .unwrap() + .0 }) } @@ -4207,17 +4399,17 @@ sleep 30 "#, ); let report = manager.create_run_from_task_spec_path(&path, 1).unwrap(); - let mut executor = FleetExecutor::new(&manager.workspace); + let executor = FleetExecutor::new(&manager.workspace); let rt = tokio::runtime::Runtime::new().unwrap(); - let status = rt + let (status, executor) = rt .block_on(async { tokio::time::timeout( Duration::from_secs(10), manager.run_to_completion( &report.run_id, 1, - &mut executor, + executor, &fake.display().to_string(), None, Duration::from_millis(20), @@ -4376,10 +4568,10 @@ sleep 30 ); let report = manager.create_run_from_task_spec_path(&path, 1).unwrap(); let worker_id = report.worker_ids[0].clone(); - let mut executor = FleetExecutor::new(&manager.workspace); + let executor = FleetExecutor::new(&manager.workspace); let rt = tokio::runtime::Runtime::new().unwrap(); - let (status, interrupted) = rt.block_on(async { + let ((status, executor), interrupted) = rt.block_on(async { tokio::time::timeout(Duration::from_secs(15), async { tokio::join!( async { @@ -4387,7 +4579,7 @@ sleep 30 .run_to_completion( &report.run_id, 1, - &mut executor, + executor, &fake.display().to_string(), None, Duration::from_millis(10), @@ -4467,10 +4659,10 @@ while :; do sleep 1; done ); let report = manager.create_run_from_task_spec_path(&path, 1).unwrap(); let worker_id = report.worker_ids[0].clone(); - let mut executor = FleetExecutor::new(&manager.workspace); + let executor = FleetExecutor::new(&manager.workspace); let rt = tokio::runtime::Runtime::new().unwrap(); - let status = rt.block_on(async { + let (status, executor) = rt.block_on(async { tokio::time::timeout(Duration::from_secs(15), async { tokio::join!( async { @@ -4478,7 +4670,7 @@ while :; do sleep 1; done .run_to_completion( &report.run_id, 1, - &mut executor, + executor, &fake.display().to_string(), None, Duration::from_millis(10), @@ -4494,7 +4686,38 @@ while :; do sleep 1; done }) .await .expect("first Fleet attempt never started"); - controller.restart_worker(&worker_id).unwrap(); + loop { + let restarting = controller.clone(); + let target = worker_id.clone(); + let (before, result, after) = + codewhale_runtime::runtime_control::owner_work(move || { + let before = restarting.rebuild_state()?; + let result = restarting.restart_worker(&target); + let after = restarting.rebuild_state()?; + Ok((before, result, after)) + }) + .await + .unwrap(); + match result { + Ok(_) => break, + Err(error) => { + assert!( + error.downcast_ref::().is_some(), + "{error:#}" + ); + let before = active_task_for_worker(&before, &worker_id) + .expect("original attempt remains active"); + let after = active_task_for_worker(&after, &worker_id) + .expect("original attempt remains active"); + assert_eq!(before.entry.run_id, after.entry.run_id); + assert_eq!(before.entry.task_id, after.entry.task_id); + assert_eq!(before.entry.attempts, 1); + assert_eq!(after.entry.attempts, 1); + assert_eq!(before.leased_to, after.leased_to); + tokio::task::yield_now().await; + } + } + } } ) .0 @@ -4575,7 +4798,7 @@ exit 0 manager.interrupt_worker(worker_id).unwrap(); let restart = manager.restart_worker(worker_id).unwrap(); - let mut executor = FleetExecutor::new(&manager.workspace); + let executor = FleetExecutor::new(&manager.workspace); let rt = tokio::runtime::Runtime::new().unwrap(); let status = rt .block_on(async { @@ -4584,7 +4807,7 @@ exit 0 manager.run_to_completion( &restart.run_id, restart.max_workers, - &mut executor, + executor, &fake.display().to_string(), None, Duration::from_millis(10), @@ -4593,7 +4816,8 @@ exit 0 .await }) .expect("standalone restart left a ghost running task") - .unwrap(); + .unwrap() + .0; assert!( marker.is_file(), @@ -4931,8 +5155,8 @@ exit 0 ), ); let report = manager.create_run_from_task_spec_path(&path, 1).unwrap(); - let mut primary_executor = FleetExecutor::new(&manager.workspace); - let mut standby_executor = FleetExecutor::new(&manager.workspace); + let primary_executor = FleetExecutor::new(&manager.workspace); + let standby_executor = FleetExecutor::new(&manager.workspace); let binary = fake.display().to_string(); let rt = tokio::runtime::Runtime::new().unwrap(); @@ -4946,7 +5170,7 @@ exit 0 manager.run_to_completion( &report.run_id, 1, - &mut primary_executor, + primary_executor, &binary, None, Duration::from_millis(10), @@ -4954,7 +5178,7 @@ exit 0 standby.run_to_completion( &report.run_id, 1, - &mut standby_executor, + standby_executor, &binary, None, Duration::from_millis(10), @@ -4965,16 +5189,14 @@ exit 0 }) .unwrap_or_else(|error| { panic!( - "competing Fleet managers did not converge: {error}; status={:?}; primary={:?}; standby={:?}; starts={:?}", + "competing Fleet managers did not converge: {error}; status={:?}; starts={:?}", manager.run_status(&report.run_id), - primary_executor.worker_ids(), - standby_executor.worker_ids(), std::fs::read_to_string(&starts), ) }); - assert_eq!(primary_status.unwrap().completed, 1); - assert_eq!(standby_status.unwrap().completed, 1); + assert_eq!(primary_status.unwrap().0.completed, 1); + assert_eq!(standby_status.unwrap().0.completed, 1); let starts = std::fs::read_to_string(starts).unwrap(); assert_eq!( starts.lines().count(), diff --git a/crates/engine/src/tools/subagent/completion_usage_tests.rs b/crates/engine/src/tools/subagent/completion_usage_tests.rs index 89b5438ae..349e544cd 100644 --- a/crates/engine/src/tools/subagent/completion_usage_tests.rs +++ b/crates/engine/src/tools/subagent/completion_usage_tests.rs @@ -94,7 +94,11 @@ async fn completion_usage_live_terminal_counts_grandchildren_and_continuations_o result.status = status; // The locked ledger wins over an earlier result snapshot. result.usage.as_mut().unwrap().total_tokens = Some(123_456); - assert!(manager.finish_terminal_result(&root, result, false, false)); + assert!( + manager + .finish_terminal_result(&root, result, false, false) + .expect("durable worker operation") + ); let completion = rx.try_recv().unwrap(); let payload = sentinel(&completion); assert_eq!(payload["usage"]["own"]["total_tokens"], 10); @@ -138,7 +142,11 @@ async fn completion_usage_recovery_restores_measured_lineage_without_recounting( let mut manager = SubAgentManager::new(base.clone(), 8).with_state_path(path.clone()); let (root, _, grandchild, continued) = family(&mut manager); let result = terminal_result(&manager, &root); - assert!(manager.finish_terminal_result(&root, result, false, false)); + assert!( + manager + .finish_terminal_result(&root, result, false, false) + .expect("durable worker operation") + ); let before = receipt(&manager, &root); manager.persist_state_synchronously().unwrap(); let mut loaded = SubAgentManager::new(base, 8).with_state_path(path); @@ -340,7 +348,11 @@ async fn completion_usage_counts_real_manifest_only_root_fork() { .unwrap(); let source = child(&mut guard, "source", None, Some(1)); let completed = terminal_result(&guard, &source); - assert!(guard.finish_terminal_result(&source, completed, true, false)); + assert!( + guard + .finish_terminal_result(&source, completed, true, false) + .expect("durable worker operation") + ); let fork = guard .spawn_background_with_assignment_options( Arc::clone(&manager), diff --git a/crates/engine/src/tools/subagent/coord.rs b/crates/engine/src/tools/subagent/coord.rs index 75c454aed..8a744e751 100644 --- a/crates/engine/src/tools/subagent/coord.rs +++ b/crates/engine/src/tools/subagent/coord.rs @@ -122,25 +122,27 @@ impl ToolSpec for AgentsListTool { .unwrap_or(false); let agent_ref = parse_agent_ref(&input)?; - let mut manager = self.manager.write().await; - manager.cleanup_for_session(&context.state_namespace, COMPLETED_AGENT_RETENTION); - let summaries = if let Some(agent_ref) = agent_ref { - let summary = manager - .coordination_summary_for_session( - &context.state_namespace, - &agent_ref, + let manager = self.manager.clone(); + let session = context.state_namespace.clone(); + let summaries = codewhale_runtime::runtime_control::owner_work(move || { + let mut manager = manager.blocking_write(); + manager.cleanup_for_session(&session, COMPLETED_AGENT_RETENTION)?; + let summaries = if let Some(agent_ref) = agent_ref { + manager + .coordination_summary_for_session(&session, &agent_ref, RECENT_PROGRESS_LIMIT) + .map(|summary| vec![summary]) + .map_err(|err| ToolError::invalid_input(err.to_string())) + } else { + Ok(manager.list_coordination_summaries_for_session( + &session, + include_archived, RECENT_PROGRESS_LIMIT, - ) - .map_err(|err| ToolError::invalid_input(err.to_string()))?; - vec![summary] - } else { - manager.list_coordination_summaries_for_session( - &context.state_namespace, - include_archived, - RECENT_PROGRESS_LIMIT, - ) - }; - drop(manager); + )) + }; + Ok(summaries) + }) + .await + .map_err(|error| ToolError::execution_failed(error.to_string()))??; let payload = json!({ "action": "list", @@ -316,66 +318,80 @@ impl AgentsFollowupTool { message: &str, context: &ToolContext, ) -> Result { - let mut manager = self.manager.write().await; - let (source, target) = manager - .continuation_target_for_caller( - &context.state_namespace, - agent_ref, - self.caller_agent_id.as_deref(), - "agents/followup", - ) - .map_err(|error| ToolError::invalid_input(error.to_string()))?; - let snapshot = manager - .get_result(&target) - .map_err(|error| ToolError::invalid_input(error.to_string()))?; - let resumed_already = source != target; - let receipt = if super::subagent_checkpoint_is_continuable(&snapshot) - && self.runtime.is_some() - { - let snapshot = manager - .resume_from_checkpoint_for_session( - &context.state_namespace, - Arc::clone(&self.manager), - self.runtime.clone().expect("runtime checked"), - &target, - message, - ) - .map_err(|error| ToolError::execution_failed(error.to_string()))?; - ParentMailReceipt { - agent_id: snapshot.agent_id.clone(), - status: subagent_status_name(&snapshot.status).to_string(), - queue_depth: 0, - woke: true, - continued_from_checkpoint: true, - continuation_handle: None, - note: format!( - "resumed from checkpoint {source} as {}; original receipt retained", - snapshot.agent_id - ), - } - } else if resumed_already && snapshot.status != SubAgentStatus::Running { - ParentMailReceipt { - agent_id: target, status: subagent_status_name(&snapshot.status).to_string(), - queue_depth: 0, woke: false, continued_from_checkpoint: true, - continuation_handle: None, - note: "Existing continuation has settled; no duplicate worker was started and no message was delivered.".to_string(), - } - } else { - manager - .followup_child_for_session(&context.state_namespace, &target, message.to_string()) - .map_err(|error| ToolError::invalid_input(error.to_string()))? - }; - let child_route = manager - .get_worker_record_for_session(&context.state_namespace, &receipt.agent_id) - .and_then(|record| record.spec.child_route); - Ok(json!({ - "action": "followup", "from": source, "to": receipt.agent_id, - "agent_id": receipt.agent_id, "queued": receipt.woke || receipt.queue_depth > 0, - "woke": receipt.woke, "queue_depth": receipt.queue_depth, "status": receipt.status, - "continued_from_checkpoint": receipt.continued_from_checkpoint || resumed_already, - "continuation_handle": receipt.continuation_handle, "note": receipt.note, - "child_route": child_route, - })) + let manager_handle = self.manager.clone(); + let manager = manager_handle.clone(); + let session = context.state_namespace.clone(); + let caller = self.caller_agent_id.clone(); + let runtime = self.runtime.clone(); + let agent_ref = agent_ref.to_owned(); + let message = message.to_owned(); + codewhale_runtime::runtime_control::owner_work(move || { + let mut manager = manager.blocking_write(); + let result = (|| { + let (source, target) = manager + .continuation_target_for_caller( + &session, + &agent_ref, + caller.as_deref(), + "agents/followup", + ) + .map_err(|error| ToolError::invalid_input(error.to_string()))?; + let snapshot = manager + .get_result(&target) + .map_err(|error| ToolError::invalid_input(error.to_string()))?; + let resumed_already = source != target; + let receipt = if super::subagent_checkpoint_is_continuable(&snapshot) + && runtime.is_some() + { + let snapshot = manager + .resume_from_checkpoint_for_session( + &session, + Arc::clone(&manager_handle), + runtime.expect("runtime checked"), + &target, + &message, + ) + .map_err(|error| ToolError::execution_failed(error.to_string()))?; + ParentMailReceipt { + agent_id: snapshot.agent_id.clone(), + status: subagent_status_name(&snapshot.status).to_string(), + queue_depth: 0, + woke: true, + continued_from_checkpoint: true, + continuation_handle: None, + note: format!( + "resumed from checkpoint {source} as {}; original receipt retained", + snapshot.agent_id + ), + } + } else if resumed_already && snapshot.status != SubAgentStatus::Running { + ParentMailReceipt { + agent_id: target, status: subagent_status_name(&snapshot.status).to_string(), + queue_depth: 0, woke: false, continued_from_checkpoint: true, + continuation_handle: None, + note: "Existing continuation has settled; no duplicate worker was started and no message was delivered.".to_string(), + } + } else { + manager + .followup_child_for_session(&session, &target, message) + .map_err(|error| ToolError::invalid_input(error.to_string()))? + }; + let child_route = manager + .get_worker_record_for_session(&session, &receipt.agent_id) + .and_then(|record| record.spec.child_route); + Ok(json!({ + "action": "followup", "from": source, "to": receipt.agent_id, + "agent_id": receipt.agent_id, "queued": receipt.woke || receipt.queue_depth > 0, + "woke": receipt.woke, "queue_depth": receipt.queue_depth, "status": receipt.status, + "continued_from_checkpoint": receipt.continued_from_checkpoint || resumed_already, + "continuation_handle": receipt.continuation_handle, "note": receipt.note, + "child_route": child_route, + })) + })(); + Ok(result) + }) + .await + .map_err(|error| ToolError::execution_failed(error.to_string()))? } } @@ -602,17 +618,19 @@ impl ToolSpec for AgentsInterruptTool { .unwrap_or("interrupted by parent via agents/interrupt") .to_string(); - let (prior, snapshot) = { - let mut manager = self.manager.write().await; - manager - .interrupt_child_for_session( - &context.state_namespace, - &agent_ref, - self.caller_agent_id.as_deref(), - reason, - ) - .map_err(|err| ToolError::invalid_input(err.to_string()))? - }; + let manager = self.manager.clone(); + let session = context.state_namespace.clone(); + let caller = self.caller_agent_id.clone(); + let (prior, snapshot) = codewhale_runtime::runtime_control::owner_work(move || { + manager.blocking_write().interrupt_child_for_session( + &session, + &agent_ref, + caller.as_deref(), + reason, + ) + }) + .await + .map_err(|error| ToolError::execution_failed(error.to_string()))?; let snapshot = super::settle_requested_child(&self.manager, snapshot).await; let worker_record = { @@ -1445,30 +1463,22 @@ impl ToolSpec for AgentsCoordinateTool { ))); } - let mut manager = self.manager.clone().write_owned().await; - if matches!(action, "claim" | "propose") { - let caller = self.caller.clone(); - let session = context.state_namespace.clone(); - let workspace = context.workspace.clone(); - return codewhale_runtime::runtime_control::owner_work(move || { - Ok(Self::mutate_coordination( - caller.as_deref(), - &session, - &workspace, - &mut manager, - input, - )) + let manager = self.manager.clone(); + let caller = self.caller.clone(); + let session = context.state_namespace.clone(); + let workspace = context.workspace.clone(); + codewhale_runtime::runtime_control::owner_work(move || { + let mut manager = manager.blocking_write(); + manager.with_store_transaction(|manager| { + Self::mutate_coordination(caller.as_deref(), &session, &workspace, manager, input) + .map_err(anyhow::Error::new) }) - .await - .map_err(|error| ToolError::execution_failed(error.to_string()))?; - } - Self::mutate_coordination( - self.caller.as_deref(), - &context.state_namespace, - &context.workspace, - &mut manager, - input, - ) + }) + .await + .map_err(|error| match error.downcast::() { + Ok(error) => error, + Err(error) => ToolError::execution_failed(error.to_string()), + }) } } @@ -1710,7 +1720,10 @@ mod tests { .await .expect_err("mutation must fail when its receipt cannot persist") .to_string(); - assert!(error.contains("failed to persist"), "{error}"); + assert!( + error.contains("Runtime coordination was opened without its writer lock"), + "{error}" + ); assert!( blocked_manager .read() @@ -1825,31 +1838,35 @@ mod tests { )); let (claimant, owner) = { let mut guard = manager.write().await; - let claimant = guard.insert_test_running_agent("claimant", tmp.path()); - let owner = guard.insert_test_running_agent("owner", tmp.path()); - let active = [claimant.clone(), owner.clone()] - .into_iter() - .collect::>(); - for claim in [ - WriteScopeClaim { - owner: claimant.clone(), - roots: vec!["src/claimant".into()], - exact_files: Vec::new(), - contracts: Vec::new(), - }, - WriteScopeClaim { - owner: owner.clone(), - roots: vec!["src/shared".into()], - exact_files: Vec::new(), - contracts: Vec::new(), - }, - ] { - guard - .coordination - .register_claim(claim, false, |candidate| active.contains(candidate)) - .expect("initial non-overlapping claim"); - } - (claimant, owner) + guard + .with_store_transaction(|guard| { + let claimant = guard.insert_test_running_agent("claimant", tmp.path()); + let owner = guard.insert_test_running_agent("owner", tmp.path()); + let active = [claimant.clone(), owner.clone()] + .into_iter() + .collect::>(); + for claim in [ + WriteScopeClaim { + owner: claimant.clone(), + roots: vec!["src/claimant".into()], + exact_files: Vec::new(), + contracts: Vec::new(), + }, + WriteScopeClaim { + owner: owner.clone(), + roots: vec!["src/shared".into()], + exact_files: Vec::new(), + contracts: Vec::new(), + }, + ] { + guard + .coordination + .register_claim(claim, false, |candidate| active.contains(candidate)) + .expect("initial non-overlapping claim"); + } + Ok((claimant, owner)) + }) + .unwrap() }; let error = AgentsCoordinateTool::new(Arc::clone(&manager), Some(claimant.clone())) @@ -1886,31 +1903,35 @@ mod tests { )); let claimant = { let mut guard = manager.write().await; - let claimant = guard.insert_test_running_agent("claimant", tmp.path()); - let owner = guard.insert_test_running_agent("owner", tmp.path()); - let active = [claimant.clone(), owner.clone()] - .into_iter() - .collect::>(); - for claim in [ - WriteScopeClaim { - owner: claimant.clone(), - roots: vec!["src/claimant".into()], - exact_files: Vec::new(), - contracts: Vec::new(), - }, - WriteScopeClaim { - owner: owner.clone(), - roots: vec!["src/shared".into()], - exact_files: Vec::new(), - contracts: Vec::new(), - }, - ] { - guard - .coordination - .register_claim(claim, false, |candidate| active.contains(candidate)) - .expect("initial non-overlapping claim"); - } - claimant + guard + .with_store_transaction(|guard| { + let claimant = guard.insert_test_running_agent("claimant", tmp.path()); + let owner = guard.insert_test_running_agent("owner", tmp.path()); + let active = [claimant.clone(), owner.clone()] + .into_iter() + .collect::>(); + for claim in [ + WriteScopeClaim { + owner: claimant.clone(), + roots: vec!["src/claimant".into()], + exact_files: Vec::new(), + contracts: Vec::new(), + }, + WriteScopeClaim { + owner: owner.clone(), + roots: vec!["src/shared".into()], + exact_files: Vec::new(), + contracts: Vec::new(), + }, + ] { + guard + .coordination + .register_claim(claim, false, |candidate| active.contains(candidate)) + .expect("initial non-overlapping claim"); + } + Ok(claimant) + }) + .unwrap() }; AgentsCoordinateTool::new(Arc::clone(&manager), Some(claimant.clone())) @@ -1923,23 +1944,27 @@ mod tests { let resolution_sequence = { let mut guard = manager.write().await; - let record = guard - .coordination - .register_claim( - WriteScopeClaim { - owner: claimant.clone(), - roots: vec!["src/isolated".into()], - exact_files: Vec::new(), - contracts: Vec::new(), - }, - true, - |_| true, - ) - .expect("later isolated claim resolves contention"); guard - .persist_state_synchronously() - .expect("persist resolved contention"); - record.sequence + .with_store_transaction(|guard| { + let record = guard + .coordination + .register_claim( + WriteScopeClaim { + owner: claimant.clone(), + roots: vec!["src/isolated".into()], + exact_files: Vec::new(), + contracts: Vec::new(), + }, + true, + |_| true, + ) + .expect("later isolated claim resolves contention"); + guard + .persist_state_synchronously() + .expect("persist resolved contention"); + Ok(record.sequence) + }) + .unwrap() }; let mut replayed = super::super::SubAgentManager::new(tmp.path().to_path_buf(), 4) @@ -1955,48 +1980,50 @@ mod tests { Some(resolution_sequence) ); - let slots = COORDINATION_RECORD_LIMIT - replayed.coordination.write_claims.len(); - for index in 0..slots { - replayed - .coordination - .register_claim( - WriteScopeClaim { - owner: format!("inactive-fill-{index:03}"), - roots: vec![format!("pkg/fill-{index:03}")], - exact_files: Vec::new(), - contracts: Vec::new(), - }, - true, - |_| false, - ) - .expect("fill inactive claim capacity"); - } - for index in 0..2 { - replayed - .coordination - .register_claim( - WriteScopeClaim { - owner: format!("inactive-overflow-{index}"), - roots: vec![format!("pkg/overflow-{index}")], - exact_files: Vec::new(), - contracts: Vec::new(), - }, - true, - |_| false, - ) - .expect("evict oldest inactive claim at capacity"); - } - assert!( - !replayed - .coordination - .write_claims - .iter() - .any(|claim| claim.claim.owner == claimant), - "the resolving claimant claim must be evicted for the durability regression" - ); replayed - .persist_state_synchronously() - .expect("persist after inactive claim eviction"); + .with_store_transaction(|held| { + let slots = COORDINATION_RECORD_LIMIT - held.coordination.write_claims.len(); + for index in 0..slots { + held.coordination + .register_claim( + WriteScopeClaim { + owner: format!("inactive-fill-{index:03}"), + roots: vec![format!("pkg/fill-{index:03}")], + exact_files: Vec::new(), + contracts: Vec::new(), + }, + true, + |_| false, + ) + .expect("fill inactive claim capacity"); + } + for index in 0..2 { + held.coordination + .register_claim( + WriteScopeClaim { + owner: format!("inactive-overflow-{index}"), + roots: vec![format!("pkg/overflow-{index}")], + exact_files: Vec::new(), + contracts: Vec::new(), + }, + true, + |_| false, + ) + .expect("evict oldest inactive claim at capacity"); + } + assert!( + !held + .coordination + .write_claims + .iter() + .any(|claim| claim.claim.owner == claimant), + "the resolving claimant claim must be evicted for the durability regression" + ); + held.persist_state_synchronously() + .expect("persist after inactive claim eviction"); + Ok(()) + }) + .unwrap(); let mut final_replay = super::super::SubAgentManager::new(tmp.path().to_path_buf(), 4) .with_state_path(state_path); diff --git a/crates/engine/src/tools/subagent/coord/ledger.rs b/crates/engine/src/tools/subagent/coord/ledger.rs index 01dbfe112..af4611d50 100644 --- a/crates/engine/src/tools/subagent/coord/ledger.rs +++ b/crates/engine/src/tools/subagent/coord/ledger.rs @@ -388,7 +388,7 @@ fn projected_claim(claim: &WriteScopeClaim, root: &Path) -> WriteScopeClaim { } /// Durable, bounded coordination state owned by `SubAgentManager`. -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] pub struct CoordinationLedger { #[serde(default = "coordination_schema_version")] pub schema_version: u32, diff --git a/crates/engine/src/tools/subagent/lifecycle_tests.rs b/crates/engine/src/tools/subagent/lifecycle_tests.rs index 607c539f3..b1ae0f0c6 100644 --- a/crates/engine/src/tools/subagent/lifecycle_tests.rs +++ b/crates/engine/src/tools/subagent/lifecycle_tests.rs @@ -171,8 +171,7 @@ async fn lifecycle_resume_lineage_survives_persist_and_rejects_cycles_and_foreig manager.insert_test_interrupted_continuable_agent("latest", &base, prior_messages()); manager.resume_targets.insert(a.clone(), b.clone()); manager.resume_targets.insert(b.clone(), c.clone()); - let (path, payload) = manager.build_persist_payload().unwrap().unwrap(); - write_json_atomic(&base, &path, &payload).unwrap(); + manager.persist_state().unwrap().join().unwrap(); let mut loaded = SubAgentManager::new(base.clone(), 6).with_state_path(state_path); loaded.load_state().unwrap(); assert_eq!(loaded.continuation_target(&a).unwrap(), c); @@ -614,8 +613,9 @@ async fn lifecycle_continuation_link_is_durable_before_the_child_can_run() { // Holding the manager lock keeps run_subagent_task_inner at its first // await. This is the earliest published snapshot, before any child step. assert_eq!(calls.load(std::sync::atomic::Ordering::SeqCst), 0); - let persisted: PersistedSubAgentState = + let persisted: DurableSubAgentStore = serde_json::from_slice(&std::fs::read(path).unwrap()).unwrap(); + let persisted = persisted.state; assert_eq!( persisted.resume_targets.get(&source), Some(&successor.agent_id) @@ -635,7 +635,7 @@ async fn lifecycle_continuation_persist_failure_rolls_back_worker_and_link() { let base = dir.path().canonicalize().unwrap(); let path = base.join(".codewhale/subagents/state.json"); let manager = Arc::new(RwLock::new( - SubAgentManager::new(base.clone(), 4).with_state_path(path), + SubAgentManager::new(base.clone(), 4).with_state_path(path.clone()), )); let mut runtime = super::tests::stub_runtime(); runtime.manager = Arc::clone(&manager); @@ -657,9 +657,17 @@ async fn lifecycle_continuation_persist_failure_rolls_back_worker_and_link() { .permissions .write ); - std::fs::create_dir_all(base.join(".codewhale")).unwrap(); - std::fs::write(base.join(".codewhale/subagents"), "not a directory").unwrap(); - let result = guard.resume_from_checkpoint(Arc::clone(&manager), runtime, &source, "Continue"); + guard.persist_state_synchronously().unwrap(); + let retained_path = path.with_extension("retained-before-failure"); + let result = guard.with_store_transaction(|held| { + std::fs::rename(&path, &retained_path)?; + std::fs::create_dir(&path)?; + let resumed = + held.resume_from_checkpoint(Arc::clone(&manager), runtime, &source, "Continue"); + std::fs::remove_dir(&path)?; + std::fs::rename(&retained_path, &path)?; + resumed + }); assert!(result.is_err()); assert_eq!(guard.agents.len(), 1); assert_eq!(guard.worker_records.len(), 1); @@ -824,7 +832,9 @@ async fn lifecycle_cleanup_keeps_source_identity_while_descendants_still_need_it record.status = AgentWorkerStatus::Completed; record.completed_at_ms = Some(epoch_millis_now().saturating_sub(120_000)); - manager.cleanup_for_session("workspace", Duration::from_secs(60)); + manager + .cleanup_for_session("workspace", Duration::from_secs(60)) + .expect("durable worker operation"); assert_eq!(manager.continuation_target(&source).unwrap(), current); assert!( manager @@ -847,7 +857,9 @@ async fn lifecycle_cleanup_keeps_source_identity_while_descendants_still_need_it for record in manager.worker_records.values_mut() { record.completed_at_ms = Some(epoch_millis_now().saturating_sub(120_000)); } - manager.cleanup_for_session("workspace", Duration::from_secs(60)); + manager + .cleanup_for_session("workspace", Duration::from_secs(60)) + .expect("durable worker operation"); assert!(manager.agents.is_empty()); assert!(manager.worker_records.is_empty()); assert!(manager.resume_targets.is_empty()); diff --git a/crates/engine/src/tools/subagent/mod.rs b/crates/engine/src/tools/subagent/mod.rs index e737a7cb3..7b37c5d90 100644 --- a/crates/engine/src/tools/subagent/mod.rs +++ b/crates/engine/src/tools/subagent/mod.rs @@ -23,7 +23,7 @@ use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; use tokio::sync::Mutex; use tokio::sync::RwLock; -use anyhow::{Result, anyhow}; +use anyhow::{Result, anyhow, bail}; use async_trait::async_trait; use serde::{Deserialize, Serialize}; use serde_json::{Value, json}; @@ -504,7 +504,8 @@ const SUBAGENT_TRANSCRIPT_MESSAGE_BUDGET_BYTES: usize = 1024 * 1024; const SUBAGENT_TRANSCRIPT_ARTIFACT_SCHEMA_VERSION: u32 = 2; const SUBAGENT_TRANSCRIPT_ARTIFACT_DIR: &str = "subagent-transcripts"; const SUBAGENT_STATE_SCHEMA_VERSION: u32 = 1; -const SUBAGENT_STATE_FILE: &str = "subagents.v1.json"; +const SUBAGENT_STATE_FILE: &str = "subagents.v3.json"; +const LEGACY_SUBAGENT_STATE_FILE: &str = "subagents.v1.json"; const SUBAGENT_STATE_LOCK_FILE: &str = "subagents.v1.lock"; const SUBAGENT_RESTART_REASON: &str = "Interrupted by process restart"; const SUBAGENT_SESSION_CLOSED_REASON: &str = "Interrupted: parent session closed"; @@ -1020,6 +1021,14 @@ pub struct AgentWorkerRecord { #[serde(default, skip_deserializing, skip_serializing_if = "Option::is_none")] pub pending_request: Option, pub status: AgentWorkerStatus, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub prepared_generation: Option, + #[serde(default)] + pub prepared_previous_attempt_unsettled: bool, + #[serde(default)] + pub launch_admission_version: u32, + #[serde(default, skip_serializing_if = "is_false")] + pub recovery_interrupted: bool, pub created_at_ms: u64, pub updated_at_ms: u64, #[serde(default, skip_serializing_if = "Option::is_none")] @@ -1046,6 +1055,8 @@ pub struct AgentWorkerRecord { #[derive(Clone)] pub struct CoordinationRegistrationSnapshot { + store_revision: u64, + writer_sequence: u64, worker_records: HashMap, coordination: CoordinationLedger, worker_origins: HashMap, @@ -1113,6 +1124,10 @@ impl AgentWorkerRecord { recommended_action, pending_request: None, status: AgentWorkerStatus::Starting, + prepared_generation: None, + prepared_previous_attempt_unsettled: false, + launch_admission_version: 1, + recovery_interrupted: false, created_at_ms: now_ms, updated_at_ms: now_ms, started_at_ms: None, @@ -1125,6 +1140,56 @@ impl AgentWorkerRecord { events: VecDeque::new(), } } + fn prepared_attempt(&self, spec: AgentWorkerSpec, sequence: &mut u64, now_ms: u64) -> Self { + let generation = spec + .launch_manifest + .as_ref() + .expect("validated launch manifest") + .generation; + let mut record = self.clone(); + let settled = matches!( + self.status, + AgentWorkerStatus::Completed | AgentWorkerStatus::Failed | AgentWorkerStatus::Cancelled + ) && self.launch_admission_version == 1 + && !self.recovery_interrupted; + let unstarted = self.started_at_ms.is_none() + && (self.prepared_generation.is_some() + || self.launch_admission_version == 1 + && (self.status == AgentWorkerStatus::Starting || self.recovery_interrupted) + && self + .spec + .launch_manifest + .as_ref() + .is_some_and(|manifest| manifest.generation == 1)); + record.prepared_previous_attempt_unsettled = + self.prepared_previous_attempt_unsettled || !(settled || unstarted); + record.spec = spec; + record.prepared_generation = Some(generation); + record.recovery_interrupted = false; + record.status = AgentWorkerStatus::Starting; + record.started_at_ms = None; + record.completed_at_ms = None; + record.result_summary = None; + record.error = None; + record.pending_request = None; + record.steps_taken = 0; + record.parked_at_turn_end = false; + record.recommended_action = + recommended_action_for_worker_status(record.status, &record.spec); + record.updated_at_ms = now_ms; + let message = format!("Fleet launch generation {generation} prepared"); + record.latest_message = Some(message.clone()); + SubAgentManager::push_worker_event( + sequence, + &mut record, + AgentWorkerStatus::Starting, + Some(message), + Some(0), + None, + now_ms, + ); + record + } } fn default_subagent_actor_kind() -> String { @@ -2336,6 +2401,35 @@ impl Default for PersistedSubAgentState { impl PersistedSubAgentState { fn validate_worker_origins(&self) -> Result<(), String> { + for worker in &self.workers { + if worker.launch_admission_version > 1 { + return Err("unsupported Fleet launch admission protocol".into()); + } + if worker.prepared_previous_attempt_unsettled && worker.prepared_generation.is_none() { + return Err("unsettled previous Fleet attempt has no prepared generation".into()); + } + if worker.recovery_interrupted && worker.status != AgentWorkerStatus::Interrupted { + return Err("invalid recovered worker lifecycle".into()); + } + if let Some(generation) = worker.prepared_generation { + if generation == 0 + || !worker + .spec + .launch_manifest + .as_ref() + .is_some_and(|manifest| manifest.generation == generation) + || worker.status != AgentWorkerStatus::Starting + || worker.started_at_ms.is_some() + || worker.completed_at_ms.is_some() + || self + .agents + .iter() + .any(|agent| agent.id == worker.spec.worker_id) + { + return Err("invalid prepared Fleet launch generation".into()); + } + } + } match (self.schema_version, self.worker_origins.as_ref()) { (1, None) => Ok(()), (2, Some(origins)) @@ -3696,6 +3790,13 @@ fn owner_state_from_worker_status(status: AgentWorkerStatus) -> OwnerState { } } +#[derive(Clone)] +struct PendingTerminalResult { + result: SubAgentResult, + worker: Option<(AgentWorkerRecord, u64)>, + prepared: bool, +} + /// A running sub-agent instance. pub struct SubAgent { pub id: String, @@ -3732,6 +3833,7 @@ pub struct SubAgent { /// queued (#1961). Competing cancellation/interrupt paths must treat the /// claim as terminal ownership and leave the task to finalize. completion_claimed: bool, + pending_terminal_result: Option, /// Process-local terminal fan-in sinks. Never serialized; restored agents /// have no live parent/mailbox/event consumers and are reconciled directly /// to interrupted state during load. @@ -3790,6 +3892,7 @@ impl SubAgent { owner_session_id: String::new(), workspace, completion_claimed: false, + pending_terminal_result: None, terminal_delivery: None, work_lifecycle: None, input_tx: Some(input_tx), @@ -3999,6 +4102,8 @@ pub struct SubAgentManager { worker_records: HashMap, worker_event_seq: u64, persist_sequence: std::sync::atomic::AtomicU64, + store_authority: Option>, + foreign_live_owners: HashSet, coordination: CoordinationLedger, admitted_coordination_roots: HashMap, worker_origins: HashMap, @@ -4515,6 +4620,8 @@ impl SubAgentManager { worker_records: HashMap::new(), worker_event_seq: 0, persist_sequence: std::sync::atomic::AtomicU64::new(0), + store_authority: None, + foreign_live_owners: HashSet::new(), coordination: CoordinationLedger::default(), admitted_coordination_roots: HashMap::new(), worker_origins: HashMap::new(), @@ -5062,6 +5169,21 @@ impl SubAgentManager { mut decision: DecisionRecord, workspace: &Path, ) -> Result { + if self.state_path.is_some() + && !self + .store_authority + .as_ref() + .is_some_and(|authority| authority.is_current_transaction()) + { + return self + .with_store_transaction(|manager| { + manager + .record_coordination_decision_in_workspace(decision, workspace) + .map_err(anyhow::Error::msg) + }) + .map_err(|error| error.to_string()); + } + self.ensure_coordination_process_lock()?; if let Some(root) = self.admitted_coordination_root(workspace) { root.validate()?; @@ -5126,6 +5248,21 @@ impl SubAgentManager { owner: &str, expected_version: u32, ) -> Result { + if self.state_path.is_some() + && !self + .store_authority + .as_ref() + .is_some_and(|authority| authority.is_current_transaction()) + { + return self + .with_store_transaction(|manager| { + manager + .update_coordination_decision(decision_id, status, owner, expected_version) + .map_err(anyhow::Error::msg) + }) + .map_err(|error| error.to_string()); + } + self.ensure_coordination_process_lock()?; self.coordination .update_decision_status(decision_id, status, owner, expected_version) @@ -5146,6 +5283,33 @@ impl SubAgentManager { verifier_evidence_handles: Vec, verification_outcome: String, ) -> Result { + if self.state_path.is_some() + && !self + .store_authority + .as_ref() + .is_some_and(|authority| authority.is_current_transaction()) + { + return self + .with_store_transaction(|manager| { + manager + .reconcile_coordination( + subject, + owner, + input_decisions, + outcome, + evidence_handles, + candidate_handles, + retry_count, + retry_limit, + reviewer_evidence_handles, + verifier_evidence_handles, + verification_outcome, + ) + .map_err(anyhow::Error::msg) + }) + .map_err(|error| error.to_string()); + } + self.ensure_coordination_process_lock()?; let expected_owner = self.nearest_common_fan_in_owner(&input_decisions)?; if owner != expected_owner { @@ -5549,6 +5713,21 @@ impl SubAgentManager { exact_files: Vec, contracts: Vec, ) -> Result { + if self.state_path.is_some() + && !self + .store_authority + .as_ref() + .is_some_and(|authority| authority.is_current_transaction()) + { + return self + .with_store_transaction(|manager| { + manager + .expand_write_claim(owner, roots, exact_files, contracts) + .map_err(anyhow::Error::msg) + }) + .map_err(|error| error.to_string()); + } + self.ensure_coordination_process_lock()?; let Some(existing) = self .coordination @@ -5621,6 +5800,21 @@ impl SubAgentManager { &mut self, owner: Option, ) -> Result, String> { + if self.state_path.is_some() + && !self + .store_authority + .as_ref() + .is_some_and(|authority| authority.is_current_transaction()) + { + return self + .with_store_transaction(|manager| { + manager + .release_stale_write_claims(owner) + .map_err(anyhow::Error::msg) + }) + .map_err(|error| error.to_string()); + } + self.ensure_coordination_process_lock()?; let active = self.active_coordination_owners(); let released = self @@ -5742,12 +5936,35 @@ impl SubAgentManager { #[must_use] fn with_state_path(mut self, path: PathBuf) -> Self { - self.state_path = Some(path); + self.state_path = Some(path.clone()); + match SubAgentStoreAuthority::open(&self.state_root, &path, &self.current_session_boot_id) { + Ok(authority) => self.store_authority = Some(authority), + Err(error) => { + self.coordination_snapshot_read_only = true; + tracing::warn!(target: "subagent", %error, "delegated store admission failed; writable state remains refused"); + } + } self } fn require_coordination_process_lock(mut self) -> Self { self.coordination_process_lock_required = true; + if self.store_authority.is_none() + && let Some(path) = self.state_path.as_ref() + { + match SubAgentStoreAuthority::open( + &self.state_root, + path, + &self.current_session_boot_id, + ) { + Ok(authority) => self.store_authority = Some(authority), + Err(error) => { + self.coordination_snapshot_read_only = true; + tracing::warn!(target: "subagent", %error, "delegated store migration or admission failed; no durable writes are admitted"); + return self; + } + } + } *self .coordination_process_lock .get_mut() @@ -5848,6 +6065,250 @@ impl SubAgentManager { true } + pub(crate) fn owns_store_record(&self, id: &str) -> bool { + self.state_path.is_none() + || self + .store_authority + .as_ref() + .is_some_and(|authority| authority.owned.lock().contains(id)) + } + + pub(crate) fn with_store_transaction( + &mut self, + operation: impl FnOnce(&mut Self) -> Result, + ) -> Result { + if self.state_path.is_none() { + return operation(self); + } + self.ensure_coordination_process_lock() + .map_err(anyhow::Error::msg)?; + let authority = self.store_authority.clone().ok_or_else(|| { + anyhow!("delegated durable store admission failed; inspect before retrying") + })?; + if authority.is_current_transaction() { + return operation(self); + } + let lease = authority.begin()?; + let image = authority.current_image()?; + self.coordination = image.state.coordination.clone(); + self.foreign_live_owners = + live_foreign_store_owners(&image, &authority.path, &authority.writer); + let owned = authority.owned.lock().clone(); + let agent_ids = image + .state + .agents + .iter() + .map(|agent| agent.id.clone()) + .collect::>(); + let worker_ids = image + .state + .workers + .iter() + .map(|record| record.spec.worker_id.clone()) + .collect::>(); + self.agents + .retain(|id, _| owned.contains(id) || agent_ids.contains(id)); + self.worker_records + .retain(|id, _| owned.contains(id) || worker_ids.contains(id)); + self.worker_origins + .retain(|id, _| owned.contains(id) || worker_ids.contains(id)); + let mut adopted = Vec::new(); + for persisted in &image.state.agents { + if !owned.contains(&persisted.id) { + let agent = self.restore_persisted_agent(persisted.clone()); + self.agents.insert(agent.id.clone(), agent); + if !worker_ids.contains(&persisted.id) + && !image + .record_writers + .get(&persisted.id) + .is_some_and(|writer| authority.writer_is_live(writer)) + { + authority.owned.lock().insert(persisted.id.clone()); + adopted.push(persisted.id.clone()); + } + } + } + for record in &image.state.workers { + let refreshed_generation = owned.contains(&record.spec.worker_id) + && !image + .state + .agents + .iter() + .any(|agent| agent.id == record.spec.worker_id) + && self + .worker_records + .get(&record.spec.worker_id) + .is_some_and(|local| { + record + .spec + .launch_manifest + .as_ref() + .map_or(0, |manifest| manifest.generation) + > local + .spec + .launch_manifest + .as_ref() + .map_or(0, |manifest| manifest.generation) + }); + if refreshed_generation { + self.worker_records.insert( + record.spec.worker_id.clone(), + normalize_worker_record(record.clone()), + ); + } + if !owned.contains(&record.spec.worker_id) { + self.worker_records.insert( + record.spec.worker_id.clone(), + normalize_worker_record(record.clone()), + ); + let writer_live = image + .record_writers + .get(&record.spec.worker_id) + .is_some_and(|writer| authority.writer_is_live(writer)); + if !writer_live { + authority.owned.lock().insert(record.spec.worker_id.clone()); + adopted.push(record.spec.worker_id.clone()); + } + let origin = image + .state + .worker_origins + .as_ref() + .and_then(|origins| origins.get(&record.spec.worker_id)) + .cloned() + .unwrap_or_else(|| WorkerOrigin::legacy(record.owner_session_id.clone())); + self.worker_origins + .insert(record.spec.worker_id.clone(), origin); + } + self.worker_event_seq = self + .worker_event_seq + .max(record.events.back().map_or(0, |event| event.seq)); + } + for worker_id in &adopted { + if let Some(record) = self.worker_records.get(worker_id) + && record.prepared_generation.is_none() + && matches!( + record.status, + AgentWorkerStatus::Queued + | AgentWorkerStatus::Starting + | AgentWorkerStatus::Running + | AgentWorkerStatus::ModelWait + | AgentWorkerStatus::RunningTool + ) + { + let steps_taken = record.steps_taken; + self.record_worker_event( + worker_id, + AgentWorkerStatus::Interrupted, + Some(SUBAGENT_RESTART_REASON.to_string()), + Some(steps_taken), + None, + ); + self.worker_records + .get_mut(worker_id) + .expect("recovered worker exists") + .recovery_interrupted = true; + } + } + if image.state.schema_version == 2 { + self.worker_origin_schema = 2; + } + self.resume_targets = image.state.resume_targets.clone(); + self.persist_sequence.fetch_max( + image + .writer_sequences + .get(&authority.writer) + .copied() + .unwrap_or(0), + std::sync::atomic::Ordering::AcqRel, + ); + let adoption = if adopted.is_empty() { + Ok(()) + } else { + self.persist_state_synchronously() + }; + let mut result = match adoption { + Ok(()) => operation(self), + Err(error) => Err(error), + }; + if result.is_ok() + && authority.transaction.lock().as_ref().is_some_and(|tx| { + tx.publishes == 0 && self.coordination != tx.image.state.coordination + }) + { + result = match self.persist_state_synchronously() { + Ok(()) => result, + Err(error) => Err(error), + }; + } + if result.is_err() { + let committed = read_durable_subagent_store(&authority.state_root, &authority.path) + .or_else(|_| authority.current_image())?; + *authority.owned.lock() = committed + .record_writers + .iter() + .filter(|(_, writer)| *writer == &authority.writer) + .map(|(id, _)| id.clone()) + .collect(); + self.foreign_live_owners = + live_foreign_store_owners(&committed, &authority.path, &authority.writer); + self.coordination = committed.state.coordination; + let origins = committed.state.worker_origins.unwrap_or_else(|| { + committed + .state + .workers + .iter() + .map(|record| { + ( + record.spec.worker_id.clone(), + WorkerOrigin::legacy(record.owner_session_id.clone()), + ) + }) + .collect() + }); + self.worker_records = committed + .state + .workers + .into_iter() + .map(normalize_worker_record) + .map(|record| (record.spec.worker_id.clone(), record)) + .collect(); + self.worker_origins = origins; + self.worker_origin_schema = committed.state.schema_version; + self.resume_targets = committed.state.resume_targets; + } + drop(lease); + result + } + + fn persist_terminal_candidate( + &self, + worker_id: &str, + payload: PersistedSubAgentState, + ) -> Result> { + self.ensure_coordination_process_lock() + .map_err(anyhow::Error::msg)?; + if self.state_path.is_none() { + return Ok(None); + } + let authority = self + .store_authority + .as_ref() + .ok_or_else(|| anyhow!("delegated terminal store authority is unavailable"))?; + anyhow::ensure!( + authority.is_current_transaction(), + "terminal publication requires its owned store transaction" + ); + authority.publish(&payload, Some(worker_id), None)?; + let image = authority.current_image()?; + let record = image + .state + .workers + .into_iter() + .find(|record| record.spec.worker_id == worker_id) + .ok_or_else(|| anyhow!("committed terminal worker receipt is unavailable"))?; + Ok(Some(record)) + } + /// Build the [`PersistedSubAgentState`] snapshot from the current fleet. /// /// This is a cheap clone operation that runs under the caller's lock. @@ -5925,11 +6386,13 @@ impl SubAgentManager { fn persist_state(&self) -> Result> { self.ensure_coordination_process_lock() .map_err(anyhow::Error::msg)?; - let Some((path, payload)) = self.build_persist_payload()? else { - // Nothing to persist — return a no-op handle. + let Some((_, payload)) = self.build_persist_payload()? else { return Ok(std::thread::spawn(|| {})); }; - let state_root = self.state_root.clone(); + let authority = self + .store_authority + .clone() + .ok_or_else(|| anyhow!("delegated store authority is unavailable"))?; let writer_locks = self .coordination_process_lock .lock() @@ -5938,24 +6401,25 @@ impl SubAgentManager { .cloned() .chain(self.workspace_writer_locks.values().cloned()) .collect::>(); - // Spawn disk I/O off the write-lock hot path. `payload` is fully - // owned (cloned from `self.agents`) so it is `Send` and safe to move. - let handle = std::thread::spawn(move || { + Ok(std::thread::spawn(move || { let _writer_locks = writer_locks; - if let Err(err) = write_json_atomic(&state_root, &path, &payload) { - tracing::warn!(target: "subagent", ?err, "failed to persist sub-agent state"); + if let Err(error) = authority.publish(&payload, None, None) { + tracing::warn!(target: "subagent", %error, "failed to persist sub-agent checkpoint"); } - }); - Ok(handle) + })) } fn persist_state_synchronously(&self) -> Result<()> { self.ensure_coordination_process_lock() .map_err(anyhow::Error::msg)?; - let Some((path, payload)) = self.build_persist_payload()? else { + let Some((_, payload)) = self.build_persist_payload()? else { return Ok(()); }; - write_json_atomic(&self.state_root, &path, &payload) + let authority = self + .store_authority + .as_ref() + .ok_or_else(|| anyhow!("delegated store authority is unavailable"))?; + authority.publish(&payload, None, None) } /// Fire-and-forget persist — logs errors, drops the join handle. @@ -6009,21 +6473,85 @@ impl SubAgentManager { /// /// Unlike `persist_state`, this performs disk I/O **synchronously** to /// guarantee data is flushed before the process exits. - pub fn flush_pending_persist(&mut self) { - if let Err(error) = self.ensure_coordination_process_lock() { - tracing::warn!(target: "subagent", %error, "skipping persist without state-root coordination lock"); - return; - } + pub fn flush_pending_persist(&mut self) -> Result<()> { + self.ensure_coordination_process_lock() + .map_err(anyhow::Error::msg)?; if self.persist_pending { + self.persist_state_synchronously()?; self.last_persist_at = Some(Instant::now()); self.persist_pending = false; - // Synchronous disk I/O — safe because we are shutting down and no - // callers depend on releasing the write lock quickly. - if let Ok(Some((path, payload))) = self.build_persist_payload() - && let Err(err) = write_json_atomic(&self.state_root, &path, &payload) - { - tracing::warn!(target: "subagent", ?err, "failed to flush pending sub-agent state"); - } + } + Ok(()) + } + + fn restore_persisted_agent(&self, persisted: PersistedSubAgent) -> SubAgent { + let nickname = persisted + .nickname + .filter(|name| generated_whale_name_base(&persisted.id, name).is_none()); + let mut status = persisted.status; + if matches!(status, SubAgentStatus::Running) + && !self.foreign_live_owners.contains(&persisted.id) + { + status = SubAgentStatus::Interrupted(SUBAGENT_RESTART_REASON.to_string()); + } + + let started_at = instant_from_duration(Duration::from_millis(persisted.duration_ms)); + // Empty vec on disk → None (full inheritance, v0.6.6 default). + // Non-empty vec → Some(list) (preserves narrow scope from older sessions). + let allowed_tools = if persisted.allowed_tools.is_empty() { + None + } else { + Some(persisted.allowed_tools) + }; + let mut assignment = persisted.assignment; + canonicalize_persisted_advisory_role(&mut assignment.role); + SubAgent { + id: persisted.id.clone(), + session_name: persisted + .session_name + .filter(|name| !name.trim().is_empty()) + .unwrap_or_else(|| persisted.id.clone()), + fork_context: persisted.fork_context, + workspace: persisted + .workspace + .unwrap_or_else(|| self.workspace.clone()), + agent_type: persisted.agent_type, + prompt: persisted.prompt, + assignment, + model: if persisted.model.is_empty() { + "unknown".to_string() + } else { + persisted.model + }, + // v0.8.68 and earlier persisted generated whale text. It may + // have been chosen under a different UI language, so never + // replay it into a new session. Explicit custom names survive. + nickname, + status, + result: persisted.result, + steps_taken: persisted.steps_taken, + checkpoint: persisted.checkpoint, + needs_input: persisted.needs_input, + started_at, + last_activity_at: started_at, + heartbeat_timeout: None, + allowed_tools, + // Empty string when loading pre-#405 records; the + // manager treats that the same as a non-matching id — + // i.e. agent classified as prior-session. + session_boot_id: persisted.session_boot_id, + owner_session_id: persisted.owner_session_id, + completion_claimed: false, + pending_terminal_result: None, + terminal_delivery: None, + work_lifecycle: None, + input_tx: None, + task_handle: None, + stop_token: None, + requested_stop: None, + unsettled_launch: None, + unsettled_foreground: None, + activation: None, } } @@ -6042,7 +6570,7 @@ impl SubAgentManager { &self.state_root, &Path::new(".deepseek") .join("state") - .join(SUBAGENT_STATE_FILE), + .join(LEGACY_SUBAGENT_STATE_FILE), )?; if legacy.exists() { tracing::info!( @@ -6057,7 +6585,10 @@ impl SubAgentManager { }; let raw = read_subagent_state_file(&self.state_root, &path)?; - let state = serde_json::from_str::(&raw)?; + let store = decode_subagent_store(&raw, true)?; + self.foreign_live_owners = + live_foreign_store_owners(&store, &path, &self.current_session_boot_id); + let state = store.state; state .validate_worker_origins() .map_err(anyhow::Error::msg)?; @@ -6089,72 +6620,8 @@ impl SubAgentManager { self.coordination = coordination; self.resume_targets = state.resume_targets; for persisted in state.agents { - let nickname = persisted - .nickname - .filter(|name| generated_whale_name_base(&persisted.id, name).is_none()); - let mut status = persisted.status; - if matches!(status, SubAgentStatus::Running) { - status = SubAgentStatus::Interrupted(SUBAGENT_RESTART_REASON.to_string()); - } - - let started_at = instant_from_duration(Duration::from_millis(persisted.duration_ms)); - // Empty vec on disk → None (full inheritance, v0.6.6 default). - // Non-empty vec → Some(list) (preserves narrow scope from older sessions). - let allowed_tools = if persisted.allowed_tools.is_empty() { - None - } else { - Some(persisted.allowed_tools) - }; - let mut assignment = persisted.assignment; - canonicalize_persisted_advisory_role(&mut assignment.role); - let agent = SubAgent { - id: persisted.id.clone(), - session_name: persisted - .session_name - .filter(|name| !name.trim().is_empty()) - .unwrap_or_else(|| persisted.id.clone()), - fork_context: persisted.fork_context, - workspace: persisted - .workspace - .unwrap_or_else(|| self.workspace.clone()), - agent_type: persisted.agent_type, - prompt: persisted.prompt, - assignment, - model: if persisted.model.is_empty() { - "unknown".to_string() - } else { - persisted.model - }, - // v0.8.68 and earlier persisted generated whale text. It may - // have been chosen under a different UI language, so never - // replay it into a new session. Explicit custom names survive. - nickname, - status, - result: persisted.result, - steps_taken: persisted.steps_taken, - checkpoint: persisted.checkpoint, - needs_input: persisted.needs_input, - started_at, - last_activity_at: started_at, - heartbeat_timeout: None, - allowed_tools, - // Empty string when loading pre-#405 records; the - // manager treats that the same as a non-matching id — - // i.e. agent classified as prior-session. - session_boot_id: persisted.session_boot_id, - owner_session_id: persisted.owner_session_id, - completion_claimed: false, - terminal_delivery: None, - work_lifecycle: None, - input_tx: None, - task_handle: None, - stop_token: None, - requested_stop: None, - unsettled_launch: None, - unsettled_foreground: None, - activation: None, - }; - self.agents.insert(persisted.id, agent); + let agent = self.restore_persisted_agent(persisted); + self.agents.insert(agent.id.clone(), agent); } for worker in state.workers { let worker = normalize_worker_record(worker); @@ -6184,6 +6651,8 @@ impl SubAgentManager { let orphaned = self .worker_records .values() + .filter(|record| !self.foreign_live_owners.contains(&record.spec.worker_id)) + .filter(|record| record.prepared_generation.is_none()) .filter(|record| { matches!( record.status, @@ -6204,6 +6673,10 @@ impl SubAgentManager { Some(*steps_taken), None, ); + self.worker_records + .get_mut(worker_id) + .expect("recovered worker exists") + .recovery_interrupted = true; } orphaned.len() } @@ -6220,19 +6693,22 @@ impl SubAgentManager { /// or prior-session owners stay put — a sibling session's live claim must /// never be released by this process. #[cfg(any(test, feature = "test-support"))] - pub fn finalize_session_close(&mut self) -> usize { - self.finalize_session_close_inner(None) + pub fn finalize_session_close(&mut self) -> Result { + self.with_store_transaction(|held| held.finalize_session_close_inner(None)) } - pub fn finalize_session_close_for_session(&mut self, active_session_id: &str) -> usize { - self.finalize_session_close_inner(Some(active_session_id)) + pub fn finalize_session_close_for_session(&mut self, active_session_id: &str) -> Result { + self.with_store_transaction(|held| { + held.finalize_session_close_inner(Some(active_session_id)) + }) } - fn finalize_session_close_inner(&mut self, active_session_id: Option<&str>) -> usize { + fn finalize_session_close_inner(&mut self, active_session_id: Option<&str>) -> Result { let reason = SUBAGENT_SESSION_CLOSED_REASON; let running_agent_ids = self .agents .values() + .filter(|agent| self.owns_store_record(&agent.id)) .filter(|agent| agent.status == SubAgentStatus::Running) .filter(|agent| { active_session_id @@ -6249,7 +6725,7 @@ impl SubAgentManager { terminal.status = SubAgentStatus::Interrupted(reason.to_string()); terminal.result = Some(reason.to_string()); terminal.needs_input = None; - if self.finish_terminal_result(agent_id, terminal, true, true) { + if self.finish_terminal_result(agent_id, terminal, true, true)? { finalized += 1; } } @@ -6265,6 +6741,7 @@ impl SubAgentManager { let live_worker_ids = self .worker_records .values() + .filter(|record| self.owns_store_record(&record.spec.worker_id)) .filter(|record| !record.status.is_terminal()) .filter(|record| !draining.contains(&record.spec.worker_id)) .filter(|record| { @@ -6303,15 +6780,21 @@ impl SubAgentManager { self.coordination.prune_claim_scopes(); // Drop finalized agents from the in-memory roster so they can never // be counted active again. Prior-session archive rows remain. - self.agents.retain(|id, _| !finalized_owners.contains(id)); - let _ = self.persist_state_synchronously(); + let finalized_agents = self + .agents + .extract_if(|id, _| finalized_owners.contains(id)) + .collect::>(); + if let Err(error) = self.persist_state_synchronously() { + self.agents.extend(finalized_agents); + return Err(error); + } tracing::debug!( target: "subagent", finalized, released, "finalized sub-agent fleet on session close" ); - finalized + Ok(finalized) } fn sorted_worker_records(&self) -> Vec { @@ -6329,7 +6812,12 @@ impl SubAgentManager { let oldest_terminal = self .worker_records .values() - .filter(|record| record.status.is_terminal()) + .filter(|record| { + self.owns_store_record(&record.spec.worker_id) + && record.prepared_generation.is_none() + && !record.recovery_interrupted + && record.status.is_terminal() + }) .filter(|record| { self.agents .get(&record.spec.worker_id) @@ -6385,6 +6873,9 @@ impl SubAgentManager { runtime_origin_turn_id: Option, ) { let worker_id = spec.worker_id.clone(); + if let Some(authority) = self.store_authority.as_ref() { + authority.owned.lock().insert(worker_id.clone()); + } let now_ms = epoch_millis_now(); let mut record = AgentWorkerRecord::new_for_session( normalize_worker_spec(spec), @@ -6393,7 +6884,8 @@ impl SubAgentManager { precomputed_evidence, ); record.runtime_origin_turn_id = runtime_origin_turn_id; - self.push_worker_event( + Self::push_worker_event( + &mut self.worker_event_seq, &mut record, AgentWorkerStatus::Starting, Some("starting".to_string()), @@ -6425,6 +6917,21 @@ impl SubAgentManager { spec: &AgentWorkerSpec, origin: &WorkerOrigin, ) -> Result<(), String> { + if self.state_path.is_some() + && !self + .store_authority + .as_ref() + .is_some_and(|authority| authority.is_current_transaction()) + { + return self + .with_store_transaction(|manager| { + manager + .preflight_worker_coordination_in_origin(spec, origin) + .map_err(anyhow::Error::msg) + }) + .map_err(|error| error.to_string()); + } + self.validate_worker_origin(origin)?; self.check_worker_origin_capacity(Some(&spec.worker_id), origin)?; let origin_workspace = self.coordination_workspace_in_origin( @@ -6493,6 +7000,21 @@ impl SubAgentManager { mut spec: AgentWorkerSpec, origin: WorkerOrigin, ) -> Result<(), String> { + if self.state_path.is_some() + && !self + .store_authority + .as_ref() + .is_some_and(|authority| authority.is_current_transaction()) + { + return self + .with_store_transaction(|manager| { + manager + .register_worker_with_coordination_in_origin(spec, origin) + .map_err(anyhow::Error::msg) + }) + .map_err(|error| error.to_string()); + } + self.validate_worker_origin(&origin)?; self.check_worker_origin_capacity(Some(&spec.worker_id), &origin)?; let origin_workspace = self.coordination_workspace_in_origin( @@ -6587,6 +7109,16 @@ impl SubAgentManager { pub fn coordination_registration_snapshot(&self) -> CoordinationRegistrationSnapshot { CoordinationRegistrationSnapshot { + store_revision: self.store_authority.as_ref().map_or(0, |authority| { + authority + .revision + .load(std::sync::atomic::Ordering::Acquire) + }), + writer_sequence: self.store_authority.as_ref().map_or(0, |authority| { + authority + .published_sequence + .load(std::sync::atomic::Ordering::Acquire) + }), worker_records: self.worker_records.clone(), coordination: self.coordination.clone(), worker_origins: self.worker_origins.clone(), @@ -6598,6 +7130,34 @@ impl SubAgentManager { &mut self, snapshot: CoordinationRegistrationSnapshot, ) -> Result<(), String> { + if self.state_path.is_some() + && !self + .store_authority + .as_ref() + .is_some_and(|authority| authority.is_current_transaction()) + { + return self + .with_store_transaction(|manager| { + manager + .restore_coordination_registration_snapshot(snapshot) + .map_err(anyhow::Error::msg) + }) + .map_err(|error| error.to_string()); + } + if let Some(authority) = self.store_authority.as_ref() { + let revision = authority + .revision + .load(std::sync::atomic::Ordering::Acquire); + let published = authority + .published_sequence + .load(std::sync::atomic::Ordering::Acquire); + if revision > snapshot.store_revision.saturating_add(1) + || published <= snapshot.writer_sequence + { + return Err("Fleet compensation snapshot is stale; committed worker records were retained for reconciliation".into()); + } + } + self.worker_records = snapshot.worker_records; self.coordination = snapshot.coordination; self.worker_origins = snapshot.worker_origins; @@ -6619,10 +7179,12 @@ impl SubAgentManager { /// persist forever and cascade-lock other agents"). A claim cannot contend /// for an owner that cannot write. fn is_live_coordination_owner(&self, id: &str) -> bool { - self.agents.get(id).is_some_and(|agent| { - agent.status == SubAgentStatus::Running && !self.is_from_prior_session(agent) - }) || self.worker_records.get(id).is_some_and(|record| { - !record.status.is_terminal() + self.foreign_live_owners.contains(id) + || self.agents.get(id).is_some_and(|agent| { + agent.status == SubAgentStatus::Running && !self.is_from_prior_session(agent) + }) + || self.worker_records.get(id).is_some_and(|record| { + !record.status.is_terminal() // A headless worker (no paired agent entry) that reaches // WaitingForUser can never be answered: no user path will // resume or finalize it, so counting it live leaks its claim @@ -6644,7 +7206,7 @@ impl SubAgentManager { .agents .get(id) .is_some_and(|agent| self.is_from_prior_session(agent)) - }) + }) } /// Resolve a normalized claim path against this manager's coordination @@ -6934,15 +7496,30 @@ impl SubAgentManager { .map_err(|error| format!("failed to persist test worker spec: {error}")) } - /// Persist the next exact launch generation for a Fleet restart without - /// changing the worker's authority, route, prompt, or retained lifecycle - /// evidence. The Fleet manager validates the full spec against the task + /// Persist the next exact launch generation as prepared for a Fleet restart, + /// preserving the worker's authority, route, prompt and retained events. + /// The Fleet manager validates the full spec against the task /// lease before calling this; this local check makes generation the only /// field that can change through this narrow seam. pub fn advance_registered_worker_generation( &mut self, spec: AgentWorkerSpec, ) -> Result<(), String> { + if self.state_path.is_some() + && !self + .store_authority + .as_ref() + .is_some_and(|authority| authority.is_current_transaction()) + { + return self + .with_store_transaction(|manager| { + manager + .advance_registered_worker_generation(spec) + .map_err(anyhow::Error::msg) + }) + .map_err(|error| error.to_string()); + } + self.ensure_coordination_process_lock()?; let worker_id = spec.worker_id.clone(); let previous = self @@ -6950,6 +7527,11 @@ impl SubAgentManager { .get(&worker_id) .cloned() .ok_or_else(|| format!("Fleet worker {worker_id} has no registered launch spec"))?; + if previous.prepared_previous_attempt_unsettled { + return Err(format!( + "Fleet worker {worker_id} still has an unsettled previous launch" + )); + } let old_generation = previous .spec .launch_manifest @@ -6963,10 +7545,13 @@ impl SubAgentManager { .ok_or_else(|| { format!("Fleet worker {worker_id} replacement has no launch manifest") })?; - if new_generation != old_generation.saturating_add(1) { + let next_generation = old_generation + .checked_add(1) + .ok_or_else(|| format!("Fleet worker {worker_id} launch generation is exhausted"))?; + if new_generation != next_generation { return Err(format!( "Fleet worker {worker_id} restart generation must advance from {old_generation} to {}", - old_generation.saturating_add(1) + next_generation )); } let mut expected = previous.spec.clone(); @@ -6981,13 +7566,23 @@ impl SubAgentManager { )); } - let record = self - .worker_records - .get_mut(&worker_id) - .expect("worker record checked above"); - record.spec = spec; - record.updated_at_ms = epoch_millis_now(); - if let Err(error) = self.persist_state_synchronously() { + if self.agents.contains_key(&worker_id) { + return Err("Fleet generation preparation requires a headless worker".to_string()); + } + let prepared = + previous.prepared_attempt(spec, &mut self.worker_event_seq, epoch_millis_now()); + self.worker_records.insert(worker_id.clone(), prepared); + let publication = (|| -> Result<()> { + let Some((_, payload)) = self.build_persist_payload()? else { + return Ok(()); + }; + let authority = self + .store_authority + .as_ref() + .ok_or_else(|| anyhow!("delegated store authority is unavailable"))?; + authority.publish(&payload, None, Some(&worker_id)) + })(); + if let Err(error) = publication { self.worker_records.insert(worker_id, previous); return Err(format!( "failed to persist Fleet restart launch generation: {error}" @@ -7002,19 +7597,126 @@ impl SubAgentManager { /// terminal. pub fn project_external_worker_status( &mut self, - worker_id: &str, + expected: &AgentWorkerSpec, status: AgentWorkerStatus, message: Option, - ) -> bool { - let Some(record) = self.worker_records.get(worker_id) else { - return false; - }; - if record.status == status { - return false; - } - self.record_worker_event(worker_id, status, message, None, None); - self.persist_state_best_effort(); - true + ) -> Result { + self.with_store_transaction(|manager| { + let worker_id = &expected.worker_id; + let Some(record) = manager.worker_records.get(worker_id) else { + return Ok(false); + }; + anyhow::ensure!( + !record.prepared_previous_attempt_unsettled, + "Fleet worker {worker_id} still has an unsettled previous launch" + ); + anyhow::ensure!( + manager.owns_store_record(worker_id), + "Fleet worker {worker_id} belongs to a live foreign writer" + ); + if &record.spec != expected { + let mut prepared = record.spec.clone(); + if let (Some(manifest), Some(expected_manifest)) = ( + prepared.launch_manifest.as_mut(), + expected.launch_manifest.as_ref(), + ) { + if record.prepared_generation == Some(manifest.generation) + && expected_manifest.generation.checked_add(1) == Some(manifest.generation) + { + manifest.generation = expected_manifest.generation; + if &prepared == expected { + return Ok(false); + } + } + } + bail!( + "Fleet worker {worker_id} lifecycle does not match its exact leased launch spec" + ); + } + if !status.is_terminal() + && (record.prepared_generation.is_some() + || record.status == AgentWorkerStatus::Starting + && record.started_at_ms.is_none()) + { + return Ok(false); + } + anyhow::ensure!( + !record.status.is_terminal() || status.is_terminal(), + "Fleet worker {worker_id} already has its winning terminal receipt" + ); + if record.status == status && record.prepared_generation.is_none() { + return Ok(false); + } + manager + .worker_records + .get_mut(worker_id) + .expect("worker checked above") + .prepared_generation = None; + manager.record_worker_event(worker_id, status, message, None, None); + manager.persist_state_synchronously()?; + Ok(true) + }) + } + + pub(crate) fn activate_registered_worker_attempt( + &mut self, + expected: &AgentWorkerSpec, + ) -> Result<()> { + self.with_store_transaction(|manager| { + let worker_id = &expected.worker_id; + let record = manager.worker_records.get(worker_id).ok_or_else(|| anyhow!("Fleet worker {worker_id} has no registered launch spec"))?; + anyhow::ensure!(!record.prepared_previous_attempt_unsettled, "Fleet worker {worker_id} still has an unsettled previous launch"); + anyhow::ensure!(manager.owns_store_record(worker_id), "Fleet worker {worker_id} belongs to a live foreign writer"); + anyhow::ensure!(&record.spec == expected, "Fleet worker {worker_id} activation does not match its exact leased launch spec"); + anyhow::ensure!(record.status == AgentWorkerStatus::Starting && record.started_at_ms.is_none(), "Fleet worker {worker_id} launch was already admitted or settled; reconcile before retrying"); + if let Some(prepared) = record.prepared_generation { + anyhow::ensure!(expected.launch_manifest.as_ref().is_some_and(|manifest| manifest.generation == prepared), "Fleet worker {worker_id} prepared generation does not match its launch"); + } + manager.worker_records.get_mut(worker_id).expect("worker checked above").prepared_generation = None; + manager.worker_records.get_mut(worker_id).expect("worker checked above").launch_admission_version = 1; + manager.record_worker_event(worker_id, AgentWorkerStatus::Starting, Some("Fleet executor launch admitted".to_string()), None, None); + manager.persist_state_synchronously() + }) + } + + pub(crate) fn confirm_previous_worker_attempt_settled( + &mut self, + previous: &AgentWorkerSpec, + ) -> Result<()> { + self.with_store_transaction(|manager| { + let worker_id = &previous.worker_id; + anyhow::ensure!( + manager.owns_store_record(worker_id), + "Fleet worker {worker_id} belongs to a live foreign writer" + ); + let record = manager + .worker_records + .get(worker_id) + .ok_or_else(|| anyhow!("Fleet worker {worker_id} has no prepared launch"))?; + let mut expected = previous.clone(); + let manifest = expected + .launch_manifest + .as_mut() + .ok_or_else(|| anyhow!("previous Fleet launch has no manifest"))?; + manifest.generation = manifest + .generation + .checked_add(1) + .ok_or_else(|| anyhow!("Fleet launch generation exhausted"))?; + let generation = manifest.generation; + anyhow::ensure!( + record.spec == expected && record.prepared_generation == Some(generation), + "Fleet worker {worker_id} stop proof does not match its exact prepared launch" + ); + if record.prepared_previous_attempt_unsettled { + manager + .worker_records + .get_mut(worker_id) + .expect("prepared worker checked") + .prepared_previous_attempt_unsettled = false; + manager.persist_state_synchronously()?; + } + Ok(()) + }) } fn record_worker_routed_usage( @@ -7194,7 +7896,7 @@ impl SubAgentManager { } fn push_worker_event( - &mut self, + sequence: &mut u64, record: &mut AgentWorkerRecord, status: AgentWorkerStatus, message: Option, @@ -7202,9 +7904,9 @@ impl SubAgentManager { tool_name: Option, now_ms: u64, ) { - self.worker_event_seq = self.worker_event_seq.saturating_add(1); + *sequence = sequence.saturating_add(1); record.events.push_back(AgentWorkerEvent { - seq: self.worker_event_seq, + seq: *sequence, worker_id: record.spec.worker_id.clone(), status, timestamp_ms: now_ms, @@ -7217,19 +7919,17 @@ impl SubAgentManager { } } - fn record_worker_event( - &mut self, - worker_id: &str, + fn project_worker_event( + sequence: &mut u64, + record: &mut AgentWorkerRecord, status: AgentWorkerStatus, message: Option, step: Option, tool_name: Option, + now_ms: u64, ) { - let now_ms = epoch_millis_now(); - let Some(mut record) = self.worker_records.remove(worker_id) else { - return; - }; record.status = status; + record.recovery_interrupted = false; record.recommended_action = recommended_action_for_worker_status(status, &record.spec); record.updated_at_ms = now_ms; record.latest_message = message.clone(); @@ -7265,7 +7965,30 @@ impl SubAgentManager { if let Some(step) = step { record.steps_taken = step; } - self.push_worker_event(&mut record, status, message, step, tool_name, now_ms); + Self::push_worker_event(sequence, record, status, message, step, tool_name, now_ms); + } + + fn record_worker_event( + &mut self, + worker_id: &str, + status: AgentWorkerStatus, + message: Option, + step: Option, + tool_name: Option, + ) { + let now_ms = epoch_millis_now(); + let Some(mut record) = self.worker_records.remove(worker_id) else { + return; + }; + Self::project_worker_event( + &mut self.worker_event_seq, + &mut record, + status, + message, + step, + tool_name, + now_ms, + ); self.worker_records.insert(worker_id.to_string(), record); self.reconcile_worker_lifecycle(worker_id); } @@ -7392,7 +8115,11 @@ impl SubAgentManager { }) } - fn complete_worker_from_result(&mut self, worker_id: &str, result: &SubAgentResult) { + fn worker_result_projection( + &self, + worker_id: &str, + result: &SubAgentResult, + ) -> Option<(AgentWorkerRecord, u64)> { // Delivery verification is deferred: `ensure_worker_delivery_verified` // computes it off the lock before the terminal commit (natural and // panic paths) or heals it on the next detail read (#6210). @@ -7411,7 +8138,8 @@ impl SubAgentManager { ), SubAgentStatus::Running => Some("running".to_string()), }; - if let Some(record) = self.worker_records.get_mut(worker_id) { + let mut record = self.worker_records.get(worker_id)?.clone(); + { record.result_summary = result.result.clone(); record.steps_taken = result.steps_taken; // #5906: carry the park/ask distinction from the checkpoint onto @@ -7426,11 +8154,41 @@ impl SubAgentManager { _ => {} } } - self.record_worker_event(worker_id, status, message, Some(result.steps_taken), None); + let mut sequence = self.worker_event_seq; + Self::project_worker_event( + &mut sequence, + &mut record, + status, + message, + Some(result.steps_taken), + None, + epoch_millis_now(), + ); + Some((record, sequence)) + } + + #[cfg(test)] + fn complete_worker_from_result(&mut self, worker_id: &str, result: &SubAgentResult) { + if let Some((record, sequence)) = self.worker_result_projection(worker_id, result) { + self.worker_event_seq = sequence; + self.worker_records.insert(worker_id.to_string(), record); + } } pub fn cancel_agent(&mut self, agent_ref: &str) -> Result { + self.with_store_transaction(|held| held.cancel_agent_inner(agent_ref)) + } + + fn cancel_agent_inner(&mut self, agent_ref: &str) -> Result { let agent_id = self.resolve_agent_ref(agent_ref)?; + if let Some(pending) = self + .agents + .get(&agent_id) + .and_then(|agent| agent.pending_terminal_result.clone()) + { + self.finish_terminal_result(&agent_id, pending.result, false, true)?; + return self.get_result(&agent_id); + } let mut terminal = { let agent = self .agents @@ -7451,7 +8209,7 @@ impl SubAgentManager { terminal.status = SubAgentStatus::Cancelled; terminal.result = Some(CANCELLED_BY_PARENT_RESULT.to_string()); terminal.needs_input = None; - if !self.finish_terminal_result(&agent_id, terminal, true, true) { + if !self.finish_terminal_result(&agent_id, terminal, true, true)? { return self.get_result(&agent_id); } self.get_result(&agent_id) @@ -8217,6 +8975,17 @@ impl SubAgentManager { agent_ref: &str, caller_agent_id: Option<&str>, reason: String, + ) -> Result<(SubAgentResult, SubAgentResult)> { + self.with_store_transaction(|held| { + held.interrupt_child_inner(agent_ref, caller_agent_id, reason) + }) + } + + fn interrupt_child_inner( + &mut self, + agent_ref: &str, + caller_agent_id: Option<&str>, + reason: String, ) -> Result<(SubAgentResult, SubAgentResult)> { if agent_ref.trim().eq_ignore_ascii_case("root") { return Err(anyhow!( @@ -8254,7 +9023,7 @@ impl SubAgentManager { terminal.steps_taken = checkpoint.steps_taken; terminal.checkpoint = Some(checkpoint); terminal.needs_input = None; - if !self.finish_terminal_result(&agent_id, terminal, true, true) { + if !self.finish_terminal_result(&agent_id, terminal, true, true)? { return Ok((prior, self.get_result(&agent_id)?)); } let snapshot = self.get_result(&agent_id)?; @@ -8775,6 +9544,27 @@ impl SubAgentManager { precomputed_delivery_evidence: Option, public_child_id: Option, ) -> Result { + if self.state_path.is_some() + && !self + .store_authority + .as_ref() + .is_some_and(|authority| authority.is_current_transaction()) + { + return self.with_store_transaction(|manager| { + manager.prepare_background_with_assignment_options( + manager_handle, + runtime, + agent_type, + prompt, + assignment, + allowed_tools, + options, + precomputed_delivery_evidence, + public_child_id, + ) + }); + } + if runtime .execution_limits .as_ref() @@ -8846,7 +9636,7 @@ impl SubAgentManager { // tool-only runtimes) keep `governor: None`. runtime.governor = Some(Arc::clone(&self.governor)); - self.cleanup(COMPLETED_AGENT_RETENTION); + self.cleanup(COMPLETED_AGENT_RETENTION)?; self.check_admission_capacity()?; if let Some(limits) = runtime.execution_limits.as_ref() { @@ -9497,6 +10287,17 @@ impl SubAgentManager { prepared: PreparedChild, mut configuration: FrozenChildConfiguration, ) -> Result<(SubAgentResult, Option)> { + if self.state_path.is_some() + && !self + .store_authority + .as_ref() + .is_some_and(|authority| authority.is_current_transaction()) + { + return self.with_store_transaction(|manager| { + manager.commit_prepared_child(prepared, configuration) + }); + } + if prepared .admission .as_ref() @@ -9655,8 +10456,23 @@ impl SubAgentManager { pub fn stop_child_activation( &mut self, identity: &ChildActivationIdentity, + ) -> Result { + self.with_store_transaction(|held| held.stop_child_activation_inner(identity)) + } + + fn stop_child_activation_inner( + &mut self, + identity: &ChildActivationIdentity, ) -> Result { for id in self.exact_activation_descendants(identity)? { + if let Some(pending) = self + .agents + .get(&id) + .and_then(|agent| agent.pending_terminal_result.clone()) + { + self.finish_terminal_result(&id, pending.result, false, true)?; + continue; + } if let Some(agent) = self.agents.get(&id) && agent.status == SubAgentStatus::Running && !agent.completion_claimed @@ -9664,7 +10480,7 @@ impl SubAgentManager { let mut result = self.snapshot_for_listing(agent); result.status = SubAgentStatus::Cancelled; result.needs_input = None; - self.finish_terminal_result(&id, result, true, true); + self.finish_terminal_result(&id, result, true, true)?; } } self.get_result(&identity.child_id) @@ -9696,6 +10512,10 @@ impl SubAgentManager { let expected_status = expected_agent.status.clone(); let expected_result = expected_agent.result.clone(); let state_root = self.state_root.clone(); + let store_authority = self + .store_authority + .clone() + .ok_or_else(|| anyhow!("child activation has no store authority"))?; let identity = identity.clone(); let writer_locks = self .coordination_process_lock @@ -9707,8 +10527,8 @@ impl SubAgentManager { .collect::>(); Ok(move || { let _writer_locks = writer_locks; - write_json_atomic(&state_root, &path, &payload)?; - let persisted: PersistedSubAgentState = serde_json::from_slice(&fs::read(&path)?)?; + store_authority.publish(&payload, None, None)?; + let persisted = read_durable_subagent_store(&state_root, &path)?.state; let worker = persisted .workers .iter() @@ -10152,20 +10972,29 @@ impl SubAgentManager { /// Clean up stale running agents and completed agents older than the /// given duration. Returns the number of running agents auto-cancelled /// during this pass. - pub fn cleanup(&mut self, max_age: Duration) -> usize { - self.cleanup_inner(None, max_age) + pub fn cleanup(&mut self, max_age: Duration) -> Result { + self.with_store_transaction(|held| held.cleanup_inner(None, max_age)) } - pub fn cleanup_for_session(&mut self, active_session_id: &str, max_age: Duration) -> usize { - self.cleanup_inner(Some(active_session_id), max_age) + pub fn cleanup_for_session( + &mut self, + active_session_id: &str, + max_age: Duration, + ) -> Result { + self.with_store_transaction(|held| held.cleanup_inner(Some(active_session_id), max_age)) } - fn cleanup_inner(&mut self, active_session_id: Option<&str>, max_age: Duration) -> usize { + fn cleanup_inner( + &mut self, + active_session_id: Option<&str>, + max_age: Duration, + ) -> Result { let before = self.agents.len(); let before_workers = self.worker_records.len(); let scoped_agent_owners = self .agents .iter() + .filter(|(id, _)| self.owns_store_record(id)) .filter(|(_, agent)| { active_session_id .is_none_or(|session_id| self.agent_is_owned_by_session(agent, session_id)) @@ -10176,6 +11005,11 @@ impl SubAgentManager { let scoped_worker_owners = self .worker_records .iter() + .filter(|(id, record)| { + self.owns_store_record(id) + && record.prepared_generation.is_none() + && !record.recovery_interrupted + }) .filter(|(_, record)| { active_session_id.is_none_or(|session_id| { !session_id.is_empty() && record.owner_session_id == session_id @@ -10191,6 +11025,16 @@ impl SubAgentManager { .collect::>(); transcript_candidates.sort(); transcript_candidates.dedup(); + let pending_terminals = self + .agents + .values() + .filter(|agent| scoped_agent_ids.contains(&agent.id)) + .filter_map(|agent| agent.pending_terminal_result.clone()) + .collect::>(); + for pending in pending_terminals { + let agent_id = pending.result.agent_id.clone(); + self.finish_terminal_result(&agent_id, pending.result, false, true)?; + } let mut auto_cancelled = 0; let timeout = self.running_heartbeat_timeout; let stale_agent_ids = self @@ -10246,13 +11090,7 @@ impl SubAgentManager { )); } terminal.needs_input = None; - // Cleanup batches stale transitions and persists the final fleet - // snapshot once below. Spawning one unordered background write - // per child could let an earlier partial snapshot rename last and - // resurrect a cancelled worker after restart. - // persist_after_commit=true still best-efforts; lock loss only - // skips disk — in-memory terminal state always commits. - if self.finish_terminal_result(&agent_id, terminal, true, true) { + if self.finish_terminal_result(&agent_id, terminal, true, true)? { auto_cancelled += 1; } } @@ -10281,19 +11119,27 @@ impl SubAgentManager { }) .map(|(id, _)| id.clone()) .collect::>(); - self.agents.retain(|agent_id, agent| { - if lineage_records.contains(agent_id) || retained_activations.contains(agent_id) { - return true; - } - if active_session_id.is_some() && !scoped_agent_ids.contains(agent_id) { - return true; - } - if agent.status == SubAgentStatus::Running { - true - } else { - agent.started_at.elapsed() < max_age - } - }); + let removed_agents = self + .agents + .extract_if(|agent_id, agent| { + if lineage_records.contains(agent_id) || retained_activations.contains(agent_id) { + return false; + } + if !scoped_agent_ids.contains(agent_id) + || self + .worker_records + .get(agent_id) + .is_some_and(|record| record.recovery_interrupted) + { + return false; + } + if agent.status == SubAgentStatus::Running { + false + } else { + agent.started_at.elapsed() >= max_age + } + }) + .collect::>(); // #4217: age-evict terminal worker ledger entries. Agents already drop // after `max_age`, but worker_records previously only had an LRU cap of // 256 — long-lived sessions rewrote multi-MB subagents.v1.json forever. @@ -10304,7 +11150,7 @@ impl SubAgentManager { if retained_activations.contains(worker_id) { return true; } - if active_session_id.is_some() && !scoped_worker_ids.contains(worker_id) { + if !scoped_worker_ids.contains(worker_id) { return true; } if !record.status.is_terminal() { @@ -10328,8 +11174,42 @@ impl SubAgentManager { // Also queue handle-store eviction for the same fully-retired agents // (#3885): handles pinned in memory per-agent are freed on the next // async drain by callers that hold the SharedHandleStore lock. + if self.agents.len() != before + || auto_cancelled > 0 + || self.worker_records.len() != before_workers + || self.resume_targets.len() != previous_links + { + if let Err(error) = self.persist_state_synchronously() { + self.agents.extend(removed_agents); + return Err(error); + } + } + let durable_ids = self + .store_authority + .as_ref() + .map(|authority| { + let state = authority.current_image()?.state; + Ok::<_, anyhow::Error>( + state + .agents + .into_iter() + .map(|agent| agent.id) + .chain( + state + .workers + .into_iter() + .map(|record| record.spec.worker_id), + ) + .collect::>(), + ) + }) + .transpose()? + .unwrap_or_default(); for agent_id in transcript_candidates { - if self.agents.contains_key(&agent_id) || self.worker_records.contains_key(&agent_id) { + if self.agents.contains_key(&agent_id) + || self.worker_records.contains_key(&agent_id) + || durable_ids.contains(&agent_id) + { continue; } if let Err(err) = remove_subagent_transcript_artifact(&self.state_root, &agent_id) { @@ -10348,15 +11228,8 @@ impl SubAgentManager { self.pending_handle_evictions .push((agent_id, owner_session_id)); } - if self.agents.len() != before - || auto_cancelled > 0 - || self.worker_records.len() != before_workers - || self.resume_targets.len() != previous_links - { - self.persist_state_best_effort(); - } self.last_cleanup_at = Some(Instant::now()); - auto_cancelled + Ok(auto_cancelled) } /// #3803: whether enough time has elapsed since the last `cleanup` that the @@ -10387,8 +11260,8 @@ impl SubAgentManager { /// The claim excludes cancellation while deliberately leaving the public /// status `Running`. `run_subagent_task` can therefore queue completion to /// the parent before the running-child gate closes (#1961). The winning - /// finisher performs only non-awaiting channel sends while it owns the - /// manager guard, then commits the terminal projections. + /// finisher records the durable terminal candidate and queues fan-in + /// while it owns the manager guard, then closes the live Running gate. fn claim_terminal_delivery(&mut self, agent_id: &str) -> bool { let Some(agent) = self.agents.get_mut(agent_id) else { return false; @@ -10400,91 +11273,120 @@ impl SubAgentManager { true } - /// Own, publish, and commit one terminal outcome. + fn terminal_result_projection( + &self, + agent_id: &str, + mut result: SubAgentResult, + ) -> (SubAgentResult, Option<(AgentWorkerRecord, u64)>) { + if let Some((status, message)) = self + .agents + .get(agent_id) + .and_then(|agent| agent.requested_stop.clone()) + && !matches!(result.status, SubAgentStatus::Failed(_)) + { + result.status = status; + result.result = message; + result.needs_input = None; + } + result.child_route = self + .worker_records + .get(agent_id) + .and_then(|record| record.spec.child_route.clone()); + let worker = self.worker_result_projection(agent_id, &result); + (result, worker) + } + + /// Own, durably commit, and publish one terminal outcome. /// - /// Claiming first makes natural completion, explicit Stop, coordination - /// interrupt, and stale cleanup race on one bit. The winning path attempts - /// every live fan-in send while both public projections still read - /// Running, then commits the matching agent and worker terminal states. - /// Late task output and repeated Stop calls cannot publish a second result. + /// Natural completion, Stop, interrupt and cleanup share one terminal + /// claim. The public Running gate closes after the durable receipt and + /// queued fan-in; failed publication retains the winning result and owner. fn finish_terminal_result( &mut self, agent_id: &str, mut result: SubAgentResult, abort_task: bool, persist_after_commit: bool, - ) -> bool { + ) -> Result { if result.status == SubAgentStatus::Running || result.agent_id != agent_id { - return false; + return Ok(false); } - if abort_task - && let Some(agent) = self.agents.get_mut(agent_id) - && let Some(token) = agent.stop_token.as_ref() - { - if agent.status != SubAgentStatus::Running - || agent.completion_claimed - || agent.requested_stop.is_some() - { - return false; - } - // Keep task/claims/fan-in ownership while Core persists its decision - // and projects the final checkpoint. The same task performs commit. - agent.requested_stop = Some((result.status, result.result)); - token.cancel(); - agent.input_tx = None; - return true; - } - if !self.claim_terminal_delivery(agent_id) { - return false; - } - if let Some((status, message)) = self + let pending = self .agents - .get_mut(agent_id) - .and_then(|agent| agent.requested_stop.take()) - { - // A failed owned join is uncertain, not a durable Cancelled receipt. - if !matches!(result.status, SubAgentStatus::Failed(_)) { - result.status = status; - result.result = message; - result.needs_input = None; - } - } - result.child_route = self - .worker_records .get(agent_id) - .and_then(|record| record.spec.child_route.clone()); - - // Nothing a finished agent asked for is pending any more. - self.withdraw_child_approvals_for_agent(agent_id); - if abort_task - && let Some(handle) = self - .agents - .get_mut(agent_id) - .and_then(|agent| agent.task_handle.take()) - { - handle.abort(); - } - + .and_then(|agent| agent.pending_terminal_result.clone()); + let worker = if let Some(pending) = pending { + result = pending.result; + if pending.prepared { + pending.worker + } else { + let (projected, worker) = self.terminal_result_projection(agent_id, result); + result = projected; + worker + } + } else { + if abort_task + && let Some(agent) = self.agents.get_mut(agent_id) + && let Some(token) = agent.stop_token.as_ref() + { + if agent.status != SubAgentStatus::Running + || agent.completion_claimed + || agent.requested_stop.is_some() + { + return Ok(false); + } + agent.requested_stop = Some((result.status, result.result)); + token.cancel(); + agent.input_tx = None; + return Ok(true); + } + if !self.claim_terminal_delivery(agent_id) { + return Ok(false); + } + let (projected, worker) = self.terminal_result_projection(agent_id, result); + result = projected; + worker + }; let delivery = self .agents .get(agent_id) .and_then(|agent| agent.terminal_delivery.clone()); - if let Some(delivery) = delivery { - delivery.deliver(self, &result); + if let Some((_, sequence)) = &worker { + self.worker_event_seq = self.worker_event_seq.max(*sequence); } - - let session_closed = matches!(&result.status, SubAgentStatus::Interrupted(reason) - if reason == SUBAGENT_SESSION_CLOSED_REASON); - let committed = - self.update_from_result_with_persist(agent_id, result, persist_after_commit); - if committed && session_closed { - // The asynchronous close request retained these exact claims while - // its task drained. Retire only after the same terminal commit. - self.coordination - .write_claims - .retain(|record| record.claim.owner != agent_id); - self.coordination.prune_claim_scopes(); - self.persist_state_best_effort(); + let committed = self.commit_terminal_result( + agent_id, + result.clone(), + worker.clone(), + persist_after_commit, + abort_task, + delivery.clone(), + ); + if let Err(error) = &committed { + if let Some(agent) = self.agents.get_mut(agent_id) { + agent.pending_terminal_result = Some(PendingTerminalResult { + result, + worker, + prepared: true, + }); + agent.result = Some(format!( + "Worker execution ended, but its durable terminal receipt is uncertain: {error}" + )); + } + if let Some(delivery) = delivery { + if let Some(owner) = delivery.event_delivery.as_ref() { + owner.mark_failed(); + } + if let Some(tx) = delivery.event_tx.as_ref() { + send_terminal_event_with_delivery( + tx, + Event::error(codewhale_error_taxonomy::ErrorEnvelope::transient(format!( + "Worker {agent_id} terminal receipt is uncertain: {error}" + ))), + delivery.event_delivery.as_ref(), + ); + } + } } committed } @@ -10598,36 +11500,116 @@ impl SubAgentManager { /// Commit a claimed natural task result. /// - /// Returns `true` only when the prior claim still owns the terminal - /// transition. External notification is deliberately queued between - /// [`Self::claim_terminal_delivery`] and this commit. + /// Returns true only when the claim still owns the terminal transition + /// and its durable receipt is acknowledged before the live projection. #[cfg(test)] - fn update_from_result(&mut self, agent_id: &str, result: SubAgentResult) -> bool { - self.update_from_result_with_persist(agent_id, result, true) + fn update_from_result(&mut self, agent_id: &str, result: SubAgentResult) -> Result { + self.with_store_transaction(|held| { + let worker = held.worker_result_projection(agent_id, &result); + held.commit_terminal_result(agent_id, result, worker, true, false, None) + }) } - fn update_from_result_with_persist( + fn commit_terminal_result( &mut self, agent_id: &str, - result: SubAgentResult, - persist_after_commit: bool, - ) -> bool { - let Some(agent) = self.agents.get_mut(agent_id) else { - return false; + mut result: SubAgentResult, + mut worker: Option<(AgentWorkerRecord, u64)>, + persist: bool, + abort_task: bool, + delivery: Option, + ) -> Result { + let Some(agent) = self.agents.get(agent_id) else { + return Ok(false); }; if agent.status != SubAgentStatus::Running || !agent.completion_claimed { - return false; + return Ok(false); } + if let Some(agent) = self.agents.get_mut(agent_id) { + agent.pending_terminal_result = Some(PendingTerminalResult { + result: result.clone(), + worker: worker.clone(), + prepared: true, + }); + } + let session_closed = matches!(&result.status, SubAgentStatus::Interrupted(reason) + if reason == SUBAGENT_SESSION_CLOSED_REASON); + if persist { + if let Some((_, mut payload)) = self.build_persist_payload()? { + let agent = payload + .agents + .iter_mut() + .find(|agent| agent.id == agent_id) + .ok_or_else(|| { + anyhow!("terminal worker is absent from persistence snapshot") + })?; + agent.status = result.status.clone(); + agent.assignment = result.assignment.clone(); + agent.result = result.result.clone(); + agent.steps_taken = result.steps_taken; + agent.checkpoint = result.checkpoint.clone(); + agent.needs_input = result.needs_input.clone(); + if let Some((record, _)) = &worker { + let persisted = payload + .workers + .iter_mut() + .find(|record| record.spec.worker_id == agent_id) + .ok_or_else(|| { + anyhow!("terminal worker record is absent from persistence snapshot") + })?; + *persisted = record.clone(); + } + if session_closed { + payload + .coordination + .write_claims + .retain(|record| record.claim.owner != agent_id); + payload.coordination.prune_claim_scopes(); + } + if let Some(record) = self.persist_terminal_candidate(agent_id, payload)? { + result.usage = Some(record.usage.clone()); + let sequence = record.events.back().map_or(0, |event| event.seq); + worker = Some((record, sequence)); + } + } + } + if let Some((record, sequence)) = &worker { + self.worker_event_seq = self.worker_event_seq.max(*sequence); + self.worker_records + .insert(agent_id.to_string(), record.clone()); + } + if let Some(agent) = self.agents.get_mut(agent_id) { + agent.pending_terminal_result = Some(PendingTerminalResult { + result: result.clone(), + worker: worker.clone(), + prepared: true, + }); + } + self.withdraw_child_approvals_for_agent(agent_id); + if abort_task + && let Some(handle) = self + .agents + .get_mut(agent_id) + .and_then(|agent| agent.task_handle.take()) + { + handle.abort(); + } + if let Some(delivery) = delivery { + delivery.deliver(self, &result); + } + let agent = self + .agents + .get_mut(agent_id) + .expect("claimed terminal owner"); agent.status = result.status.clone(); - agent.assignment = result.assignment.clone(); - agent.result = result.result.clone(); + agent.assignment = result.assignment; + agent.result = result.result; agent.steps_taken = result.steps_taken; - agent.checkpoint = result.checkpoint.clone(); - agent.needs_input = result.needs_input.clone(); - if result.status != SubAgentStatus::Running { - agent.input_tx = None; - } + agent.checkpoint = result.checkpoint; + agent.needs_input = result.needs_input; + agent.input_tx = None; agent.completion_claimed = false; + agent.pending_terminal_result = None; agent.task_handle = None; agent.stop_token = None; agent.requested_stop = None; @@ -10635,11 +11617,16 @@ impl SubAgentManager { drop(agent.unsettled_foreground.take()); agent.terminal_delivery = None; release_resident_leases_for(agent_id); - self.complete_worker_from_result(agent_id, &result); - if persist_after_commit { - self.persist_state_best_effort(); + if worker.is_some() { + self.reconcile_worker_lifecycle(agent_id); } - true + if session_closed { + self.coordination + .write_claims + .retain(|record| record.claim.owner != agent_id); + self.coordination.prune_claim_scopes(); + } + Ok(true) } fn update_checkpoint(&mut self, agent_id: &str, checkpoint: SubAgentCheckpoint) -> bool { @@ -10874,6 +11861,20 @@ pub async fn persist_child_settlement( manager: &SharedSubAgentManager, identity: &ChildActivationIdentity, ) -> Result { + let (pending, activation) = { + let held = manager.read().await; + let agent = held.exact_activation(identity)?; + ( + agent.pending_terminal_result.clone(), + agent.activation.clone().expect("validated activation"), + ) + }; + if let Some(pending) = pending + && let Err(error) = finish_terminal_result_owned(manager.clone(), pending.result).await + { + fail_child_settlement(&activation, &error); + return Err(error); + } let (write, state) = { let held = manager.read().await; let agent = held.exact_activation(identity)?; @@ -10914,12 +11915,7 @@ pub async fn persist_child_settlement( Err(error) => Err(error), }; if let Err(error) = outcome { - if state.admission.borrow().is_none() { - state.admission.send_replace(Some(Err(format!("child stopped before startup acknowledgement; durable receipt is uncertain: {error}")))); - } - state.settlement.send_replace(Some(Err(format!( - "child stopped but its durable receipt is unacknowledged: {error}" - )))); + fail_child_settlement(&state, &error); return Err(error); } let mut held = manager.write().await; @@ -10952,43 +11948,60 @@ pub async fn persist_child_settlement( Ok(result) } +fn fail_child_settlement(state: &ChildActivationState, error: &anyhow::Error) { + if state.admission.borrow().is_none() { + state.admission.send_replace(Some(Err(format!( + "child stopped before startup acknowledgement; durable receipt is uncertain: {error}" + )))); + } + state.settlement.send_replace(Some(Err(format!( + "child stopped but its durable receipt is unacknowledged: {error}" + )))); +} + pub async fn dispose_child_activation( manager: &SharedSubAgentManager, identity: &ChildActivationIdentity, ) -> Result { let deadline = tokio::time::Instant::now() + CHILD_STOP_SETTLE_GRACE; loop { - let settled = { - let mut held = tokio::time::timeout_at(deadline, manager.write()) - .await - .map_err(|_| anyhow!("child disposal could not reach its execution owner"))?; - held.stop_child_activation(identity)?; - let exact = held.exact_activation(identity)?; - if exact.status == SubAgentStatus::Running - && exact.activation.as_ref().is_some_and(|state| { - state - .execution_settled - .load(std::sync::atomic::Ordering::Acquire) - }) - { - return Err(anyhow!( - "child execution teardown is unresolved; ownership remains retained" - )); - } - held.exact_activation_descendants(identity)? - .iter() - .all(|id| { - held.agents.get(id).is_some_and(|agent| { - agent.status != SubAgentStatus::Running - && !agent.completion_claimed - && agent.activation.as_ref().is_none_or(|state| { - state - .execution_settled - .load(std::sync::atomic::Ordering::Acquire) - }) + let owner = manager.clone(); + let owned_identity = identity.clone(); + let settled = tokio::time::timeout_at( + deadline, + codewhale_runtime::runtime_control::owner_work(move || { + let mut held = owner.blocking_write(); + held.stop_child_activation(&owned_identity)?; + let exact = held.exact_activation(&owned_identity)?; + if exact.status == SubAgentStatus::Running + && exact.activation.as_ref().is_some_and(|state| { + state + .execution_settled + .load(std::sync::atomic::Ordering::Acquire) }) - }) - }; + { + return Err(anyhow!( + "child execution teardown is unresolved; ownership remains retained" + )); + } + Ok(held + .exact_activation_descendants(&owned_identity)? + .iter() + .all(|id| { + held.agents.get(id).is_some_and(|agent| { + agent.status != SubAgentStatus::Running + && !agent.completion_claimed + && agent.activation.as_ref().is_none_or(|state| { + state + .execution_settled + .load(std::sync::atomic::Ordering::Acquire) + }) + }) + })) + }), + ) + .await + .map_err(|_| anyhow!("child disposal could not reach its execution owner"))??; if settled { return persist_child_settlement(manager, identity).await; } @@ -11035,25 +12048,37 @@ async fn run_prepared_subagent_task( task.runtime.activation = launch.activation; run_subagent_task(task).await; } else { - let identity = { + let (identity, result) = { let mut manager = task.manager_handle.write().await; let identity = manager .agents .get(&task.agent_id) .and_then(|agent| agent.activation.as_ref()) .map(|state| state.identity.clone()); - if let Ok(mut result) = manager.get_result(&task.agent_id) { - result.status = SubAgentStatus::Cancelled; - result.result = None; - manager.finish_terminal_result(&task.agent_id, result, false, true); + if let Some(agent) = manager.agents.get_mut(&task.agent_id) + && identity.is_none() + && let Some(foreground) = task._foreground_child_registration.take() + { + agent.unsettled_foreground = Some(foreground); } - identity + (identity, manager.get_result(&task.agent_id)) }; + if let Ok(mut result) = result { + result.status = SubAgentStatus::Cancelled; + result.result = None; + if let Err(error) = + finish_terminal_result_owned(task.manager_handle.clone(), result).await + { + tracing::error!(target: "subagent", ?error, "prepared child terminal receipt is uncertain"); + } + } if let Some(identity) = identity { { let mut held = task.manager_handle.write().await; if let Some(agent) = held.agents.get_mut(&task.agent_id) { - agent.unsettled_foreground = task._foreground_child_registration.take(); + if let Some(foreground) = task._foreground_child_registration.take() { + agent.unsettled_foreground = Some(foreground); + } if let Some(state) = agent.activation.as_ref() { state .execution_settled @@ -11119,8 +12144,13 @@ pub fn load_persisted_agent_worker_records_with_state_root( state_root: &Path, ) -> Result> { let mut manager = - SubAgentManager::new_with_state_root(workspace.to_path_buf(), state_root.to_path_buf(), 1) - .with_state_path(default_state_path(state_root)?); + SubAgentManager::new_with_state_root(workspace.to_path_buf(), state_root.to_path_buf(), 1); + let path = default_state_path(state_root)?; + manager.state_path = Some(if path.exists() { + path + } else { + path.with_file_name(LEGACY_SUBAGENT_STATE_FILE) + }); manager.load_state()?; Ok(manager.list_worker_records()) } @@ -11666,9 +12696,8 @@ pub fn write_subagent_transcript_artifact_for_test( fn default_state_path(state_root: &Path) -> Result { let state_root = normalize_subagent_workspace(state_root); - // Canonical post-rebrand state path. On first run the file won't exist yet; - // write_json_atomic creates parent directories. Legacy .deepseek/state/ data - // is migrated on load (see load_state). + // Canonical delegated state path. Store admission creates its parents and + // imports legacy state before allowing durable writes. checked_subagent_state_path( &state_root, &Path::new(".codewhale") @@ -12124,30 +13153,847 @@ fn instant_from_duration(duration: Duration) -> Instant { .unwrap_or_else(Instant::now) } -/// `persist_state_best_effort` fires a fresh thread per call, so multiple -/// persists of the same `state.json` can be in flight at once; the publish -/// sequence below keeps an older snapshot from replacing a newer one. -static STATE_PUBLISH_SEQUENCES: std::sync::OnceLock>> = - std::sync::OnceLock::new(); +#[derive(Debug, Clone, Serialize, Deserialize)] +struct DurableSubAgentStore { + store_schema: u32, + revision: u64, + state: PersistedSubAgentState, + #[serde(default)] + record_writers: HashMap, + #[serde(default)] + writer_sequences: HashMap, +} -fn write_json_atomic(state_root: &Path, path: &Path, value: &PersistedSubAgentState) -> Result<()> { +impl DurableSubAgentStore { + fn from_legacy(state: PersistedSubAgentState) -> Self { + Self { + store_schema: 3, + revision: 0, + state, + record_writers: HashMap::new(), + writer_sequences: HashMap::new(), + } + } + + fn validate(&mut self) -> Result<()> { + anyhow::ensure!(self.store_schema == 3, "unsupported delegated store format"); + self.state + .validate_worker_origins() + .map_err(anyhow::Error::msg)?; + self.state + .coordination + .validate_replay() + .map_err(anyhow::Error::msg)?; + anyhow::ensure!( + self.record_writers.len() <= MAX_AGENT_WORKER_RECORDS * 2 + && self.writer_sequences.len() <= MAX_AGENT_WORKER_RECORDS * 2, + "delegated writer provenance exceeds its capacity" + ); + for writer in self + .record_writers + .values() + .chain(self.writer_sequences.keys()) + { + anyhow::ensure!( + valid_store_writer(writer), + "invalid delegated writer identity" + ); + } + Ok(()) + } +} + +struct SubAgentStoreTransaction { + _file: fs::File, + thread: std::thread::ThreadId, + image: DurableSubAgentStore, + publishes: u64, +} + +struct SubAgentStoreAuthority { + state_root: PathBuf, + path: PathBuf, + writer: String, + _lease: fs::File, + owned: parking_lot::Mutex>, + transaction: parking_lot::Mutex>, + revision: std::sync::atomic::AtomicU64, + published_sequence: std::sync::atomic::AtomicU64, +} + +struct SubAgentStoreTransactionLease { + authority: Arc, +} + +impl Drop for SubAgentStoreTransactionLease { + fn drop(&mut self) { + self.authority.transaction.lock().take(); + } +} + +fn valid_store_writer(writer: &str) -> bool { + writer.starts_with("boot_") + && writer.len() <= 64 + && writer + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || byte == b'_' || byte == b'-') +} + +fn open_store_private_file(root: &Path, path: &Path) -> Result { + let path = checked_subagent_state_path(root, path)?; + let parent = path + .parent() + .ok_or_else(|| anyhow!("delegated store lock has no parent"))?; + let name = path + .file_name() + .and_then(std::ffi::OsStr::to_str) + .ok_or_else(|| anyhow!("delegated store lock has no UTF-8 name"))?; + let directory = codewhale_config::private_directory::PrivateDirectory::open(parent)?; + let file = directory.open_internal_file(name)?; + anyhow::ensure!( + directory.is_at_selected_path()?, + "delegated store directory changed" + ); + Ok(file) +} + +fn acquire_store_transaction(root: &Path, path: &Path) -> Result { + anyhow::ensure!( + cfg!(any(unix, windows)), + "delegated cross-process transactions are unsupported on this platform" + ); + let file = open_store_private_file(root, &path.with_extension("transaction.lock"))?; + let deadline = Instant::now() + Duration::from_secs(5); + loop { + match codewhale_util::lock::try_lock_file_exclusive(&file) { + Ok(true) => return Ok(file), + Ok(false) if Instant::now() < deadline => std::thread::sleep(Duration::from_millis(10)), + Ok(false) => { + return Err(anyhow!( + "{COORDINATION_LOCK_TIMEOUT_MARKER}; retry delegated operation" + )); + } + Err(error) => return Err(error.into()), + } + } +} + +fn decode_subagent_store(raw: &str, allow_legacy: bool) -> Result { + let mut value: Value = serde_json::from_str(raw)?; + let state = if value.get("state").is_some() { + value.get_mut("state").expect("state checked") + } else { + &mut value + }; + if let Some(workers) = state.get_mut("workers").and_then(Value::as_array_mut) { + for worker in workers { + if !worker.get("prepared_generation").is_none_or(Value::is_null) + && let Some(object) = worker.as_object_mut() + { + object + .entry("prepared_previous_attempt_unsettled") + .or_insert(Value::Bool(true)); + } + } + } + let object = value + .as_object() + .ok_or_else(|| anyhow!("delegated store must be an object"))?; + let envelope = [ + "store_schema", + "revision", + "state", + "record_writers", + "writer_sequences", + ] + .iter() + .any(|key| object.contains_key(*key)); + let mut store = if envelope { + serde_json::from_value::(value)? + } else { + anyhow::ensure!(allow_legacy, "delegated store has no format3 discriminator"); + DurableSubAgentStore::from_legacy(serde_json::from_value::(value)?) + }; + store.validate()?; + Ok(store) +} + +fn read_durable_subagent_store(root: &Path, path: &Path) -> Result { + decode_subagent_store(&read_subagent_state_file(root, path)?, false) +} + +fn sync_subagent_store_directory(path: &Path) -> Result<()> { + #[cfg(unix)] + fs::File::open(path)?.sync_all()?; + let _ = path; + Ok(()) +} + +fn ensure_subagent_store_cutover(root: &Path, path: &Path) -> Result<()> { + if path.file_name() != Some(std::ffi::OsStr::new(SUBAGENT_STATE_FILE)) { + if !path.exists() { + write_json_atomic( + root, + path, + &DurableSubAgentStore::from_legacy(PersistedSubAgentState::default()), + )?; + } else { + let raw = read_subagent_state_file(root, path)?; + let store = decode_subagent_store(&raw, true)?; + if serde_json::from_str::(&raw)? + .get("store_schema") + .is_none() + { + write_json_atomic(root, path, &store)?; + } + } + return Ok(()); + } + let parent = path + .parent() + .ok_or_else(|| anyhow!("delegated store has no parent"))?; + let legacy = parent.join(LEGACY_SUBAGENT_STATE_FILE); + reject_root_relative_symlinks(root, &legacy)?; + if path.exists() { + read_durable_subagent_store(root, path)?; + anyhow::ensure!( + legacy.is_dir(), + "delegated v3 store lost its legacy-write tombstone; inspect retained data before continuing" + ); + let _ = codewhale_config::private_directory::PrivateDirectory::inspect(&legacy)?; + return Ok(()); + } + let _migration = if legacy.is_dir() { + None + } else { + Some(CoordinationProcessLock::acquire(root, true)?) + }; + if path.exists() { + read_durable_subagent_store(root, path)?; + anyhow::ensure!( + legacy.is_dir(), + "delegated migration did not retain its legacy-write tombstone" + ); + return Ok(()); + } + let staging = parent.join("subagents.v1.cutover"); + let backup_name = "legacy.json"; + let original_name = "original.json"; + if !legacy.is_dir() { + let staged = codewhale_config::private_directory::PrivateDirectory::open(&staging)?; + let original = staging.join(original_name); + let backup = staging.join(backup_name); + let source = if legacy.is_file() { + Some(legacy.clone()) + } else if original.is_file() { + Some(original.clone()) + } else if backup.is_file() { + Some(backup.clone()) + } else { + let old = checked_subagent_state_path( + root, + &Path::new(".deepseek") + .join("state") + .join(LEGACY_SUBAGENT_STATE_FILE), + )?; + old.is_file().then_some(old) + }; + let raw = source + .as_ref() + .map(|source| read_subagent_state_file(root, source)) + .transpose()? + .unwrap_or(serde_json::to_string(&PersistedSubAgentState::default())?); + let state = decode_subagent_store(&raw, true)?.state; + anyhow::ensure!( + serde_json::from_str::(&raw)? + .get("store_schema") + .is_none(), + "legacy migration input is already a versioned envelope; retain it for inspection" + ); + state + .validate_worker_origins() + .map_err(anyhow::Error::msg)?; + let mut ledger = state.coordination.clone(); + ledger.validate_replay().map_err(anyhow::Error::msg)?; + codewhale_util::utils::write_atomic(&backup, raw.as_bytes())?; + if legacy.is_file() { + anyhow::ensure!( + !original.exists(), + "delegated migration retains a previous legacy file; inspect both before retrying" + ); + fs::rename(&legacy, &original)?; + sync_subagent_store_directory(parent)?; + sync_subagent_store_directory(&staging)?; + } + anyhow::ensure!( + staged.is_at_selected_path()?, + "delegated migration directory changed" + ); + drop(staged); + fs::rename(&staging, &legacy)?; + sync_subagent_store_directory(parent)?; + } + let tombstone = codewhale_config::private_directory::PrivateDirectory::inspect(&legacy)?; + let raw = read_subagent_state_file(root, &legacy.join(backup_name))?; + let state = decode_subagent_store(&raw, true)?.state; + anyhow::ensure!( + serde_json::from_str::(&raw)? + .get("store_schema") + .is_none(), + "legacy tombstone backup is a versioned envelope; retain it for inspection" + ); + state + .validate_worker_origins() + .map_err(anyhow::Error::msg)?; + anyhow::ensure!( + tombstone.is_at_selected_path()?, + "delegated migration tombstone changed" + ); + write_json_atomic(root, path, &DurableSubAgentStore::from_legacy(state)) +} + +fn store_writer_is_live(path: &Path, writer: &str) -> bool { + if !valid_store_writer(writer) { + return true; + } + let path = path.with_file_name(format!("subagents.v3.writer.{writer}.lock")); + let Some(directory) = path.parent() else { + return true; + }; + let Ok(parent) = codewhale_config::private_directory::PrivateDirectory::inspect(directory) + else { + return true; + }; + let Some(name) = path.file_name().and_then(std::ffi::OsStr::to_str) else { + return true; + }; + let file = match parent.open_owned_file_for_read(name) { + Ok(Some(file)) => file, + Ok(None) | Err(_) => return true, + }; + !matches!( + codewhale_util::lock::try_lock_file_exclusive(&file), + Ok(true) + ) +} + +fn live_foreign_store_owners( + image: &DurableSubAgentStore, + path: &Path, + writer: &str, +) -> HashSet { + image + .record_writers + .iter() + .filter(|(id, owner)| { + owner.as_str() != writer + && store_writer_is_live(path, owner) + && image + .state + .workers + .iter() + .find(|record| &record.spec.worker_id == *id) + .map_or_else( + || { + image.state.agents.iter().any(|agent| { + &agent.id == *id && agent.status == SubAgentStatus::Running + }) + }, + |record| { + !record.status.is_terminal() + && !record.parked_at_turn_end + && !(record.status == AgentWorkerStatus::WaitingForUser + && !image + .state + .agents + .iter() + .any(|agent| agent.id == record.spec.worker_id)) + }, + ) + }) + .map(|(id, _)| id.clone()) + .collect() +} + +impl SubAgentStoreAuthority { + fn open(root: &Path, path: &Path, writer: &str) -> Result> { + anyhow::ensure!( + valid_store_writer(writer), + "invalid delegated writer identity" + ); + let _opening = acquire_store_transaction(root, path)?; + ensure_subagent_store_cutover(root, path)?; + let image = read_durable_subagent_store(root, path)?; + sync_subagent_store_directory( + path.parent() + .ok_or_else(|| anyhow!("delegated store has no parent"))?, + )?; + let lease_path = path.with_file_name(format!("subagents.v3.writer.{writer}.lock")); + let lease = open_store_private_file(root, &lease_path)?; + anyhow::ensure!( + codewhale_util::lock::try_lock_file_exclusive(&lease)?, + "delegated writer identity is already live" + ); + Ok(Arc::new(Self { + state_root: root.to_path_buf(), + path: path.to_path_buf(), + writer: writer.into(), + _lease: lease, + owned: parking_lot::Mutex::new(HashSet::new()), + transaction: parking_lot::Mutex::new(None), + revision: std::sync::atomic::AtomicU64::new(image.revision), + published_sequence: std::sync::atomic::AtomicU64::new( + image.writer_sequences.get(writer).copied().unwrap_or(0), + ), + })) + } + + fn writer_is_live(&self, writer: &str) -> bool { + writer == self.writer || store_writer_is_live(&self.path, writer) + } + + fn is_current_transaction(&self) -> bool { + self.transaction + .lock() + .as_ref() + .is_some_and(|tx| tx.thread == std::thread::current().id()) + } + + fn begin(self: &Arc) -> Result { + anyhow::ensure!( + self.transaction.lock().is_none(), + "delegated store transaction already belongs to another thread" + ); + let file = acquire_store_transaction(&self.state_root, &self.path)?; + let image = read_durable_subagent_store(&self.state_root, &self.path)?; + let observed = self.revision.load(std::sync::atomic::Ordering::Acquire); + anyhow::ensure!( + image.revision >= observed, + "delegated store revision regressed; inspect retained state" + ); + if image.revision > observed { + sync_subagent_store_directory( + self.path + .parent() + .ok_or_else(|| anyhow!("delegated store has no parent"))?, + )?; + self.revision + .store(image.revision, std::sync::atomic::Ordering::Release); + self.published_sequence.fetch_max( + image + .writer_sequences + .get(&self.writer) + .copied() + .unwrap_or(0), + std::sync::atomic::Ordering::AcqRel, + ); + } + *self.transaction.lock() = Some(SubAgentStoreTransaction { + _file: file, + thread: std::thread::current().id(), + image, + publishes: 0, + }); + Ok(SubAgentStoreTransactionLease { + authority: self.clone(), + }) + } + + fn current_image(&self) -> Result { + if let Some(tx) = self.transaction.lock().as_ref() + && tx.thread == std::thread::current().id() + { + return Ok(tx.image.clone()); + } + read_durable_subagent_store(&self.state_root, &self.path) + } + + fn publish( + &self, + value: &PersistedSubAgentState, + terminal: Option<&str>, + generation_prepare: Option<&str>, + ) -> Result<()> { + let mut current = self.transaction.lock(); + if let Some(tx) = current.as_mut() + && tx.thread == std::thread::current().id() + { + self.merge_publish(&mut tx.image, value, terminal, true, generation_prepare)?; + tx.publishes = tx.publishes.saturating_add(1); + return Ok(()); + } + drop(current); + anyhow::ensure!( + generation_prepare.is_none(), + "generation preparation requires its owned store transaction" + ); + let _file = acquire_store_transaction(&self.state_root, &self.path)?; + let mut image = read_durable_subagent_store(&self.state_root, &self.path)?; + self.merge_publish(&mut image, value, terminal, false, None) + } + + fn merge_publish( + &self, + image: &mut DurableSubAgentStore, + value: &PersistedSubAgentState, + terminal: Option<&str>, + coordination: bool, + generation_prepare: Option<&str>, + ) -> Result<()> { + value + .validate_worker_origins() + .map_err(anyhow::Error::msg)?; + if let Some(id) = terminal { + let candidate = value + .workers + .iter() + .find(|worker| worker.spec.worker_id == id) + .ok_or_else(|| anyhow!("terminal candidate is missing its worker"))?; + anyhow::ensure!( + candidate.status.is_terminal(), + "terminal candidate is not terminal" + ); + let stable_agent = |agent: Option<&PersistedSubAgent>| -> Result> { + agent + .map(|agent| { + let mut value = serde_json::to_value(agent)?; + if let Some(value) = value.as_object_mut() { + value.remove("updated_at_ms"); + value.remove("duration_ms"); + } + Ok(value) + }) + .transpose() + }; + let stable_worker = |worker: &AgentWorkerRecord| -> Result { + let mut value = serde_json::to_value(worker)?; + if let Some(value) = value.as_object_mut() { + for key in [ + "usage", + "usage_source_fingerprints", + "missing_usage_sources", + "missing_usage_overflowed", + "legacy_unreported_usage", + "has_unreported_usage", + "updated_at_ms", + ] { + value.remove(key); + } + } + Ok(value) + }; + if let Some(existing) = image + .state + .workers + .iter() + .find(|worker| worker.spec.worker_id == id) + && stable_worker(existing)? == stable_worker(candidate)? + && stable_agent(image.state.agents.iter().find(|agent| agent.id == id))? + == stable_agent(value.agents.iter().find(|agent| agent.id == id))? + { + sync_subagent_store_directory( + self.path + .parent() + .ok_or_else(|| anyhow!("delegated store has no parent"))?, + )?; + return Ok(()); + } + } + if image + .writer_sequences + .get(&self.writer) + .is_some_and(|sequence| *sequence >= value.snapshot_sequence) + { + anyhow::ensure!( + terminal.is_none(), + "terminal candidate was superseded before durable commit; retain and inspect its winning result" + ); + return Ok(()); + } + let owned = self.owned.lock().clone(); + let mut selected = owned; + if let Some(id) = terminal { + selected.insert(id.to_string()); + } + if let Some(id) = generation_prepare { + anyhow::ensure!( + coordination && terminal.is_none(), + "invalid generation preparation publication" + ); + selected.insert(id.to_string()); + } + for agent in &value.agents { + if agent.session_boot_id == self.writer + || !image.state.agents.iter().any(|saved| saved.id == agent.id) + && !image + .state + .workers + .iter() + .any(|saved| saved.spec.worker_id == agent.id) + { + selected.insert(agent.id.clone()); + } + } + let mut next = image.clone(); + for id in selected { + let preparing = generation_prepare == Some(id.as_str()); + let retained_writer = next + .record_writers + .get(&id) + .cloned() + .filter(|writer| preparing && self.writer_is_live(writer)); + if let Some(writer) = next.record_writers.get(&id) + && writer != &self.writer + && self.writer_is_live(writer) + && !preparing + { + return Err(anyhow!( + "delegated worker {id} belongs to a live foreign writer" + )); + } + let incoming = value + .workers + .iter() + .find(|worker| worker.spec.worker_id == id); + let existing = next + .state + .workers + .iter() + .find(|worker| worker.spec.worker_id == id); + let generation = |worker: &AgentWorkerRecord| { + worker + .spec + .launch_manifest + .as_ref() + .map_or(0, |manifest| manifest.generation) + }; + if preparing { + let existing = existing + .cloned() + .map(normalize_worker_record) + .ok_or_else(|| anyhow!("prepared worker {id} has no durable record"))?; + anyhow::ensure!( + !existing.prepared_previous_attempt_unsettled, + "worker {id} still has an unsettled previous launch" + ); + let incoming = incoming + .ok_or_else(|| anyhow!("prepared worker {id} is absent from publication"))?; + anyhow::ensure!( + !next.state.agents.iter().any(|agent| agent.id == id) + && !value.agents.iter().any(|agent| agent.id == id), + "generation preparation requires a headless worker" + ); + let mut expected_spec = existing.spec.clone(); + let manifest = expected_spec + .launch_manifest + .as_mut() + .ok_or_else(|| anyhow!("prepared worker {id} has no launch manifest"))?; + manifest.generation = manifest + .generation + .checked_add(1) + .ok_or_else(|| anyhow!("worker {id} launch generation is exhausted"))?; + anyhow::ensure!( + incoming.spec == expected_spec, + "generation preparation changed the worker's exact launch authority" + ); + let event = incoming + .events + .back() + .ok_or_else(|| anyhow!("prepared worker {id} has no event receipt"))?; + let latest_seq = next + .state + .workers + .iter() + .flat_map(|record| &record.events) + .map(|event| event.seq) + .max() + .unwrap_or(0); + anyhow::ensure!( + event.seq > latest_seq, + "generation preparation has a stale event sequence" + ); + let mut sequence = event + .seq + .checked_sub(1) + .ok_or_else(|| anyhow!("invalid generation event sequence"))?; + let expected = + existing.prepared_attempt(expected_spec, &mut sequence, incoming.updated_at_ms); + anyhow::ensure!( + serde_json::to_value(&expected)? == serde_json::to_value(incoming)?, + "generation preparation changed unrelated worker receipts" + ); + anyhow::ensure!( + next.state + .worker_origins + .as_ref() + .and_then(|origins| origins.get(&id)) + == value + .worker_origins + .as_ref() + .and_then(|origins| origins.get(&id)), + "generation preparation changed worker provenance" + ); + } + if let (Some(existing), Some(incoming)) = (existing, incoming) { + let resolving_recovery = existing.recovery_interrupted + && !incoming.recovery_interrupted + && incoming.status.is_terminal(); + anyhow::ensure!( + generation(incoming) >= generation(existing), + "delegated worker {id} has a newer durable launch generation" + ); + if existing.status.is_terminal() + && incoming.status.is_terminal() + && generation(incoming) == generation(existing) + && !resolving_recovery + { + anyhow::ensure!( + existing.status == incoming.status + && existing.result_summary == incoming.result_summary + && existing.error == incoming.error + && existing.completed_at_ms == incoming.completed_at_ms, + "delegated worker {id} already has a different winning terminal outcome" + ); + } + if terminal == Some(id.as_str()) + && existing.status.is_terminal() + && generation(incoming) == generation(existing) + && !resolving_recovery + { + return Err(anyhow!( + "delegated worker {id} already has its winning durable terminal receipt" + )); + } + } + if let (Some(existing), Some(incoming)) = (existing, incoming) + && existing.status.is_terminal() + && !incoming.status.is_terminal() + && generation(incoming) <= generation(existing) + { + continue; + } + next.state.agents.retain(|agent| agent.id != id); + next.state + .workers + .retain(|worker| worker.spec.worker_id != id); + if let Some(agent) = value.agents.iter().find(|agent| agent.id == id) { + next.state.agents.push(agent.clone()); + } + if let Some(worker) = incoming { + next.state.workers.push(worker.clone()); + next.record_writers.insert( + id.clone(), + retained_writer.unwrap_or_else(|| self.writer.clone()), + ); + } else if value.agents.iter().any(|agent| agent.id == id) { + next.record_writers.insert(id.clone(), self.writer.clone()); + } else { + next.record_writers.remove(&id); + } + let origins = next.state.worker_origins.get_or_insert_with(HashMap::new); + if let Some(origin) = value + .worker_origins + .as_ref() + .and_then(|origins| origins.get(&id)) + { + origins.insert(id.clone(), origin.clone()); + } else if let Some(worker) = incoming { + origins.insert( + id.clone(), + WorkerOrigin::legacy(worker.owner_session_id.clone()), + ); + } else { + origins.remove(&id); + } + next.state + .resume_targets + .retain(|source, target| source != &id && target != &id); + for (source, target) in &value.resume_targets { + if source == &id || target == &id { + next.state + .resume_targets + .insert(source.clone(), target.clone()); + } + } + } + if coordination { + next.state.coordination = value.coordination.clone(); + } + let origin_schema = next.state.schema_version.max(value.schema_version); + next.state.schema_version = origin_schema; + let ids = next + .state + .workers + .iter() + .map(|worker| worker.spec.worker_id.clone()) + .collect::>(); + next.state + .worker_origins + .get_or_insert_with(HashMap::new) + .retain(|id, _| ids.contains(id)); + for worker in &next.state.workers { + next.state + .worker_origins + .as_mut() + .unwrap() + .entry(worker.spec.worker_id.clone()) + .or_insert_with(|| WorkerOrigin::legacy(worker.owner_session_id.clone())); + } + if origin_schema == 1 { + next.state.worker_origins = None; + } + next.state.agents.sort_by(|a, b| a.id.cmp(&b.id)); + next.state + .workers + .sort_by(|a, b| a.spec.worker_id.cmp(&b.spec.worker_id)); + let retained_writers = next + .record_writers + .values() + .cloned() + .collect::>(); + next.writer_sequences + .retain(|writer, _| retained_writers.contains(writer) || self.writer_is_live(writer)); + next.writer_sequences + .insert(self.writer.clone(), value.snapshot_sequence); + next.state.snapshot_sequence = next.state.snapshot_sequence.max(value.snapshot_sequence); + next.revision = next + .revision + .checked_add(1) + .ok_or_else(|| anyhow!("delegated store revision exhausted"))?; + next.validate()?; + write_json_atomic(&self.state_root, &self.path, &next)?; + self.revision + .store(next.revision, std::sync::atomic::Ordering::Release); + self.published_sequence.store( + value.snapshot_sequence, + std::sync::atomic::Ordering::Release, + ); + *self.owned.lock() = next + .record_writers + .iter() + .filter(|(_, writer)| *writer == &self.writer) + .map(|(id, _)| id.clone()) + .collect(); + *image = next; + Ok(()) + } +} + +fn write_json_atomic(state_root: &Path, path: &Path, value: &DurableSubAgentStore) -> Result<()> { let state_root = normalize_subagent_workspace(state_root); reject_state_path_symlinks(&state_root, path)?; if let Some(parent) = path.parent() { fs::create_dir_all(parent)?; } - let payload = serde_json::to_string_pretty(value)?; - let publish_sequences = - STATE_PUBLISH_SEQUENCES.get_or_init(|| parking_lot::Mutex::new(HashMap::new())); - let mut published = publish_sequences.lock(); - if published - .get(path) - .is_some_and(|sequence| *sequence > value.snapshot_sequence) - { - return Ok(()); + let payload = serde_json::to_vec_pretty(value)?; + anyhow::ensure!( + payload.len() as u64 <= MAX_SUBAGENT_STATE_BYTES, + "delegated store exceeds the 16 MiB limit" + ); + codewhale_util::utils::write_atomic(path, &payload)?; + if let Some(parent) = path.parent() { + sync_subagent_store_directory(parent)?; } - codewhale_util::utils::write_atomic(path, payload.as_bytes())?; - published.insert(path.to_path_buf(), value.snapshot_sequence); Ok(()) } @@ -12275,8 +14121,13 @@ pub fn new_shared_subagent_manager_with_state_root_and_timeout( let state_path = match default_state_path(&state_root) { Ok(path) => Some(path), Err(err) => { - tracing::warn!(target: "subagent", ?err, "failed to resolve sub-agent state path"); - None + tracing::warn!(target: "subagent", ?err, "failed to resolve sub-agent state path; retained path requires validated admission"); + Some( + state_root + .join(".codewhale") + .join("state") + .join(SUBAGENT_STATE_FILE), + ) } }; let manager = if state_root == workspace { @@ -12306,9 +14157,8 @@ pub fn new_shared_subagent_manager_with_state_root_and_timeout( // `persist_state`/`persist_state_synchronously` still checks this // manager's coordinator admission before publishing. if let Err(err) = manager.load_state() { - // Routed through tracing instead of stderr — see comment in - // `persist_state_best_effort` above. - tracing::warn!(target: "subagent", ?err, "failed to load sub-agent state"); + manager.coordination_snapshot_read_only = true; + tracing::warn!(target: "subagent", ?err, "failed to load sub-agent state; durable writes remain refused"); } Arc::new(RwLock::new(manager)) } @@ -13164,8 +15014,16 @@ async fn inspect_agent_from_input( &context.state_namespace, ) .await; + let owner = manager.clone(); + let session = context.state_namespace.clone(); + codewhale_runtime::runtime_control::owner_work(move || { + owner + .blocking_write() + .cleanup_for_session(&session, COMPLETED_AGENT_RETENTION) + }) + .await + .map_err(|error| ToolError::execution_failed(error.to_string()))?; let mut manager = manager.write().await; - manager.cleanup_for_session(&context.state_namespace, COMPLETED_AGENT_RETENTION); let payload = if let Some(agent_ref) = agent_ref.as_ref() { let source = manager .resolve_agent_ref_for_session(&context.state_namespace, agent_ref) @@ -13240,8 +15098,16 @@ async fn inspect_agent_from_input( &context.state_namespace, ) .await; + let owner = manager.clone(); + let session = context.state_namespace.clone(); + codewhale_runtime::runtime_control::owner_work(move || { + owner + .blocking_write() + .cleanup_for_session(&session, COMPLETED_AGENT_RETENTION) + }) + .await + .map_err(|error| ToolError::execution_failed(error.to_string()))?; let mut manager = manager.write().await; - manager.cleanup_for_session(&context.state_namespace, COMPLETED_AGENT_RETENTION); let evicted_ids = manager.drain_pending_handle_evictions_for_session(&context.state_namespace); let source = manager @@ -13319,23 +15185,28 @@ async fn cancel_agent_from_input( caller_agent_id: Option<&str>, ) -> Result { let agent_ref = parse_agent_ref(input)?.ok_or_else(|| ToolError::missing_field("agent_id"))?; - let (snapshot, worker_record) = { - let mut manager = manager.write().await; + let owner = manager.clone(); + let session = context.state_namespace.clone(); + let caller_agent_id = caller_agent_id.map(str::to_string); + let (snapshot, worker_record) = codewhale_runtime::runtime_control::owner_work(move || { + let mut manager = owner.blocking_write(); manager .continuation_target_for_caller( - &context.state_namespace, + &session, &agent_ref, - caller_agent_id, + caller_agent_id.as_deref(), "agent/cancel", ) - .map_err(|err| ToolError::invalid_input(err.to_string()))?; - let snapshot = manager - .cancel_agent_for_session(&context.state_namespace, &agent_ref) - .map_err(|err| ToolError::invalid_input(err.to_string()))?; - let worker_record = - manager.get_worker_record_for_session(&context.state_namespace, &snapshot.agent_id); - (snapshot, worker_record) - }; + .map_err(|error| anyhow::Error::new(ToolError::invalid_input(error.to_string())))?; + let snapshot = manager.cancel_agent_for_session(&session, &agent_ref)?; + let worker_record = manager.get_worker_record_for_session(&session, &snapshot.agent_id); + Ok((snapshot, worker_record)) + }) + .await + .map_err(|error| match error.downcast::() { + Ok(error) => error, + Err(error) => ToolError::execution_failed(error.to_string()), + })?; let snapshot = settle_requested_child(&manager, snapshot).await; let snapshot = preserve_cancelled_work(&manager, snapshot).await; let worker_record = manager @@ -13744,7 +15615,7 @@ async fn spawn_subagent_from_input( }) .await .map_err(|error| ToolError::execution_failed(error.to_string()))?; - let (child_workspace, mut pending_worktree) = { + let (child_workspace, pending_worktree) = { let parent_workspace = runtime.context.workspace.clone(); let cwd = spawn_request.cwd.clone(); let worktree = spawn_request.worktree.clone(); @@ -13930,9 +15801,6 @@ async fn spawn_subagent_from_input( let resident_lease = resident_context .as_ref() .map(|resident| (resident.lease_key.clone(), resident.display_path.clone())); - if let Some((lease_key, display_path)) = resident_lease.as_ref() { - reserve_resident_lease(lease_key, display_path)?; - } // #6210: fingerprint the delivery baseline BEFORE the manager write // lock. The child workspace is final here (worktree creation already // ran); write capability resolves under the lock, so capture assuming @@ -13948,8 +15816,6 @@ async fn spawn_subagent_from_input( } else { None }; - let mut manager_guard = manager.write().await; - let spawn_options = SubAgentSpawnOptions { origin: Some(captured_origin), name: spawn_request.session_name.clone(), @@ -13980,19 +15846,15 @@ async fn spawn_subagent_from_input( preserve_runtime_profile: None, workflow_run_id: workflow_identity.map(|identity| identity.workflow_run_id.clone()), }; - let result = match activation.as_ref() { - None => manager_guard.spawn_background_with_assignment_options( - Arc::clone(&manager), - child_runtime, - spawn_request.agent_type, - effective_prompt, - spawn_request.assignment, - spawn_request.allowed_tools, - spawn_options, - precomputed_delivery_evidence, - ), - Some(request) => manager_guard - .prepare_background_with_assignment_options( + let spawn_owner = manager.clone(); + let result = codewhale_runtime::runtime_control::owner_work(move || { + let mut pending_worktree = pending_worktree; + if let Some((lease_key, display_path)) = resident_lease.as_ref() { + reserve_resident_lease(lease_key, display_path).map_err(anyhow::Error::new)?; + } + let mut manager_guard = spawn_owner.blocking_write(); + let result = match activation.as_ref() { + None => manager_guard.spawn_background_with_assignment_options( Arc::clone(&manager), child_runtime, spawn_request.agent_type, @@ -14001,35 +15863,51 @@ async fn spawn_subagent_from_input( spawn_request.allowed_tools, spawn_options, precomputed_delivery_evidence, - Some(request.public_child_id.clone()), - ) - .and_then(|prepared| { - manager_guard.commit_prepared_child(prepared, request.configuration.clone()) - }) - .and_then(|(result, context)| { - let context = - context.ok_or_else(|| anyhow!("child activation was not recorded"))?; - *request.launched.lock().expect("child activation lock") = - Some((Arc::clone(&manager), context)); + ), + Some(request) => manager_guard + .prepare_background_with_assignment_options( + Arc::clone(&manager), + child_runtime, + spawn_request.agent_type, + effective_prompt, + spawn_request.assignment, + spawn_request.allowed_tools, + spawn_options, + precomputed_delivery_evidence, + Some(request.public_child_id.clone()), + ) + .and_then(|prepared| { + manager_guard.commit_prepared_child(prepared, request.configuration.clone()) + }) + .and_then(|(result, context)| { + let context = + context.ok_or_else(|| anyhow!("child activation was not recorded"))?; + *request.launched.lock().expect("child activation lock") = + Some((Arc::clone(&manager), context)); + Ok(result) + }), + }; + match result { + Ok(result) => { + if let Some((lease_key, _)) = resident_lease.as_ref() { + commit_resident_lease(lease_key, &result.agent_id); + } + pending_worktree.0 = None; Ok(result) - }), - }; - let result = match result { - Ok(result) => result, - Err(error) => { - if let Some((lease_key, _)) = resident_lease.as_ref() { - rollback_pending_resident_lease(lease_key); } - return Err(ToolError::execution_failed(format!( - "Failed to spawn sub-agent: {error}" - ))); + Err(error) => { + if let Some((lease_key, _)) = resident_lease.as_ref() { + rollback_pending_resident_lease(lease_key); + } + Err(error) + } } - }; - - if let Some((lease_key, _)) = resident_lease.as_ref() { - commit_resident_lease(lease_key, &result.agent_id); - } - pending_worktree.0 = None; + }) + .await + .map_err(|error| match error.downcast::() { + Ok(error) => error, + Err(error) => ToolError::execution_failed(format!("Failed to spawn sub-agent: {error}")), + })?; Ok((result, spawn_metadata)) } @@ -14647,9 +16525,16 @@ async fn run_subagent_task(mut task: SubAgentTask) { let agent_id = task.agent_id.clone(); let manager_handle = task.manager_handle.clone(); let activation = task.runtime.activation.clone(); - let foreground = activation - .as_ref() - .and_then(|_| task._foreground_child_registration.take()); + let foreground = if activation.is_some() { + task._foreground_child_registration.take() + } else { + if let Some(foreground) = task._foreground_child_registration.take() + && let Some(agent) = manager_handle.write().await.agents.get_mut(&agent_id) + { + agent.unsettled_foreground = Some(foreground); + } + None + }; let outcome = std::panic::AssertUnwindSafe(supervise_subagent_task_body( manager_handle.clone(), agent_id.clone(), @@ -14675,6 +16560,52 @@ async fn run_subagent_task(mut task: SubAgentTask) { } } +async fn finish_terminal_result_owned( + manager: SharedSubAgentManager, + result: SubAgentResult, +) -> Result { + let delivery = { + let mut held = manager.write().await; + let agent_id = &result.agent_id; + if result.status != SubAgentStatus::Running + && held.claim_terminal_delivery(agent_id) + && let Some(agent) = held.agents.get_mut(agent_id) + { + agent.pending_terminal_result = Some(PendingTerminalResult { + result: result.clone(), + worker: None, + prepared: false, + }); + } + held.agents + .get(agent_id) + .and_then(|agent| agent.terminal_delivery.as_ref()) + .and_then(|delivery| delivery.event_delivery.clone()) + .map(|owner| { + owner + .pending + .fetch_add(1, std::sync::atomic::Ordering::AcqRel); + PendingTerminalEvent { + owner, + delivered: false, + } + }) + }; + codewhale_runtime::runtime_control::owner_work(move || { + let mut delivery = delivery; + let agent_id = result.agent_id.clone(); + let mut manager = manager.blocking_write(); + let committed = manager.with_store_transaction(|held| { + held.finish_terminal_result(&agent_id, result, false, true) + }); + if let Some(delivery) = delivery.as_mut() { + delivery.delivered = committed.is_ok(); + } + committed + }) + .await +} + async fn supervise_subagent_task_body( manager_handle: SharedSubAgentManager, agent_id: String, @@ -14703,12 +16634,11 @@ async fn supervise_subagent_task_body( result.needs_input = None; // Verification precedes the write lock like the natural epilogue (#6210). ensure_worker_delivery_verified(&manager_handle, &agent_id, &result).await; - { - let mut manager = manager_handle.write().await; - // Arbitrated exactly like the natural terminal commit: when a - // cancel or another terminal outcome already won, this is a - // no-op rather than a second result. - manager.finish_terminal_result(&agent_id, result, false, true); + // Arbitrated exactly like the natural terminal commit: when a + // cancel or another terminal outcome already won, this is a + // no-op rather than a second result. + if let Err(error) = finish_terminal_result_owned(manager_handle, result).await { + tracing::error!(target: "subagent", %agent_id, ?error, "panicked child terminal receipt is uncertain"); } std::panic::resume_unwind(panic); } @@ -15108,10 +17038,17 @@ async fn run_subagent_task_inner(mut task: SubAgentTask) { } } - let turn_end_parking = task - ._foreground_child_registration - .as_ref() - .map(ForegroundChildRegistration::parking_signal); + let turn_end_parking = if let Some(foreground) = task._foreground_child_registration.as_ref() { + Some(foreground.parking_signal()) + } else { + task.manager_handle + .read() + .await + .agents + .get(&task.agent_id) + .and_then(|agent| agent.unsettled_foreground.as_ref()) + .map(ForegroundChildRegistration::parking_signal) + }; // Every inner stage bounds itself by the owner-selected `deadline`, // so the inner run can always produce its own receipt: the specific stop // cause and preserved usage. This outer backstop @@ -15230,7 +17167,9 @@ async fn run_subagent_task_inner(mut task: SubAgentTask) { ) }); agent.unsettled_launch = Some(std::mem::take(&mut _launch_permits)); - agent.unsettled_foreground = task._foreground_child_registration.take(); + if let Some(foreground) = task._foreground_child_registration.take() { + agent.unsettled_foreground = Some(foreground); + } } if let Some(record) = manager.worker_records.get_mut(&agent_id) { record.error = Some(note.clone()); @@ -15323,16 +17262,28 @@ async fn run_subagent_task_inner(mut task: SubAgentTask) { // coordination interrupt, and stale cleanup — arbitrates and publishes // through `finish_terminal_result`. Cancellation that already won leaves // this late epilogue with no claim and therefore no duplicate fan-in. - let terminal_committed = { + { let mut manager = task.manager_handle.write().await; - manager.finish_terminal_result(&agent_id, terminal, false, true) - }; - if !terminal_committed { - tracing::debug!( + if let Some(agent) = manager.agents.get_mut(&agent_id) { + if !_launch_permits.is_empty() { + agent.unsettled_launch = Some(std::mem::take(&mut _launch_permits)); + } + if let Some(foreground) = task._foreground_child_registration.take() { + agent.unsettled_foreground = Some(foreground); + } + } + } + match finish_terminal_result_owned(task.manager_handle.clone(), terminal).await { + Ok(true) => {} + Ok(false) => tracing::debug!( target: "subagent", agent_id = %agent_id, "suppressing late task completion after another terminal outcome won" - ); + ), + Err(error) => tracing::error!( + target: "subagent", agent_id = %agent_id, ?error, + "child terminal receipt is uncertain; ownership retained" + ), } } diff --git a/crates/engine/src/tools/subagent/tests.rs b/crates/engine/src/tools/subagent/tests.rs index 60be50653..4192edea9 100644 --- a/crates/engine/src/tools/subagent/tests.rs +++ b/crates/engine/src/tools/subagent/tests.rs @@ -1008,7 +1008,9 @@ fn persisted_subagent_state_has_bounded_serialized_size() { // Sanity-check: file is valid JSON and contains expected records. let parsed: serde_json::Value = serde_json::from_slice(&serialized).expect("persisted state is valid JSON"); - let workers = parsed["workers"].as_array().expect("workers array"); + let workers = parsed["state"]["workers"] + .as_array() + .expect("workers array"); assert_eq!( workers.len(), 10, @@ -1023,31 +1025,36 @@ fn coordination_ledger_persists_and_replays_bounded_contracts() { let mut manager = SubAgentManager::new(tmp.path().to_path_buf(), 4).with_state_path(state_path.clone()); manager - .coordination - .register_claim( - WriteScopeClaim { - owner: "agent_writer".into(), - roots: vec!["src".into()], - exact_files: vec!["Cargo.toml".into()], - contracts: vec!["public-api".into()], - }, - false, - |_| false, - ) - .expect("claim"); - manager - .record_coordination_decision(DecisionRecord { - decision_id: "decision_storage".into(), - subject: "storage".into(), - status: DecisionStatus::Accepted, - owner: "agent_writer".into(), - scope: vec!["router".into()], - constraints: vec!["bounded".into()], - evidence_handles: vec!["test:coordination".into()], - version: 1, - sequence: 0, + .with_store_transaction(|manager| { + manager + .coordination + .register_claim( + WriteScopeClaim { + owner: "agent_writer".into(), + roots: vec!["src".into()], + exact_files: vec!["Cargo.toml".into()], + contracts: vec!["public-api".into()], + }, + false, + |_| false, + ) + .expect("claim"); + manager + .record_coordination_decision(DecisionRecord { + decision_id: "decision_storage".into(), + subject: "storage".into(), + status: DecisionStatus::Accepted, + owner: "agent_writer".into(), + scope: vec!["router".into()], + constraints: vec!["bounded".into()], + evidence_handles: vec!["test:coordination".into()], + version: 1, + sequence: 0, + }) + .expect("decision"); + Ok(()) }) - .expect("decision"); + .unwrap(); manager.persist_state().unwrap().join().unwrap(); let mut loaded = SubAgentManager::new(tmp.path().to_path_buf(), 4).with_state_path(state_path); @@ -1411,7 +1418,12 @@ fn session_close_finalizes_live_fleet_and_releases_write_claims() { assert!(overlap.contains("worker-a"), "{overlap}"); // Session close finalizes both owners and releases their claims. - assert!(manager.finalize_session_close() > 0); + assert!( + manager + .finalize_session_close() + .expect("durable worker operation") + > 0 + ); assert!(!manager.active_coordination_owners().contains(&legacy_id)); assert!(!manager.active_coordination_owners().contains("worker-a")); assert!(manager.coordination.write_claims.is_empty()); @@ -1427,7 +1439,12 @@ fn session_close_finalizes_live_fleet_and_releases_write_claims() { // Idempotent: a second close pass has no live fleet to finalize and // must not release (or re-own) anything. - assert_eq!(manager.finalize_session_close(), 0); + assert_eq!( + manager + .finalize_session_close() + .expect("durable worker operation"), + 0 + ); assert!(manager.coordination.write_claims.is_empty()); manager .preflight_worker_coordination(&make_write_worker_spec( @@ -1650,7 +1667,9 @@ fn a_second_session_without_the_lock_keeps_its_running_agents_live() { let agent_id = agent.id.clone(); contender.agents.insert(agent_id.clone(), agent); - let cancelled = contender.cleanup(Duration::from_secs(3600)); + let cancelled = contender + .cleanup(Duration::from_secs(3600)) + .expect("durable worker operation"); assert_eq!( cancelled, 0, "lock loss alone must not terminalize anything" @@ -1780,7 +1799,9 @@ fn cleanup_terminalizes_orphans_on_heartbeat_evidence_not_on_lock_ownership() { let fresh = insert_orphan(&mut contender, "fresh-orphan", Duration::from_secs(1)); let stale = insert_orphan(&mut contender, "stale-orphan", Duration::from_secs(300)); - let cancelled = contender.cleanup(Duration::from_secs(3600)); + let cancelled = contender + .cleanup(Duration::from_secs(3600)) + .expect("durable worker operation"); assert_eq!( cancelled, 1, "only the agent with stale heartbeat evidence terminalizes" @@ -7420,7 +7441,9 @@ async fn late_completion_does_not_overwrite_cancelled_outcome() { late.status = SubAgentStatus::Completed; late.result = Some("late success".to_string()); assert!( - !manager.update_from_result(&agent_id, late), + !manager + .update_from_result(&agent_id, late) + .expect("durable worker operation"), "late completion must lose the terminal transition" ); @@ -7504,7 +7527,11 @@ async fn completion_claim_preserves_running_gate_and_excludes_late_cancel() { let mut result = manager.get_result(&agent_id).unwrap(); result.status = SubAgentStatus::Completed; result.result = Some("done".to_string()); - assert!(manager.update_from_result(&agent_id, result)); + assert!( + manager + .update_from_result(&agent_id, result) + .expect("durable worker operation") + ); assert_eq!( manager.get_result(&agent_id).unwrap().status, SubAgentStatus::Completed @@ -11188,7 +11215,12 @@ async fn cleanup_auto_cancels_stale_running_agent_and_releases_slot() { 0, "stale running agents must not keep the concurrency slot occupied" ); - assert_eq!(manager.cleanup(Duration::from_secs(60 * 60)), 1); + assert_eq!( + manager + .cleanup(Duration::from_secs(60 * 60)) + .expect("durable worker operation"), + 1 + ); let snapshot = manager .get_result(&agent_id) @@ -11302,7 +11334,12 @@ async fn cleanup_keeps_recent_running_agent() { manager.agents.insert(agent_id.clone(), agent); assert_eq!(manager.running_count(), 1); - assert_eq!(manager.cleanup(Duration::from_secs(60 * 60)), 0); + assert_eq!( + manager + .cleanup(Duration::from_secs(60 * 60)) + .expect("durable worker operation"), + 0 + ); assert_eq!( manager.get_result(&agent_id).expect("agent").status, SubAgentStatus::Running @@ -11351,7 +11388,12 @@ async fn touch_refreshes_stale_running_agent_heartbeat() { assert_eq!(manager.running_count(), 0); assert!(manager.touch(&agent_id)); assert_eq!(manager.running_count(), 1); - assert_eq!(manager.cleanup(Duration::from_secs(60 * 60)), 0); + assert_eq!( + manager + .cleanup(Duration::from_secs(60 * 60)) + .expect("durable worker operation"), + 0 + ); manager .agents .get_mut(&agent_id) @@ -11388,6 +11430,8 @@ fn test_persist_and_reload_marks_running_agent_as_interrupted() { .join() .expect("persist thread"); + drop(manager); + let mut reloaded = SubAgentManager::new(workspace, 2) .with_state_path(default_state_path(tmp.path()).expect("default state path")); reloaded.load_state().expect("load state"); @@ -11435,13 +11479,13 @@ fn generated_whale_name_is_not_persisted_or_replayed_on_load() { serde_json::from_str(&std::fs::read_to_string(&state_path).expect("read persisted state")) .expect("parse persisted state"); assert!( - persisted["agents"][0].get("nickname").is_none(), + persisted["state"]["agents"][0].get("nickname").is_none(), "generated locale text is not durable identity" ); // Recreate a pre-fix state file whose generated display came from a // Japanese session. Loading under a later session must discard it. - persisted["agents"][0]["nickname"] = json!(generated); + persisted["state"]["agents"][0]["nickname"] = json!(generated); std::fs::write( &state_path, serde_json::to_string_pretty(&persisted).expect("serialize legacy state"), @@ -11493,7 +11537,7 @@ fn explicit_nonmatching_whale_word_is_persisted_and_loaded() { serde_json::from_str(&std::fs::read_to_string(&state_path).expect("read persisted state")) .expect("parse persisted state"); assert_eq!( - persisted["agents"][0]["nickname"], + persisted["state"]["agents"][0]["nickname"], json!(explicit_whale), "the explicit whale-word nickname remains durable" ); @@ -14257,7 +14301,9 @@ fn park_child_at_turn_end(manager: &mut SubAgentManager, agent_id: &str) { terminal.checkpoint = checkpoint; terminal.needs_input = needs_input; assert!( - manager.finish_terminal_result(agent_id, terminal, true, false), + manager + .finish_terminal_result(agent_id, terminal, true, false) + .expect("durable worker operation"), "the parked projection must own the terminal transition" ); } @@ -15636,10 +15682,13 @@ fn persisted_advisory_assignment_roles_replay_and_repersist_as_consultant() { ) .expect("parse canonical state"); assert_eq!( - repersisted["agents"][0]["assignment"]["role"], + repersisted["state"]["agents"][0]["assignment"]["role"], + json!("advisor") + ); + assert_eq!( + repersisted["state"]["agents"][0]["agent_type"], json!("advisor") ); - assert_eq!(repersisted["agents"][0]["agent_type"], json!("advisor")); } } @@ -16342,7 +16391,9 @@ fn ownerless_legacy_rows_fail_closed_and_scoped_close_preserves_foreign_workers( assign_test_session_owner(&mut manager, &agent_b, "session-b"); assert_eq!( - manager.finalize_session_close_for_session("session-a"), + manager + .finalize_session_close_for_session("session-a") + .expect("durable worker operation"), 1, "paired agent and worker record are one finalized worker" ); @@ -17110,12 +17161,14 @@ async fn run_subagent_task_claims_before_delivery_and_then_finalizes() { .denied_tools .contains(&"write".to_string()) ); - held.cleanup(Duration::ZERO); + held.cleanup(Duration::ZERO) + .expect("durable worker operation"); assert!(held.worker_records.contains_key(&identity.child_id)); drop(held); - let persisted: PersistedSubAgentState = + let persisted: DurableSubAgentStore = serde_json::from_slice(&std::fs::read(&state_path).expect("persisted file")) .expect("persisted worker state"); + let persisted = persisted.state; assert_eq!( persisted .workers @@ -17155,9 +17208,9 @@ async fn run_subagent_task_claims_before_delivery_and_then_finalizes() { .expect("activation state") .clone(), }; - let blocked_path = workspace.path().join("blocked-worker-store"); - std::fs::create_dir(&blocked_path).expect("block publication with a directory"); - manager.write().await.state_path = Some(blocked_path); + let retained_path = state_path.with_extension("retained-before-failure"); + std::fs::rename(&state_path, &retained_path).expect("retain admitted store bytes"); + std::fs::create_dir(&state_path).expect("block the admitted publication destination"); drop(prepared); let error = tokio::time::timeout(Duration::from_secs(30), failed.settled()) .await @@ -17167,10 +17220,12 @@ async fn run_subagent_task_claims_before_delivery_and_then_finalizes() { assert!(failed.accepted().await.is_err()); { let mut held = manager.write().await; - held.cleanup(Duration::ZERO); + held.cleanup(Duration::ZERO) + .expect_err("blocked durable worker store"); assert!(held.agents.contains_key(&identity.child_id)); assert!(held.worker_records.contains_key(&identity.child_id)); - held.state_path = Some(state_path); + std::fs::remove_dir(&state_path).expect("remove the fixture obstruction"); + std::fs::rename(&retained_path, &state_path).expect("restore the admitted store bytes"); } let retried = persist_child_settlement(&manager, &identity) .await @@ -19866,7 +19921,9 @@ fn cleanup_evicts_stale_terminal_worker_records_and_keeps_live_ones() { } // One-hour retention matches COMPLETED_AGENT_RETENTION used by cleanup callers. - let auto_cancelled = manager.cleanup(Duration::from_secs(60 * 60)); + let auto_cancelled = manager + .cleanup(Duration::from_secs(60 * 60)) + .expect("durable worker operation"); assert_eq!(auto_cancelled, 0); assert!( @@ -19919,7 +19976,9 @@ fn cleanup_removes_complete_transcript_after_worker_retention_expires() { .expect("write transcript artifact"); assert!(artifact.exists()); - manager.cleanup(Duration::from_secs(60 * 60)); + manager + .cleanup(Duration::from_secs(60 * 60)) + .expect("durable worker operation"); assert!(manager.get_worker_record(agent_id).is_none()); assert!( @@ -19942,7 +20001,9 @@ fn cleanup_due_gates_write_locked_cleanup_to_a_bounded_cadence() { "a never-cleaned manager should be due" ); - manager.cleanup(Duration::from_secs(3600)); + manager + .cleanup(Duration::from_secs(3600)) + .expect("durable worker operation"); assert!( !manager.cleanup_due(Duration::from_secs(3600)), "immediately after cleanup it should not be due again within the interval" @@ -20329,7 +20390,7 @@ fn write_json_atomic_survives_concurrent_writers() { snapshot_sequence: i + 1, ..PersistedSubAgentState::default() }; - let _ = write_json_atomic(&ws, &p, &payload); + let _ = write_json_atomic(&ws, &p, &DurableSubAgentStore::from_legacy(payload)); })); } for h in handles { @@ -20339,7 +20400,7 @@ fn write_json_atomic_survives_concurrent_writers() { let contents = std::fs::read_to_string(&*path).expect("read state.json"); let parsed: serde_json::Value = serde_json::from_str(&contents).expect("state.json must be complete/valid JSON"); - assert!(parsed.get("snapshot_sequence").is_some()); + assert!(parsed["state"].get("snapshot_sequence").is_some()); // No stray temp files left behind. let leftover: Vec<_> = std::fs::read_dir(path.parent().unwrap()) .expect("read subagents dir") @@ -21298,7 +21359,12 @@ async fn tracked_running_background_shell_keeps_its_owner_off_the_heartbeat_reap 1, "a worker waiting on a tracked background shell must survive the heartbeat" ); - assert_eq!(manager.cleanup(Duration::from_secs(60 * 60)), 0); + assert_eq!( + manager + .cleanup(Duration::from_secs(60 * 60)) + .expect("durable worker operation"), + 0 + ); // The converse: once the job stops running, nothing extends the heartbeat. { @@ -24232,7 +24298,11 @@ fn user_follow_up_to_completed_child_requires_a_runtime_to_resume() { let mut terminal = manager.get_result(&agent_id).expect("snapshot"); terminal.status = SubAgentStatus::Completed; terminal.result = Some("done".to_string()); - assert!(manager.finish_terminal_result(&agent_id, terminal, true, false)); + assert!( + manager + .finish_terminal_result(&agent_id, terminal, true, false) + .expect("durable worker operation") + ); let err = manager .continue_child_from_user(handle, None, &agent_id, "one more") .expect_err("no runtime, no resume"); @@ -25302,7 +25372,11 @@ mod child_permission_gate { let mut terminal = manager.get_result(&agent_id).expect("owned agent"); terminal.status = SubAgentStatus::Cancelled; terminal.needs_input = None; - assert!(manager.finish_terminal_result(&agent_id, terminal, false, true)); + assert!( + manager + .finish_terminal_result(&agent_id, terminal, false, true) + .expect("durable worker operation") + ); } }); let approval_id = next_child_approval_id(&mut rx).await; @@ -25463,7 +25537,12 @@ mod child_permission_gate { "held", ) .unwrap(); - assert!(manager.finalize_session_close_for_session("workspace") > 0); + assert!( + manager + .finalize_session_close_for_session("workspace") + .expect("durable worker operation") + > 0 + ); assert_eq!( manager.pending_child_approvals(), 0, diff --git a/crates/engine/src/tools/workflow/mod.rs b/crates/engine/src/tools/workflow/mod.rs index 726ae12f9..27101e431 100644 --- a/crates/engine/src/tools/workflow/mod.rs +++ b/crates/engine/src/tools/workflow/mod.rs @@ -3909,6 +3909,7 @@ struct SubAgentWorkflowDriver { owner_session_id: String, manager: SharedSubAgentManager, runtime: SubAgentRuntime, + owner_runtime: Option, parent_cancel_token: CancellationToken, state: Arc, completion_tx: mpsc::Sender, @@ -4007,6 +4008,7 @@ impl SubAgentWorkflowDriver { owner_session_id, manager, runtime, + owner_runtime: tokio::runtime::Handle::try_current().ok(), parent_cancel_token, state, completion_tx, @@ -4046,7 +4048,13 @@ impl SubAgentWorkflowDriver { if let Ok(mut permits) = self.spawn_permits.lock() { permits.clear(); } - cancel_child_agents(self.manager.clone(), ids); + cancel_child_agents( + self.manager.clone(), + self.owner_session_id.clone(), + ids, + self.owner_runtime.clone(), + self.runtime.event_tx.clone(), + ); if let Ok(mut state) = self.completion_state.lock() { for (_, waiter) in state.waiters.drain() { let _ = waiter.send(TaskCompletion::Cancelled); @@ -4892,7 +4900,22 @@ impl SubAgentWorkflowDriver { }; let task_id = result.result.agent_id.clone(); if let Err(err) = self.ensure_admission_open() { - let _ = self.manager.write().await.cancel_agent(&task_id); + let manager = self.manager.clone(); + let session = self.owner_session_id.clone(); + let target = task_id.clone(); + let cancelled = codewhale_runtime::runtime_control::owner_work(move || { + manager + .blocking_write() + .cancel_agent_for_session(&session, &target) + .map(|_| ()) + }) + .await; + let err = match cancelled { + Ok(()) => err, + Err(error) => DriverError::Rejected(format!( + "{err}; cancellation of child {task_id} is uncertain: {error}" + )), + }; if let Some(receipt) = &fleet_receipt { self.record_orphaned_fleet_receipt(receipt, &err.to_string()); } @@ -5700,27 +5723,56 @@ fn provider_usage_was_reported( .any(Option::is_some) } -fn cancel_child_agents(manager: SharedSubAgentManager, ids: Vec) { +fn cancel_child_agents( + manager: SharedSubAgentManager, + session: String, + ids: Vec, + owner_runtime: Option, + event_tx: Option>, +) { if ids.is_empty() { return; } - if let Ok(mut manager_guard) = manager.try_write() { - for id in ids { - let _ = manager_guard.cancel_agent(&id); - } - return; - } - if tokio::runtime::Handle::try_current().is_ok() { + if let Some(owner_runtime) = + owner_runtime.or_else(|| tokio::runtime::Handle::try_current().ok()) + { + let _runtime = owner_runtime.enter(); spawn_supervised( "workflow-cancel-children", std::panic::Location::caller(), async move { - let mut manager_guard = manager.write().await; - for id in ids { - let _ = manager_guard.cancel_agent(&id); + let cancelled = codewhale_runtime::runtime_control::owner_work(move || { + let mut manager = manager.blocking_write(); + let mut failure = None; + for id in ids { + if let Err(error) = manager.cancel_agent_for_session(&session, &id) { + failure.get_or_insert(error); + } + } + failure.map_or(Ok(()), Err) + }) + .await; + if let Err(error) = cancelled { + let message = format!("Workflow child cancellation is uncertain: {error}"); + tracing::error!(%message); + if let Some(event_tx) = event_tx { + let _ = event_tx + .send(Event::error( + codewhale_error_taxonomy::ErrorEnvelope::transient(message), + )) + .await; + } } }, ); + } else { + let message = "Workflow child cancellation is uncertain: no runtime is available"; + tracing::error!(%message); + if let Some(event_tx) = event_tx { + let _ = event_tx.try_send(Event::error( + codewhale_error_taxonomy::ErrorEnvelope::transient(message), + )); + } } } diff --git a/crates/runtime/src/runtime_control.rs b/crates/runtime/src/runtime_control.rs index b1339ecb7..dc609ed67 100644 --- a/crates/runtime/src/runtime_control.rs +++ b/crates/runtime/src/runtime_control.rs @@ -15,6 +15,8 @@ where T: Send + 'static, F: FnOnce() -> anyhow::Result + Send + 'static, { + #[cfg(any(test, feature = "test-support"))] + let env_ticket = crate::test_support::env_scope_ticket(); static SLOTS: std::sync::OnceLock> = std::sync::OnceLock::new(); let permit = SLOTS @@ -23,6 +25,8 @@ where .acquire_owned() .await?; tokio::task::spawn_blocking(move || { + #[cfg(any(test, feature = "test-support"))] + let _membership = crate::test_support::join_env_scope(env_ticket); let _permit = permit; work() }) diff --git a/crates/tui/CHANGELOG.md b/crates/tui/CHANGELOG.md index 3906d449e..b406daceb 100644 --- a/crates/tui/CHANGELOG.md +++ b/crates/tui/CHANGELOG.md @@ -160,6 +160,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- Worker coordination serializes durable admissions across processes, preserves foreign workers and winning completion receipts, and fences stale checkpoints. Fleet restart reservations retain uncertain prior processes until exact teardown is confirmed; final output, usage and the receipt publish together without duplicate accounting on retry. Retained transcripts follow committed ownership. + - JSONPath projections bound path length, traversal steps and intermediate matches before cloning results. Direct fetches share the same parser and enforce one aggregate serialized-field budget; oversized selections refuse with narrower-selection guidance. - Long tool output and diffs use bounded display windows and a byte-limited row cache. Retained details load verified byte ranges and label what search and copy include; explicit source copy keeps stored text with control-sequence cleanup. Oversized Unicode is shown safely without changing question answers; long answers retain their exact bytes in the existing owned output artifact before bounded model-context projection. New pager notices are translated across all 15 complete language packs. diff --git a/crates/tui/src/exec_agent.rs b/crates/tui/src/exec_agent.rs index 6d27618ff..a64d2369f 100644 --- a/crates/tui/src/exec_agent.rs +++ b/crates/tui/src/exec_agent.rs @@ -665,7 +665,7 @@ pub(crate) async fn run_exec_agent( .unwrap_or_else(codewhale_engine::tools::subagent::AdvisorConfig::disabled), }; - let engine_handle = spawn_engine(engine_config, &execution_config, None); + let engine_handle = spawn_engine(engine_config, &execution_config, None).await; // A restricted posture that resolves to no enforcing sandbox must reach // the person running exec, not only the model's posture line: policy-only // is accepted (read-only already fails closed in the shell tool), but it diff --git a/crates/tui/src/lib.rs b/crates/tui/src/lib.rs index b544bc450..96edc3388 100644 --- a/crates/tui/src/lib.rs +++ b/crates/tui/src/lib.rs @@ -3793,17 +3793,6 @@ async fn run_fleet_command(workspace: &Path, config: &Config, args: FleetArgs) - .active_provider_identity() .map_err(anyhow::Error::msg)?; - let max_subagents = config.max_subagents_for_provider(&identity); - let coordination_manager = - codewhale_engine::tools::subagent::new_shared_subagent_manager_with_timeout( - workspace.to_path_buf(), - max_subagents, - config - .max_admitted_subagents_for_provider(&identity) - .max(max_subagents), - Duration::from_secs(config.subagent_heartbeat_timeout_secs_for_provider(&identity)), - config.launch_concurrency_for_provider(&identity), - ); // Probe the durable ledger *before* opening the manager: FleetManager::open // creates `.codewhale/fleet.jsonl` as a side effect, so a later probe would // always find a ledger and the CLI would report availability differently @@ -3837,12 +3826,30 @@ async fn run_fleet_command(workspace: &Path, config: &Config, args: FleetArgs) - // The configured route is the operator: fleet workers without a // task/profile model pin inherit the session's active model. - let manager = FleetManager::open(workspace)? - .with_exec_config(fleet_config.exec.clone()) - .with_fleet_config(fleet_config) - .with_sub_agent_manager(coordination_manager) - .with_session_model(config.default_model()) - .with_route_config(config.clone()); + let owned_workspace = workspace.to_path_buf(); + let owned_config = config.clone(); + let manager = codewhale_runtime::runtime_control::owner_work(move || { + let config = &owned_config; + let max_subagents = config.max_subagents_for_provider(&identity); + let coordination_manager = + codewhale_engine::tools::subagent::new_shared_subagent_manager_with_timeout( + owned_workspace.clone(), + max_subagents, + config + .max_admitted_subagents_for_provider(&identity) + .max(max_subagents), + Duration::from_secs(config.subagent_heartbeat_timeout_secs_for_provider(&identity)), + config.launch_concurrency_for_provider(&identity), + ); + let manager = FleetManager::open(&owned_workspace)? + .with_exec_config(fleet_config.exec.clone()) + .with_fleet_config(fleet_config) + .with_sub_agent_manager(coordination_manager) + .with_session_model(config.default_model()) + .with_route_config(config.clone()); + Ok(manager) + }) + .await?; match args.command { FleetCommand::Init => { println!("Fleet ledger: {}", manager.ledger_path().display()); @@ -3853,7 +3860,12 @@ async fn run_fleet_command(workspace: &Path, config: &Config, args: FleetArgs) - let max_workers = args.max_workers.clamp(1, 128); let manager = manager.with_stale_after(Duration::from_secs(args.stale_after_seconds.max(1))); - let report = manager.create_run_from_task_spec_path(&args.task_spec, max_workers)?; + let launch_manager = manager.clone(); + let task_spec = args.task_spec.clone(); + let report = codewhale_runtime::runtime_control::owner_work(move || { + launch_manager.create_run_from_task_spec_path(&task_spec, max_workers) + }) + .await?; println!( "Fleet run: {} tasks={} leased={} queued={}", report.run_id.0, report.task_count, report.leased, report.queued @@ -3872,19 +3884,20 @@ async fn run_fleet_command(workspace: &Path, config: &Config, args: FleetArgs) - println!( "manager loop running; use `codewhale fleet status`, `inspect`, `interrupt`, or `stop --all` from another terminal." ); - let mut executor = FleetExecutor::new(workspace) + let executor = FleetExecutor::new(workspace) .with_sessions_dir(session_manager::default_sessions_dir()?); let codewhale_binary = fleet::executor::configured_codewhale_binary(); let status = manager .run_to_completion( &report.run_id, max_workers, - &mut executor, + executor, &codewhale_binary, None, Duration::from_secs(2), ) - .await?; + .await? + .0; print_status(&status); Ok(()) } @@ -3928,25 +3941,30 @@ async fn run_fleet_command(workspace: &Path, config: &Config, args: FleetArgs) - )) } FleetCommand::Restart { worker_id } => { - let report = manager.restart_worker(&worker_id)?; + let restart_manager = manager.clone(); + let report = codewhale_runtime::runtime_control::owner_work(move || { + restart_manager.restart_worker(&worker_id) + }) + .await?; print_inspection(&report.inspection); println!( "manager loop running for restarted run {}; use `codewhale fleet status`, `inspect`, `interrupt`, or `stop --all` from another terminal.", report.run_id.0 ); - let mut executor = FleetExecutor::new(workspace) + let executor = FleetExecutor::new(workspace) .with_sessions_dir(session_manager::default_sessions_dir()?); let codewhale_binary = fleet::executor::configured_codewhale_binary(); let status = manager .run_to_completion( &report.run_id, report.max_workers, - &mut executor, + executor, &codewhale_binary, None, Duration::from_secs(2), ) - .await?; + .await? + .0; print_status(&status); Ok(()) } @@ -3955,14 +3973,19 @@ async fn run_fleet_command(workspace: &Path, config: &Config, args: FleetArgs) - stale_after_seconds, } => { let manager = manager.with_stale_after(Duration::from_secs(stale_after_seconds.max(1))); - emit_fleet_receipt(&fleet_control::execute_fleet_control_with( - ControlSurface::Cli, - workspace, - fleet_context, - &manager, - ControlOperation::FleetResume, - Some(&run_id), - )) + let owned_workspace = workspace.to_path_buf(); + let receipt = codewhale_runtime::runtime_control::owner_work(move || { + Ok(fleet_control::execute_fleet_control_with( + ControlSurface::Cli, + &owned_workspace, + fleet_context, + &manager, + ControlOperation::FleetResume, + Some(&run_id), + )) + }) + .await?; + emit_fleet_receipt(&receipt) } FleetCommand::Stop { all } => { if !all { @@ -14933,16 +14956,25 @@ async fn build_direct_workflow_tool( context = context.with_sandbox_backend(Arc::from(backend)); } - let max_subagents = config.max_subagents_for_provider(&identity); - let manager = new_shared_subagent_manager_with_timeout( - workspace.to_path_buf(), - max_subagents, - config - .max_admitted_subagents_for_provider(&identity) - .max(max_subagents), - Duration::from_secs(config.subagent_heartbeat_timeout_secs_for_provider(&identity)), - config.launch_concurrency_for_provider(&identity), - ); + let owned_workspace = workspace.to_path_buf(); + let owned_config = config.clone(); + let owned_identity = identity.clone(); + let manager = codewhale_runtime::runtime_control::owner_work(move || { + let config = &owned_config; + let identity = owned_identity; + let max_subagents = config.max_subagents_for_provider(&identity); + let manager = new_shared_subagent_manager_with_timeout( + owned_workspace, + max_subagents, + config + .max_admitted_subagents_for_provider(&identity) + .max(max_subagents), + Duration::from_secs(config.subagent_heartbeat_timeout_secs_for_provider(&identity)), + config.launch_concurrency_for_provider(&identity), + ); + Ok(manager) + }) + .await?; let roster = Arc::new(codewhale_engine::fleet::identity::load_effective_roster( &config.fleet_config(), workspace, diff --git a/crates/tui/src/runtime_api.rs b/crates/tui/src/runtime_api.rs index 29f5eb2d3..49eadc4fb 100644 --- a/crates/tui/src/runtime_api.rs +++ b/crates/tui/src/runtime_api.rs @@ -2222,7 +2222,12 @@ pub async fn run_http_server( existing_binding.as_ref(), )?; let sessions_dir = runtime_threads.sessions_dir().to_path_buf(); - let sub_agent_manager = runtime_api_sub_agent_manager(&workspace, options.workers)?; + let sub_agent_manager = codewhale_runtime::runtime_control::owner_work({ + let workspace = workspace.clone(); + let workers = options.workers; + move || runtime_api_sub_agent_manager(&workspace, workers) + }) + .await?; runtime_threads .attach_subagent_manager(sub_agent_manager.clone()) .await?; @@ -4496,7 +4501,13 @@ async fn start_fleet_run( )); } let run_id = FleetRunId::from(run_id); - let report = manager.activate_run(&run_id).map_err(|error| { + let activation_manager = manager.clone(); + let activation_run = run_id.clone(); + let report = codewhale_runtime::runtime_control::owner_work(move || { + activation_manager.activate_run(&activation_run) + }) + .await + .map_err(|error| { let message = format!("Failed to start Fleet run '{}': {error}", run_id.0); if message.contains("already terminal") { ApiError::conflict(message) @@ -4516,12 +4527,12 @@ async fn start_fleet_run( let workspace_scope = state.workspace_scope.clone(); tokio::spawn(async move { let _workspace_scope = workspace_scope; - let mut executor = FleetExecutor::new(&workspace).with_sessions_dir(sessions_dir); + let executor = FleetExecutor::new(&workspace).with_sessions_dir(sessions_dir); if let Err(error) = manager .run_to_completion( &execution_run_id, max_workers, - &mut executor, + executor, &codewhale_binary, None, Duration::from_millis(250), @@ -4848,7 +4859,13 @@ async fn restart_fleet_worker( Path(worker_id): Path, ) -> Result, ApiError> { let manager = open_fleet_manager(&state)?; - let report = manager.restart_worker(&worker_id).map_err(|err| { + let restart_manager = manager.clone(); + let restart_worker = worker_id.clone(); + let report = codewhale_runtime::runtime_control::owner_work(move || { + restart_manager.restart_worker(&restart_worker) + }) + .await + .map_err(|err| { ApiError::bad_request(format!( "Failed to restart Fleet worker '{worker_id}': {err}" )) @@ -4862,12 +4879,12 @@ async fn restart_fleet_worker( let workspace_scope = state.workspace_scope.clone(); tokio::spawn(async move { let _workspace_scope = workspace_scope; - let mut executor = FleetExecutor::new(&workspace).with_sessions_dir(sessions_dir); + let executor = FleetExecutor::new(&workspace).with_sessions_dir(sessions_dir); if let Err(err) = manager .run_to_completion( &run_id, max_workers, - &mut executor, + executor, &codewhale_binary, None, Duration::from_millis(250), diff --git a/crates/tui/src/runtime_api/tests.rs b/crates/tui/src/runtime_api/tests.rs index 1210eaf75..97b4297f8 100644 --- a/crates/tui/src/runtime_api/tests.rs +++ b/crates/tui/src/runtime_api/tests.rs @@ -3113,6 +3113,10 @@ async fn agent_runs_runtime_api_exposes_persisted_worker_receipts() -> Result<() }, pending_request: None, status: AgentWorkerStatus::Completed, + prepared_generation: None, + prepared_previous_attempt_unsettled: false, + launch_admission_version: 1, + recovery_interrupted: false, created_at_ms: 1, updated_at_ms: 2, started_at_ms: Some(1), @@ -3379,9 +3383,8 @@ async fn agent_run_cancel_refuses_a_run_owned_by_a_session_it_does_not_host() -> // A child parked on a question in another terminal session: in flight on // disk, but no engine in this runtime owns it. let mut record = { - let manager = - codewhale_engine::tools::subagent::new_shared_subagent_manager(workspace.clone(), 1); - let mut guard = manager.write().await; + let mut guard = + codewhale_engine::tools::subagent::SubAgentManager::new(workspace.clone(), 1); let id = guard.insert_test_running_agent("elsewhere", &workspace); guard.assign_test_session_owner(&id, "terminal-session"); guard diff --git a/crates/tui/src/tui/ui.rs b/crates/tui/src/tui/ui.rs index ccd23f737..822bde915 100644 --- a/crates/tui/src/tui/ui.rs +++ b/crates/tui/src/tui/ui.rs @@ -496,7 +496,7 @@ mod memory_quick_add_tests { } } -fn spawn_tui_engine( +async fn spawn_tui_engine( app: &mut App, api_config: &Config, mcp_handoff: Option, @@ -505,7 +505,8 @@ fn spawn_tui_engine( build_engine_config(app, api_config), api_config, mcp_handoff, - ); + ) + .await; app.mcp_snapshot_generation = 0; app.mcp_snapshot_generation_invalidated = false; // Prime durable agent + coordination state through the same engine event @@ -551,7 +552,7 @@ async fn retire_tui_engine( async fn replace_tui_engine(app: &mut App, api_config: &Config, engine_handle: &mut EngineHandle) { let mcp_handoff = retire_tui_engine(app, engine_handle).await; - *engine_handle = spawn_tui_engine(app, api_config, mcp_handoff); + *engine_handle = spawn_tui_engine(app, api_config, mcp_handoff).await; } /// Startup and consent-triggered replacement restore the same conversation @@ -562,7 +563,7 @@ async fn spawn_tui_engine_with_session( config: &Config, mcp_handoff: Option, ) -> Result { - let handle = spawn_tui_engine(app, config, mcp_handoff); + let handle = spawn_tui_engine(app, config, mcp_handoff).await; let restored = async { if !app.api_messages.is_empty() { handle diff --git a/docs/architecture/delegated-coordination.md b/docs/architecture/delegated-coordination.md index 481ab92e7..0d9b9cadb 100644 --- a/docs/architecture/delegated-coordination.md +++ b/docs/architecture/delegated-coordination.md @@ -107,52 +107,128 @@ they did is reachable through an `agent` action — `status`/`peek` for `list`, `message`, `followup`, `interrupt`, `wait`, and `claim` for the one capability that had no equivalent, write-scope expansion. -## The workspace lock, and what losing it does and does not mean +## Durable workspace transactions and writer ownership -The ledger lives in one file, `.codewhale/state/subagents.v1.json`, written as a -whole-document atomic replace. One per-workspace advisory flock -(`subagents.v1.lock`) decides who may *write* the file, in one of two modes: +The current ledger is `.codewhale/state/subagents.v3.json`: one format3 +envelope around the existing agent, worker and coordination schemas. It records +writer provenance, publication sequences and a durable revision. It is the sole +current state authority. Transcripts remain their existing separate artifacts; +legacy recovery bytes are retained backups, not another writable ledger. -- **Shared, warn on failure.** Interactive and `exec` sessions, `codewhale - fleet`, `codewhale resume`/`fork` mounts and the TUI's background task runtime - (`/task`, `task_create`, automations) take the flock shared, so several - sessions in one folder hold it at once. Failing to get it only logs a warning. -- **Exclusive, degrade read-only.** `codewhale serve` and the other standalone - Runtime hosts take it exclusively and refuse every durable ledger append when - another process already holds it; they log the refusal and pick the ledger back - up when the process restarts after that owner exits. Beside such an owner, - shared holders cannot append. +A short OS transaction lock serializes store initialization, migration and every +read-current/check/mutate/publish operation. Claim overlap, decisions and their +sequence/session/scope receipts are checked against that current state before +acknowledgement. Unrelated writers' records are preserved. A stale snapshot +cannot replace a newer publication from its writer, an older launch generation +cannot replace a newer one, and a terminal receipt cannot be resurrected as +Running by a checkpoint. An explicit terminal retry accepts only the same +winning outcome; authoritative newer accounting metadata is retained. -Every shared holder may append, and each append replaces the whole file, so the -last rename wins. The ledger is loaded once at boot; agents another process -started reload as non-live, so no other process's write claims are enforced -here. A resume or fork mount only reads the ledger. +Normal interactive/exec/Fleet/background sessions continue to share the +existing admission flock. The explicit exclusive Runtime constructor retains +its host admission policy; it is not the mechanism that conserves shared +snapshots. No manager holds the transaction lock while a child executes or +while it waits for a provider or channel capacity. Conflicting lock contention +returns a bounded retry error. Unsupported locking or failed store admission +refuses durable writes rather than silently creating an empty writable ledger. -A shared holder beside an exclusive owner, or one whose acquisition failed, does -not get the flock. That is an ordinary state, not a failure: +Each writer holds its own boot lease. Other managers preserve its live records +and use its active claims during admission. An existing unlocked lease can +establish that its writer ended; a missing, invalid or unreadable lease is +uncertain and fails closed. Local task handles and a failed admission flock do +not establish that another process's work stopped. Parked and headless +WaitingForUser claim rules retain their existing lifecycle meaning; they do not +transfer a still-live writer's record ownership to a different boot. -- **It does not affect liveness.** A session that cannot write the ledger runs - its own agents normally. Whether an agent is alive is decided by heartbeat - evidence, never by lock ownership. (Before v0.9.4 the cleanup pass - terminalized every running agent with no live task handle purely because this - process lacked the flock; that coupling is gone.) -- **It does not affect reads.** The boot-time load is unconditional. Such a - session sees the workspace's decisions and write claims even though it cannot - append to them. Gating the load on the write flock previously left a session - without the flock holding an empty default ledger, which it would write straight over - the real one the moment the first session exited and the flock became - acquirable. -- **It does mean no durable ledger appends.** Decision, claim, contention, and - reconciliation mutations still require the flock, and so does any - shared-workspace write-capable launch, because such a launch must be durably - replayable before it executes. A session without the flock can therefore - delegate only read-only and isolated-worktree work. Two shared holders can - both launch shared-workspace writers, and neither enforces the other's claims - (see the known gap below). +Shared coordination mutations run through the existing manager transaction. +Failed operations do not automatically publish partial state. A deliberately +persisted contention receipt remains inspectable even though admission returns +an error. Fleet compensation is fenced by the exact store revision and the +same writer's successful publication; a stale snapshot cannot undo another +writer's work. Worker retention removes its owned records durably before +removing their transcript artifacts or evicting handles. -Known gap: every session holds the ledger as of its own boot, and shared holders -rewrite the whole file. Records another session appended after this one booted -can be lost when this one writes, and write claims are only enforced within one -process. Closing that needs per-session ledger segments unioned on read, so that -no two processes ever write the same file and the claim-overlap check runs -against the union. That work is not done. +Taking over a proven-dead writer refreshes its exact durable worker and agent +projection and records that recovery before the requested mutation. The newer +snapshot sequence fences any older captured checkpoint, including for preflight. +Failed admission restores ownership from the last committed recovery image. +Synthetic recovery interruption is recorded as uncertainty and can be resolved +by the exact authoritative terminal outcome; it cannot authorize replay of an +admitted attempt. Unresolved recovery records remain retained. Local heartbeat +cleanup and retention act only on owned +records; transcript deletion also checks the committed retained identities. + +Fleet restart preparation records the next exact generation as an unstarted +reservation. Cold recovery preserves that reservation until the matching Fleet +lease consumes it. The executor durably admits the full registered launch spec +before starting a process; an admitted or settled attempt cannot launch again +under the same generation. Current ledger cancellation can settle an unstarted +reservation only after any previous attempt has a confirmed outcome; an old +ledger attempt cannot consume its prepared successor. +An explicit controller may prepare that exact next generation for a live +foreign headless worker under the existing Fleet lease comparison, preserving +the execution owner's writer identity and every unrelated receipt. The owner +refreshes its durable generation before its next mutation; old-generation +checkpoints cannot undo the preparation. +The existing Fleet ledger and executor move through the bounded blocking owner +with the same manager-lock file. Successful completion reconciles the worker +projection after processing the executor's terminal event. + +The per-worker `launch_admission_version` distinguishes the current durable +pre-spawn protocol from legacy rows. A missing legacy start timestamp or a +legacy terminal projection does not prove process teardown. Preparation retains +`prepared_previous_attempt_unsettled` until the owning executor confirms the +exact prior run, task, generation and full launch spec, or an authoritative +attempt receipt supplies that proof. A cold owner without either proof refuses +the reserved launch and retains its uncertainty. It does not invent a Failed +outcome or release a live writer's claim. Busy coordination yields without +leasing; unrelated foreign workers remain untouched. + +The existing Fleet executor retains observed terminal output, the prepared +receipt and stopped host until publication is acknowledged. Final stream tails, +including usage, and the terminal receipt form one `TaskAttemptFinalized` ledger +record. An exact retry synchronizes the same candidate instead of appending +usage again. A visible record after a failed sync is not an acknowledgment. +The ledger remembers successful durability only for the same owned file and +length; appends, replacement and compaction invalidate that receipt. + +Terminal completion keeps the public Running gate and its ownership until the +exact terminal result is durably recorded and parent/mailbox/host fan-in is +queued. The admitted bounded blocking owner performs that complete sequence, +so dropping its waiter cannot discard an admitted commit. A filesystem or +publication failure produces uncertainty, retains the winning pending result +and claims, and sends no successful completion. A later retry uses that same +winner. Failed live delivery after a successful commit leaves the durable +terminal fact intact and reports the delivery failure. Explicit in-memory +managers remain in-memory; their results are not durable receipts. + +Current store publication propagates file-write and Unix directory-sync errors. +Opening or observing a new store revision resolves its directory durability +before acknowledging idempotent state, and exact terminal retries repeat that +barrier. Unchanged preflight does not add a directory sync. +An error after rename can still mean the new image is visible, so recovery reads +the committed image and never announces success solely from a failed waiter. +Native filesystem and power-loss qualification remain separate evidence. +Canceling a Fleet waiter retains an admitted blocking batch and its lock until +the batch returns. Dropped executor ownership uses the existing parent-death +pipe or Windows Job containment; that fallback does not create a successful +terminal receipt or claim that a remote effect was undone. + +### Legacy cutover and recovery + +The first current-format admission takes the legacy admission lock exclusively +only for migration, after previous holders have left. It retains the original +legacy bytes and a validated recovery copy, then leaves +`subagents.v1.json` as a private directory containing that recovery material. +The current format3 file is published under the same short transaction lock. +Interrupted cutover states are validated and resumed without replacing a later +current-format revision with the old backup. Future or malformed envelopes +cannot fall back to parsing as legacy state. + +The retained directory prevents the inspected older atomic file publisher from +replacing that pathname with its stale snapshot. It does not prevent an old +binary from executing tools or changing a workspace. Concurrent arbitrary old +binaries are not a supported current coordination domain; close them before +cutover. Recovery data is not silently deleted, and migration qualification +uses isolated synthetic state rather than existing user sessions. Different +explicit state roots remain separate coordination domains as described above. diff --git a/scripts/check-blocking-calls-budget.json b/scripts/check-blocking-calls-budget.json index 0be119d50..54f42f24f 100644 --- a/scripts/check-blocking-calls-budget.json +++ b/scripts/check-blocking-calls-budget.json @@ -211,7 +211,8 @@ }, "crates/engine/src/tools/subagent/mod.rs": { "path_canonicalize": 4, - "std_fs": 4 + "std_fs": 4, + "thread_sleep": 1 }, "crates/engine/src/tools/subagent/worktree.rs": { "path_canonicalize": 6