fix(ci): unred running-threads race and lock execute_tools contract

Serialize every turn write under a reentrant turn_mutation guard so a
monitor RMW cannot stomp a concurrent terminal settle; late item attaches
skip terminal turns. Interrupt before asserting /v1/threads/running clears.
Lock Act/Operate full catalogs for deferred execute_tools (52→53 tools).

crates/tui/src/runtime_api/tests.rs

@@ -17486,9 +17486,28 @@ async fn threads_running_lists_active_turns_and_clears_on_settle() -> Result<()>
     assert_eq!(active[0]["status"], "in_progress");
     let turn_id = active[0]["turn_id"].as_str().context("missing turn id")?;
 
-    let mut turn = manager.test_store().load_turn(turn_id)?;
-    turn.status = crate::runtime_threads::RuntimeTurnStatus::Completed;
-    manager.test_store().save_turn(&turn)?;
+    // Stop the live runner before the store-side settle. Mid-turn monitor
+    // projection used to load an in-flight snapshot and save_turn it back over
+    // a concurrent Completed write; save_turn now serializes every turn write
+    // (reentrant turn_mutation), late item attaches skip terminal turns, and
+    // interrupting drops the engine so no further projection races the settle.
+    // Prefer the runner's own terminalization when it lands quickly; otherwise
+    // pin Completed the same way the InProgress pin above is authoritative.
+    let _ = manager.interrupt_turn(thread_id, turn_id).await;
+    let settle_deadline = std::time::Instant::now() + Duration::from_millis(500);
+    loop {
+        let turn = manager.test_store().load_turn(turn_id)?;
+        if !turn.status.is_active_work() {
+            break;
+        }
+        if std::time::Instant::now() >= settle_deadline {
+            let mut turn = turn;
+            turn.status = crate::runtime_threads::RuntimeTurnStatus::Completed;
+            manager.test_store().save_turn(&turn)?;
+            break;
+        }
+        sleep(Duration::from_millis(20)).await;
+    }
 
     let settled: serde_json::Value = client
         .get(format!("{base}/v1/threads/running"))

crates/tui/src/runtime_threads.rs

@@ -666,6 +666,13 @@ pub enum RuntimeTurnStatus {
     Canceled,
 }
 
+impl RuntimeTurnStatus {
+    /// Queued or in-progress — the statuses `GET /v1/threads/running` counts.
+    pub const fn is_active_work(self) -> bool {
+        matches!(self, Self::Queued | Self::InProgress)
+    }
+}
+
 #[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
 #[serde(rename_all = "snake_case")]
 pub enum TurnItemKind {
@@ -1734,7 +1741,11 @@ pub struct RuntimeThreadStore {
     thread_mutation: Arc<parking_lot::Mutex<()>>,
     /// Serializes load-modify-save operations on turn records. Like the
     /// thread guard, it is synchronous and never crosses an `.await`.
-    turn_mutation: Arc<parking_lot::Mutex<()>>,
+    /// Reentrant so [`Self::save_turn`] can take the guard while an outer
+    /// RMW transaction already holds it; every turn write goes through that
+    /// path so a store-side settle cannot be stomped by a concurrent monitor
+    /// save that loaded a stale in-flight copy.
+    turn_mutation: Arc<parking_lot::ReentrantMutex<()>>,
     /// Serializes envelope claim/state transitions. The durable envelope is
     /// the queue; this guard prevents concurrent replay/wake requests from
     /// starting more than one turn for the same message.
@@ -1794,7 +1805,7 @@ impl RuntimeThreadStore {
             state_path,
             event_lock_path,
             thread_mutation: Arc::new(parking_lot::Mutex::new(())),
-            turn_mutation: Arc::new(parking_lot::Mutex::new(())),
+            turn_mutation: Arc::new(parking_lot::ReentrantMutex::new(())),
             goal_mutation: Arc::new(parking_lot::Mutex::new(())),
             mail_mutation: Arc::new(parking_lot::Mutex::new(())),
             #[cfg(test)]
@@ -2211,6 +2222,10 @@ impl RuntimeThreadStore {
     }
 
     pub fn save_turn(&self, turn: &TurnRecord) -> Result<()> {
+        // Hold the turn mutation guard for every write — including callers that
+        // already hold it (reentrant) — so unlocked test/API settles cannot be
+        // overwritten by a monitor RMW that loaded an older in-flight snapshot.
+        let _turn_mutation = self.turn_mutation.lock();
         turn.validate_output_token_limit()?;
         validated_record_id(&turn.thread_id, "thread id")?;
         let path = self.turn_path(&turn.id)?;
@@ -4314,7 +4329,7 @@ struct RecoveredTurnReceipt {
 /// - the Runtime event-file transaction lock — serializes writes across processes.
 /// - `RuntimeThreadStore::thread_mutation` — synchronizes short, synchronous
 ///   thread-record load-modify-save transactions and never crosses `.await`.
-/// - `RuntimeThreadStore::turn_mutation` — does the same for turn records.
+/// - `RuntimeThreadStore::turn_mutation` — reentrant guard for turn records; `save_turn` always acquires it.
 /// - `RuntimeThreadStore::goal_mutation` — serializes goal controls and progress;
 ///   acquired after `active` and before `thread_mutation` at turn admission.
 /// - `RuntimeThreadManager::active` — protects the set of loaded engine handles.
@@ -7256,10 +7271,7 @@ impl RuntimeThreadManager {
         let mut active_by_thread: std::collections::BTreeMap<String, Vec<ActiveTurn>> =
             std::collections::BTreeMap::new();
         for turn in self.store.list_all_turns()? {
-            if matches!(
-                turn.status,
-                RuntimeTurnStatus::Queued | RuntimeTurnStatus::InProgress
-            ) {
+            if turn.status.is_active_work() {
                 active_by_thread
                     .entry(turn.thread_id.clone())
                     .or_default()
@@ -11227,6 +11239,8 @@ impl RuntimeThreadManager {
                     {
                         let _turn_mutation = self.store.turn_mutation.lock();
                         let mut turn = self.store.load_turn(&turn_id)?;
+                        // Load-under-lock sees any concurrent terminal settle;
+                        // field updates below preserve that status.
                         turn.started_at = Some(created_at);
                         // A lifecycle start carries no billing envelope, so
                         // there is nothing to persist yet. The dispatch event
@@ -11258,6 +11272,8 @@ impl RuntimeThreadManager {
                     {
                         let _turn_mutation = self.store.turn_mutation.lock();
                         let mut turn = self.store.load_turn(&turn_id)?;
+                        // Preserve whatever status the store already has (including
+                        // a concurrent terminal settle); only refresh route fields.
                         if let Some(envelope) = route.cost_envelope() {
                             turn.persist_effective_route(&envelope);
                         }
@@ -12593,7 +12609,8 @@ impl RuntimeThreadManager {
     fn attach_item_to_turn(&self, turn_id: &str, item_id: &str) -> Result<()> {
         let _turn_mutation = self.store.turn_mutation.lock();
         let mut turn = self.store.load_turn(turn_id)?;
-        if !turn.item_ids.iter().any(|id| id == item_id) {
+        // A terminal settle must not be rewritten by a late stream item attach.
+        if turn.status.is_active_work() && !turn.item_ids.iter().any(|id| id == item_id) {
             turn.item_ids.push(item_id.to_string());
             self.store.save_turn(&turn)?;
         }

scripts/runtime-contract-budget.json

@@ -1,5 +1,5 @@
 {
-  "_comment": "One-way numeric ceilings and exact structural identities for the provider-free runtime contract. Decreases pass; increases or identity changes fail. Lock in decreases with: python3 scripts/check-runtime-contract-budget.py --update The v0.9.8 child-receipt restore grew every production tool surface by 1496 schema bytes / 374 estimated tokens (agent tool). The v0.9.8 workshop read/tool-result byte fields then grew every production tool surface by 371 schema bytes / 93 estimated tokens. Both raises are explicit maintainer decisions; identities stay on the pre-raise digests only if the name set is unchanged \u2014 re-measure on Linux CI if Lint reports identity drift. The v0.9.8 pinned session prefix added the <context_update> sentence to the base prompt (5848 -> 6084 bytes, every representative stage re-hashed), and the host-side Workflow/Goal verbs plus honest child posture grew the tool catalog (active 16531 -> 16602 bytes, full 71473 -> 72371); both are explicit v0.9.8 maintainer decisions measured from the release train. The v0.9.9 configured-skills change hides only custom configured-root paths, preserves discoverable default-root paths, normalizes Windows prompt separators, and trims 50 redundant skills-prompt bytes. The skill/memory/goal/handoff identities were re-measured without raising any ceiling. Explicit maintainer decision for #5473/#5492. The v0.9.10 full surfaces intentionally add the safe read_media tool; their measured schemas remain below the prior byte/token ceilings. Representative prompt byte metrics now use the same host-independent normalized text as their identities; the normalized base is 6089 bytes. The v0.9.11 model-visible sub-agent surface intentionally retires six legacy agents/* tools in favor of the canonical agent tool; all affected schema and prompt metrics decrease. The v0.9.12 plugin prompt-match slice intentionally adds the request_plugin_install tool to the full tool surfaces (plan full: +518 schema bytes / +130 estimated tokens / 29 -> 30 tools) so a strong prompt match can surface the human review CTA; explicit maintainer decision for #5663/#5579. The v0.9.13 profile pins a non-executed bare bash shell so interpreter guidance is reproducible across hosts. The duplicate tts catalog entry is intentionally hidden; speech remains canonical and the alias remains available for saved-transcript dispatch. Explicit v0.9.13 maintainer decision (2026-09-08): after removing 1426 repeated guidance bytes and pinning the bash-v2 fixture, accept only the measured tool byte/token ceilings from all-features macOS source e27735bb63c897f88061c71567701fd971f5d396, verified libtest SHA-256 5e8cbe213f32c4ecdec63494c4de5e31857b4a40134edf7b21a55bca926b1b38: active 13274/3319 in every mode, Plan full 39885/9972, Act/Operate full 67603/16901, with no margin. Against the prior budget, active +390 bytes is agent -41 plus retained bash command syntax +431. Plan full also retains Git commit_plan +253, update_goal progress +583, github bounded local-report guidance +127, review complete-input refusal +35, and send_later dispatching status +14. Act/Operate full instead has github +2151 and additionally speech +230, hidden tts -2120, and tasks/automation exact model-route fields +274 each. The older budget predates v0.9.12: that tag had already removed 361 agent bytes and added the two 274-byte route fields; the retained initial increase versus the tag is 751 source-attributed bytes (agent +320, bash +431), not the +390 budget delta. Only the seven active definitions form the initial request; full catalogs include deferred tools. Estimated tokens use the existing bytes/4 heuristic, not provider usage or billing. Prompt, representative-context, skill-discovery and tool-name identities/ceilings are unchanged. Explicit v0.9.13 maintainer decision (2026-09-09): source ccc5dadfa2279545bf084d37cff3617e41ceaae2 intentionally exposes create_goal, get_goal and update_goal before continuation, so all three initial surfaces now contain ten tools. Measure exact source 4648d148eea64782be857eda6952af2c539cbfcc with the hosted macOS all-features libtest SHA-256 4485c88c7a8b66b8bb9a135807321e417a1e266457ecb74cffc7dfb92f850fc4: four exact provider-free metric tests pass. Active schemas are exactly 17847 bytes / 4462 estimated tokens (+4573 / +1143 for the three eager goal definitions); Plan full is 40597 / 10150 and Act/Operate full is 68315 / 17079, with no margin. The +712 full-catalog bytes are request_user_input guidance +358, update_goal state-change guidance +98, list_dir home-relative path guidance +56, explicit review max_passes schema +197, and three defer_loading true-to-false values +3. The three active name sets/digests, their counts, and measured active/full byte/token ceilings change; full name identities and all prompt, representative-context and skill-discovery measurements remain unchanged. This updates the earlier seven-tool initial-request receipt; bytes/4 remains an estimate, not measLine truncated
+  "_comment": "One-way numeric ceilings and exact structural identities for the provider-free runtime contract. Decreases pass; increases or identity changes fail. Lock in decreases with: python3 scripts/check-runtime-contract-budget.py --update The v0.9.8 child-receipt restore grew every production tool surface by 1496 schema bytes / 374 estimated tokens (agent tool). The v0.9.8 workshop read/tool-result byte fields then grew every production tool surface by 371 schema bytes / 93 estimated tokens. Both raises are explicit maintainer decisions; identities stay on the pre-raise digests only if the name set is unchanged \u2014 re-measure on Linux CI if Lint reports identity drift. The v0.9.8 pinned session prefix added the <context_update> sentence to the base prompt (5848 -> 6084 bytes, every representative stage re-hashed), and the host-side Workflow/Goal verbs plus honest child posture grew the tool catalog (active 16531 -> 16602 bytes, full 71473 -> 72371); both are explicit v0.9.8 maintainer decisions measured from the release train. The v0.9.9 configured-skills change hides only custom configured-root paths, preserves discoverable default-root paths, normalizes Windows prompt separators, and trims 50 redundant skills-prompt bytes. The skill/memory/goal/handoff identities were re-measured without raising any ceiling. Explicit maintainer decision for #5473/#5492. The v0.9.10 full surfaces intentionally add the safe read_media tool; their measured schemas remain below the prior byte/token ceilings. Representative prompt byte metrics now use the same host-independent normalized text as their identities; the normalized base is 6089 bytes. The v0.9.11 model-visible sub-agent surface intentionally retires six legacy agents/* tools in favor of the canonical agent tool; all affected schema and prompt metrics decrease. The v0.9.12 plugin prompt-match slice intentionally adds the request_plugin_install tool to the full tool surfaces (plan full: +518 schema bytes / +130 estimated tokens / 29 -> 30 tools) so a strong prompt match can surface the human review CTA; explicit maintainer decision for #5663/#5579. The v0.9.13 profile pins a non-executed bare bash shell so interpreter guidance is reproducible across hosts. The duplicate tts catalog entry is intentionally hidden; speech remains canonical and the alias remains available for saved-transcript dispatch. Explicit v0.9.13 maintainer decision (2026-09-08): after removing 1426 repeated guidance bytes and pinning the bash-v2 fixture, accept only the measured tool byte/token ceilings from all-features macOS source e27735bb63c897f88061c71567701fd971f5d396, verified libtest SHA-256 5e8cbe213f32c4ecdec63494c4de5e31857b4a40134edf7b21a55bca926b1b38: active 13274/3319 in every mode, Plan full 39885/9972, Act/Operate full 67603/16901, with no margin. Against the prior budget, active +390 bytes is agent -41 plus retained bash command syntax +431. Plan full also retains Git commit_plan +253, update_goal progress +583, github bounded local-report guidance +127, review complete-input refusal +35, and send_later dispatching status +14. Act/Operate full instead has github +2151 and additionally speech +230, hidden tts -2120, and tasks/automation exact model-route fields +274 each. The older budget predates v0.9.12: that tag had already removed 361 agent bytes and added the two 274-byte route fields; the retained initial increase versus the tag is 751 source-attributed bytes (agent +320, bash +431), not the +390 budget delta. Only the seven active definitions form the initial request; full catalogs include deferred tools. Estimated tokens use the existing bytes/4 heuristic, not provider usage or billing. Prompt, representative-context, skill-discovery and tool-name identities/ceilings are unchanged. Explicit v0.9.13 maintainer decision (2026-09-09): source ccc5dadfa2279545bf084d37cff3617e41ceaae2 intentionally exposes create_goal, get_goal and update_goal before continuation, so all three initial surfaces now contain ten tools. Measure exact source 4648d148eea64782be857eda6952af2c539cbfcc with the hosted macOS all-features libtest SHA-256 4485c88c7a8b66b8bb9a135807321e417a1e266457ecb74cffc7dfb92f850fc4: four exact provider-free metric tests pass. Active schemas are exactly 17847 bytes / 4462 estimated tokens (+4573 / +1143 for the three eager goal definitions); Plan full is 40597 / 10150 and Act/Operate full is 68315 / 17079, with no margin. The +712 full-catalog bytes are request_user_input guidance +358, update_goal state-change guidance +98, list_dir home-relative path guidance +56, explicit review max_passes schema +197, and three defer_loading true-to-false values +3. The three active name sets/digests, their counts, and measured active/full byte/token ceilings change; full name identities and all prompt, representative-context and skill-discovery measurements remain unchanged. This updates the earlier seven-tool initial-request receipt; bytes/4 remains an estimate, not measLine truncated
   "document_kind": "codewhale.runtime_contract_budget",
   "representative_context": {
     "fixture_id": "representative-v1",
@@ -105,9 +105,9 @@
           "tools": 11
         },
         "full": {
-          "bytes": 81817,
-          "identity_sha256": "1203d192385fd2b02227ef9e4212e5379bc5cbb813a373f12539406b5958aef1",
-          "tokens_est": 20455,
+          "bytes": 82702,
+          "identity_sha256": "45e989bbe5ac0bb1f2d9084361c021009f30a06539f132ebd4fd2331a1bb1954",
+          "tokens_est": 20676,
           "tool_names": [
             "Git",
             "Run",
@@ -119,6 +119,7 @@
             "create_goal",
             "diagnostics",
             "edit",
+            "execute_tools",
             "file_search",
             "fim_edit",
             "finance",
@@ -162,7 +163,7 @@
             "workflow",
             "write"
           ],
-          "tools": 52
+          "tools": 53
         }
       },
       "operate": {
@@ -186,9 +187,9 @@
           "tools": 11
         },
         "full": {
-          "bytes": 81817,
-          "identity_sha256": "1203d192385fd2b02227ef9e4212e5379bc5cbb813a373f12539406b5958aef1",
-          "tokens_est": 20455,
+          "bytes": 82702,
+          "identity_sha256": "45e989bbe5ac0bb1f2d9084361c021009f30a06539f132ebd4fd2331a1bb1954",
+          "tokens_est": 20676,
           "tool_names": [
             "Git",
             "Run",
@@ -200,6 +201,7 @@
             "create_goal",
             "diagnostics",
             "edit",
+            "execute_tools",
             "file_search",
             "fim_edit",
             "finance",
@@ -243,7 +245,7 @@
             "workflow",
             "write"
           ],
-          "tools": 52
+          "tools": 53
         }
       },
       "plan": {