Merge pull request #6759 from Hmbown/fix/bh2-tools-shell-process-lifecycle

Verified original PR #6759 at 4c31a5314f: exact-head Linux, macOS and Windows tests and doctests, required checks and resolved review threads. Contributor authorship and existing history are preserved.

Signed-off-by: CodeWhale Bot <[email protected]>
This commit is contained in:
Hunter Bown authored and GitHub committed 2026-09-30 09:42:38 -07:00
commit 636c700230
15 files changed
+1234 -145

No files matched your search

+10 -4
View File
@@ -1637,10 +1637,16 @@ pub(super) async fn execute_code_execution_tool(
})?;
cmd.arg(&script_path).current_dir(workspace);
let output = tokio::time::timeout(Duration::from_secs(120), cmd.output())
.await
.map_err(|_| ToolError::Timeout { seconds: 120 })
.and_then(|res| res.map_err(|e| ToolError::execution_failed(e.to_string())))?;
// Contained: at the timeout (or a cancelled turn) the interpreter and
// anything the script started die together; a bare `output()` left both
// running.
let output = tokio::time::timeout(
Duration::from_secs(120),
crate::process_tree::contained_output(&mut cmd),
)
.await
.map_err(|_| ToolError::Timeout { seconds: 120 })
.and_then(|res| res.map_err(|e| ToolError::execution_failed(e.to_string())))?;
let stdout = String::from_utf8_lossy(&output.stdout).to_string();
let stderr = String::from_utf8_lossy(&output.stderr).to_string();
@@ -425,3 +425,26 @@ fn execute_tools_is_eager_in_code_mode_and_deferred_in_direct() {
.all(|definition| definition.name != "execute_tools")
);
}
/// A timed-out or cancelled `code_execution` kills the interpreter and what the
/// script started; a bare `output()` left both running.
#[cfg(unix)]
#[tokio::test]
async fn dropped_code_execution_kills_the_interpreter_tree() {
if crate::dependencies::resolve_python_interpreter().is_none() {
return;
}
let tmp = tempfile::tempdir().expect("tempdir");
let pid_file = tmp.path().join("grandchild.pid");
let code = format!(
"import subprocess\np = subprocess.Popen(['sleep', '300'])\nopen({:?}, 'w').write(str(p.pid))\np.wait()\n",
pid_file.display().to_string()
);
let input = json!({ "code": code });
let run = super::execute_code_execution_tool(&input, tmp.path());
let grandchild = crate::process_tree::drop_once_pid_written(run, &pid_file).await;
assert!(
crate::process_tree::wait_for_pid_exit(grandchild, std::time::Duration::from_secs(5)),
"a process the script started outlived the dropped code_execution call"
);
}
+2
View File
@@ -859,6 +859,8 @@ fn spawn_signal_cleanup_task() {
if !CLEANED_UP.swap(true, std::sync::atomic::Ordering::SeqCst) {
#[cfg(unix)]
crate::tools::shell::abort_pending_persistent_process_groups_for_exit();
#[cfg(unix)]
crate::process_tree::kill_contained_trees_for_exit();
crate::tui::ui::emergency_restore_terminal();
// Nothing async survives the `exit` below, so this is the last
// chance to say how the session ended. `record_blocking` is one
+497 -5
View File
@@ -11,13 +11,13 @@
//! * **Windows:** the child is assigned to a Job Object configured with
//! `JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE`, so closing the job kills every
//! process in it. Hooks are created suspended and resumed only after the
//! assignment, so no descendant can escape; the extension host is not
//! (tokio's `Command` does not expose a suspended spawn), which leaves a
//! window of microseconds before assignment in which Node has not yet run
//! any plugin code.
//! assignment. Tokio children are assigned after spawning and can start
//! descendants before assignment. This bounds ordinary process lifetimes;
//! it is not a security sandbox.
//!
//! Dropping the guard kills the tree. That is deliberate: the guard's lifetime
//! is the tree's lifetime.
//! is the tree's lifetime, unless [`ProcessTree::release`] explicitly lets the
//! tree outlive it.
#[cfg(windows)]
use windows::Win32::Foundation::{CloseHandle, HANDLE};
@@ -130,6 +130,22 @@ impl ProcessTree {
}
}
impl ProcessTree {
/// Give up containment without killing anything: the tree outlives the
/// guard. For a command that exited on its own and may have deliberately
/// left something running.
pub(crate) fn release(self) {
#[cfg(windows)]
{
// Closing the handle kills the job unless the limit is cleared
// first. If clearing fails, the close still kills: the safe side.
let _ = self.job.clear_kill_on_close();
}
#[cfg(not(windows))]
std::mem::forget(self);
}
}
impl Drop for ProcessTree {
fn drop(&mut self) {
#[cfg(unix)]
@@ -144,6 +160,222 @@ impl Drop for ProcessTree {
}
}
/// `Command::output()` for a child whose whole process tree dies with the
/// returned future. Dropping it — a caller's `timeout` elapsing, or a
/// cancelled tool call — kills the child and everything it started, where a
/// bare `output()` left them running. Stdin is closed: callers run
/// non-interactive work that must never wait on input.
pub(crate) async fn contained_output(
cmd: &mut tokio::process::Command,
) -> std::io::Result<std::process::Output> {
contained_output_until(cmd, std::future::pending())
.await
.map(|run| run.output)
}
/// What [`contained_output_until`] collected.
pub(crate) struct ContainedOutput {
pub(crate) output: std::process::Output,
/// `stop` fired before the command exited: the tree was killed, and
/// `output` holds everything it wrote until then.
pub(crate) stopped: bool,
}
/// [`contained_output`], plus a `stop` future (a deadline, a cancel token).
/// When `stop` fires first the tree is killed and the output written so far is
/// still returned, so a hung command leaves evidence of where it hung.
///
/// A command that exits on its own is released, not killed: whatever it
/// deliberately left running with its output redirected (`nohup server >log
/// &`) keeps running, as it did with a bare `output()`. Only a dropped future
/// or `stop` kills the tree. Until then the group is also listed for
/// [`kill_contained_trees_for_exit`], because a terminating signal ends
/// Codewhale through `process::exit`, where no destructor runs, and a child in
/// its own process group no longer receives the terminal's Ctrl+C itself.
pub(crate) async fn contained_output_until(
cmd: &mut tokio::process::Command,
stop: impl std::future::Future<Output = ()>,
) -> std::io::Result<ContainedOutput> {
contained_run(cmd, None, stop, ProcessTree::attach_tokio).await
}
/// [`contained_output`] for a command that reads a request on stdin (a plugin
/// tool's JSON input): `input` is written while the output pipes drain, then
/// stdin is closed.
pub(crate) async fn contained_output_with_input(
cmd: &mut tokio::process::Command,
input: Vec<u8>,
) -> std::io::Result<std::process::Output> {
contained_run(
cmd,
Some(input),
std::future::pending(),
ProcessTree::attach_tokio,
)
.await
.map(|run| run.output)
}
async fn contained_run(
cmd: &mut tokio::process::Command,
input: Option<Vec<u8>>,
stop: impl std::future::Future<Output = ()>,
attach: impl FnOnce(&tokio::process::Child) -> std::io::Result<ProcessTree>,
) -> std::io::Result<ContainedOutput> {
use std::process::Stdio;
cmd.stdin(if input.is_some() {
Stdio::piped()
} else {
Stdio::null()
})
.stdout(Stdio::piped())
.stderr(Stdio::piped())
.kill_on_drop(true);
#[cfg(unix)]
cmd.process_group(0);
let mut child = cmd.spawn()?;
// Never report a contained run when attachment failed. The child's
// kill-on-drop guard still cleans up the direct child on this error path.
let tree = attach(&child)?;
// Declared after `tree`, so a dropped future unlists the group first and
// the tree guard then kills it.
#[cfg(unix)]
let _listed = child.id().map(ContainedExitListing::new);
let mut stdout_pipe = child.stdout.take();
let mut stderr_pipe = child.stderr.take();
let stdin_pipe = child.stdin.take();
let mut stdout = Vec::new();
let mut stderr = Vec::new();
let exited = {
// Written alongside the drains: a child that answers before it has
// read all of its input would otherwise deadlock on a full pipe.
let feed = async move {
use tokio::io::AsyncWriteExt;
if let (Some(mut pipe), Some(input)) = (stdin_pipe, input) {
// A child that exits without reading its input closes the
// pipe; that is its answer, not a run failure.
if pipe.write_all(&input).await.is_ok() {
let _ = pipe.shutdown().await;
}
}
};
let run = async {
let (out, err, status, ()) = tokio::join!(
drain_pipe(stdout_pipe.as_mut(), &mut stdout),
drain_pipe(stderr_pipe.as_mut(), &mut stderr),
child.wait(),
feed,
);
out?;
err?;
status
};
tokio::select! {
status = run => Some(status?),
() = stop => None,
}
};
let (status, stopped) = match exited {
Some(status) => {
tree.release();
(status, false)
}
None => {
drop(tree);
let _ = child.start_kill();
let status = child.wait().await?;
// Collect what is still buffered in the pipes. Bounded: a process
// that escaped the tree may still hold one open.
let _ = tokio::time::timeout(std::time::Duration::from_secs(1), async {
let _ = tokio::join!(
drain_pipe(stdout_pipe.as_mut(), &mut stdout),
drain_pipe(stderr_pipe.as_mut(), &mut stderr),
);
})
.await;
(status, true)
}
};
Ok(ContainedOutput {
output: std::process::Output {
status,
stdout,
stderr,
},
stopped,
})
}
/// Read `pipe` to EOF into `into`. Each chunk lands in `into` as soon as it is
/// read, so a caller that stops polling keeps everything read so far.
async fn drain_pipe<R: tokio::io::AsyncRead + Unpin>(
pipe: Option<&mut R>,
into: &mut Vec<u8>,
) -> std::io::Result<()> {
use tokio::io::AsyncReadExt;
let Some(pipe) = pipe else {
return Ok(());
};
let mut chunk = [0_u8; 8 * 1024];
loop {
let read = pipe.read(&mut chunk).await?;
if read == 0 {
return Ok(());
}
into.extend_from_slice(&chunk[..read]);
}
}
/// Process groups of [`contained_output_until`] runs still in flight.
#[cfg(unix)]
static CONTAINED_EXIT_GROUPS: std::sync::OnceLock<
std::sync::Mutex<std::collections::HashSet<u32>>,
> = std::sync::OnceLock::new();
#[cfg(unix)]
fn contained_exit_groups() -> std::sync::MutexGuard<'static, std::collections::HashSet<u32>> {
CONTAINED_EXIT_GROUPS
.get_or_init(Default::default)
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
}
/// Lists one in-flight process group until dropped.
#[cfg(unix)]
struct ContainedExitListing(u32);
#[cfg(unix)]
impl ContainedExitListing {
fn new(process_group_id: u32) -> Self {
contained_exit_groups().insert(process_group_id);
Self(process_group_id)
}
}
#[cfg(unix)]
impl Drop for ContainedExitListing {
fn drop(&mut self) {
contained_exit_groups().remove(&self.0);
}
}
/// Kill every in-flight contained tree. The process-wide signal path calls
/// this immediately before `process::exit`, where Rust destructors cannot run.
#[cfg(unix)]
pub(crate) fn kill_contained_trees_for_exit() {
let groups = contained_exit_groups().drain().collect::<Vec<_>>();
for process_group_id in groups {
if let Ok(process_group_id) = libc::pid_t::try_from(process_group_id) {
// SAFETY: the id is a child spawned with `process_group(0)` whose
// run is still in flight. A negative pid names that group, never
// Codewhale's own.
unsafe {
libc::kill(-process_group_id, libc::SIGKILL);
}
}
}
}
#[cfg(windows)]
struct WindowsJob {
handle: HANDLE,
@@ -172,6 +404,20 @@ impl WindowsJob {
Ok(job)
}
fn clear_kill_on_close(&self) -> std::io::Result<()> {
let limits = JOBOBJECT_EXTENDED_LIMIT_INFORMATION::default();
// SAFETY: `limits` is live with matching size; the handle is live.
unsafe {
SetInformationJobObject(
self.handle,
JobObjectExtendedLimitInformation,
&limits as *const _ as *const core::ffi::c_void,
std::mem::size_of::<JOBOBJECT_EXTENDED_LIMIT_INFORMATION>() as u32,
)
.map_err(windows_io_error)
}
}
fn terminate(&self) -> std::io::Result<()> {
// SAFETY: `self.handle` is a live owned job handle.
unsafe { TerminateJobObject(self.handle, 1).map_err(windows_io_error) }
@@ -192,3 +438,249 @@ impl Drop for WindowsJob {
pub(crate) fn windows_io_error(error: windows::core::Error) -> std::io::Error {
std::io::Error::other(error)
}
/// Test support: wait for `pid` to stop existing. `true` once it is gone.
/// A zombie counts as gone: it was killed and only awaits a reaper, which a
/// container whose PID 1 does not reap orphans may never provide.
#[cfg(all(test, unix))]
pub(crate) fn wait_for_pid_exit(pid: libc::pid_t, within: std::time::Duration) -> bool {
let deadline = std::time::Instant::now() + within;
loop {
// SAFETY: signal 0 only checks that the process exists.
if unsafe { libc::kill(pid, 0) } != 0 || is_zombie(pid) {
return true;
}
if std::time::Instant::now() >= deadline {
// Do not leak the fixture when the assertion is about to fail.
// SAFETY: kill(2) dereferences no pointers.
unsafe { libc::kill(pid, libc::SIGKILL) };
return false;
}
std::thread::sleep(std::time::Duration::from_millis(50));
}
}
/// Linux only (`/proc`); elsewhere launchd/init reaps orphans promptly.
#[cfg(all(test, unix))]
fn is_zombie(pid: libc::pid_t) -> bool {
std::fs::read_to_string(format!("/proc/{pid}/stat"))
.ok()
.and_then(|stat| {
stat.rsplit_once(')')
.map(|(_, rest)| rest.trim_start().starts_with('Z'))
})
.unwrap_or(false)
}
/// Test support: read the pid a fixture wrote to `path`, waiting for it.
#[cfg(all(test, unix))]
pub(crate) fn read_pid_file(path: &std::path::Path, within: std::time::Duration) -> libc::pid_t {
let deadline = std::time::Instant::now() + within;
loop {
if let Some(pid) = parse_pid_file(path) {
return pid;
}
assert!(
std::time::Instant::now() < deadline,
"fixture never wrote {}",
path.display()
);
std::thread::sleep(std::time::Duration::from_millis(20));
}
}
#[cfg(all(test, unix))]
fn parse_pid_file(path: &std::path::Path) -> Option<libc::pid_t> {
std::fs::read_to_string(path)
.ok()
.and_then(|text| text.trim().parse().ok())
}
/// Test support: drive `run` until the fixture has written its pid to `path`,
/// then drop `run`. No wall-clock guess about how long the fixture needs to
/// start; `run` finishing first is a fixture bug.
#[cfg(all(test, unix))]
pub(crate) async fn drop_once_pid_written<F: std::future::Future>(
run: F,
path: &std::path::Path,
) -> libc::pid_t {
let written = async {
loop {
if let Some(pid) = parse_pid_file(path) {
return pid;
}
tokio::time::sleep(std::time::Duration::from_millis(20)).await;
}
};
tokio::time::timeout(std::time::Duration::from_secs(60), async {
tokio::select! {
_ = run => panic!("fixture exited before writing {}", path.display()),
pid = written => pid,
}
})
.await
.unwrap_or_else(|_| panic!("fixture never wrote {}", path.display()))
}
#[cfg(all(test, unix))]
mod tests {
use super::*;
use std::time::Duration;
#[tokio::test]
async fn failed_containment_attachment_is_reported_and_kills_the_child() {
let pid = std::cell::Cell::new(None);
let mut cmd = tokio::process::Command::new("/bin/sleep");
cmd.arg("300");
let result = tokio::time::timeout(
Duration::from_secs(1),
contained_run(&mut cmd, None, std::future::pending(), |child| {
pid.set(child.id());
Err(std::io::Error::new(
std::io::ErrorKind::PermissionDenied,
"injected containment attachment failure",
))
}),
)
.await;
let pid = libc::pid_t::try_from(pid.get().expect("child spawned")).expect("pid");
// Let Tokio reap while checking cleanup; the helper kills the fixture
// itself if the child ever outlives this failed run.
assert!(
tokio::task::spawn_blocking(move || wait_for_pid_exit(pid, Duration::from_secs(5)))
.await
.expect("cleanup probe"),
"the direct child outlived failed containment attachment"
);
let Err(error) = result
.expect("attachment failure must return without running the command to completion")
else {
panic!("a containment failure must not be accepted as a successful run");
};
assert_eq!(error.kind(), std::io::ErrorKind::PermissionDenied);
assert_eq!(error.to_string(), "injected containment attachment failure");
}
#[tokio::test]
async fn dropping_contained_output_kills_the_whole_tree() {
let tmp = tempfile::tempdir().expect("tempdir");
let pid_file = tmp.path().join("grandchild.pid");
let mut cmd = tokio::process::Command::new("/bin/sh");
cmd.arg("-c")
.arg("sleep 300 & echo $! > grandchild.pid; wait")
.current_dir(tmp.path());
let grandchild = drop_once_pid_written(contained_output(&mut cmd), &pid_file).await;
assert!(
wait_for_pid_exit(grandchild, Duration::from_secs(5)),
"a process started by the dropped command is still running"
);
}
#[tokio::test]
async fn contained_output_captures_output_and_closes_stdin() {
let mut cmd = tokio::process::Command::new("/bin/sh");
cmd.arg("-c").arg("cat; echo done");
let output = tokio::time::timeout(Duration::from_secs(10), contained_output(&mut cmd))
.await
.expect("stdin must be closed, not inherited")
.expect("run");
assert!(output.status.success());
assert_eq!(String::from_utf8_lossy(&output.stdout), "done\n");
}
/// The input reaches stdin, and stdin is closed after it.
#[tokio::test]
async fn contained_output_with_input_feeds_then_closes_stdin() {
let mut cmd = tokio::process::Command::new("/bin/sh");
cmd.arg("-c").arg("cat; echo done");
let output = tokio::time::timeout(
Duration::from_secs(10),
contained_output_with_input(&mut cmd, b"request\n".to_vec()),
)
.await
.expect("stdin must be closed after the input")
.expect("run");
assert!(output.status.success());
assert_eq!(String::from_utf8_lossy(&output.stdout), "request\ndone\n");
}
/// A command that exits on its own is not reaped along with what it
/// deliberately left running, as with a bare `output()`.
#[tokio::test]
async fn contained_output_leaves_a_detached_daemon_of_a_clean_exit_running() {
let tmp = tempfile::tempdir().expect("tempdir");
let mut cmd = tokio::process::Command::new("/bin/sh");
cmd.arg("-c")
.arg("sleep 300 >/dev/null 2>&1 & echo $! > daemon.pid")
.current_dir(tmp.path());
let output = contained_output(&mut cmd).await.expect("run");
assert!(output.status.success());
let daemon = read_pid_file(&tmp.path().join("daemon.pid"), Duration::from_secs(5));
// `wait_for_pid_exit` kills the fixture when it times out.
assert!(
!wait_for_pid_exit(daemon, Duration::from_millis(500)),
"the daemon a clean command left behind was killed"
);
}
/// `stop` kills the tree but keeps what the command already wrote.
#[tokio::test]
async fn stopped_contained_output_keeps_partial_output_and_kills_the_tree() {
let tmp = tempfile::tempdir().expect("tempdir");
let mut cmd = tokio::process::Command::new("/bin/sh");
cmd.arg("-c")
.arg("echo before-stop; sleep 300 & echo $! > grandchild.pid; wait")
.current_dir(tmp.path());
let pid_file = tmp.path().join("grandchild.pid");
let stop = async {
while parse_pid_file(&pid_file).is_none() {
tokio::time::sleep(Duration::from_millis(20)).await;
}
};
let run = tokio::time::timeout(
Duration::from_secs(60),
contained_output_until(&mut cmd, stop),
)
.await
.expect("stop must end the run")
.expect("run");
assert!(run.stopped);
assert_eq!(String::from_utf8_lossy(&run.output.stdout), "before-stop\n");
let grandchild = read_pid_file(&pid_file, Duration::from_secs(5));
assert!(
wait_for_pid_exit(grandchild, Duration::from_secs(5)),
"a process started by the stopped command is still running"
);
}
/// An in-flight run is listed for the signal-exit kill, and unlisted once
/// the future is gone.
#[tokio::test]
async fn in_flight_contained_run_is_listed_for_signal_exit() {
let tmp = tempfile::tempdir().expect("tempdir");
let mut cmd = tokio::process::Command::new("/bin/sh");
cmd.arg("-c")
.arg("echo $$ > leader.pid; sleep 300")
.current_dir(tmp.path());
let pid_file = tmp.path().join("leader.pid");
let mut run = Box::pin(contained_output(&mut cmd));
let leader = drop_once_pid_written(
async {
run.as_mut().await.ok();
},
&pid_file,
)
.await;
let leader_group = u32::try_from(leader).expect("pid");
assert!(contained_exit_groups().contains(&leader_group));
drop(run);
assert!(!contained_exit_groups().contains(&leader_group));
// Let this current-thread runtime poll Tokio's orphan reaper while the
// cleanup probe waits; kill(pid, 0) still sees an unreaped macOS child.
assert!(
tokio::task::spawn_blocking(move || wait_for_pid_exit(leader, Duration::from_secs(5)))
.await
.expect("cleanup probe")
);
}
}
+35 -9
View File
@@ -42,7 +42,6 @@
use std::collections::{BTreeMap, BTreeSet};
use std::path::{Path as FsPath, PathBuf};
use std::process::Stdio;
use std::time::Duration;
use axum::Json;
@@ -112,8 +111,12 @@ async fn git_read(workspace: &FsPath, args: &[&str]) -> Result<GitRun, ApiError>
.map_err(|_| ApiError::internal("git read setup failed"))?
.map_err(|error| ApiError::internal(format!("git is unavailable: {error}")))?;
let mut command = tokio::process::Command::from(command);
command.args(args).kill_on_drop(true);
finish_git(command.output(), GIT_READ_TIMEOUT).await
command.args(args);
finish_git(
crate::process_tree::contained_output(&mut command),
GIT_READ_TIMEOUT,
)
.await
}
/// Write path for operator-driven mutations. Non-interactive by contract:
@@ -124,12 +127,14 @@ async fn git_read(workspace: &FsPath, args: &[&str]) -> Result<GitRun, ApiError>
async fn git_write(workspace: &FsPath, args: Vec<String>) -> Result<GitRun, ApiError> {
let mut command = Git::tokio_command()
.ok_or_else(|| ApiError::internal("git is not installed or not in PATH"))?;
command
.args(&args)
.current_dir(workspace)
.stdin(Stdio::null())
.kill_on_drop(true);
finish_git(command.output(), GIT_WRITE_TIMEOUT).await
command.args(&args).current_dir(workspace);
// Contained: hooks and filters run here, and a timeout or a dropped
// request must end them too, not only the `git` process itself.
finish_git(
crate::process_tree::contained_output(&mut command),
GIT_WRITE_TIMEOUT,
)
.await
}
async fn finish_git(
@@ -2692,6 +2697,27 @@ mod tests {
assert_eq!(ok.files.unwrap().keys().collect::<Vec<_>>(), vec!["a.txt"]);
}
/// A dropped (or timed-out) write takes down what git started — a hook, a
/// filter, an alias — not just the `git` process.
#[cfg(unix)]
#[tokio::test]
async fn dropped_git_write_kills_what_git_started() {
let tmp = repo();
let pid_file = tmp.path().join("hang.pid");
let args = vec![
"-c".to_string(),
"alias.hang=!sleep 300 & echo $! > hang.pid; wait".to_string(),
"hang".to_string(),
];
let grandchild =
crate::process_tree::drop_once_pid_written(git_write(tmp.path(), args), &pid_file)
.await;
assert!(
crate::process_tree::wait_for_pid_exit(grandchild, Duration::from_secs(5)),
"a process git started outlived the dropped request"
);
}
#[test]
fn file_diff_reads_the_path_literally_inside_a_subdirectory_workspace() {
let tmp = repo();
+36 -44
View File
@@ -891,11 +891,11 @@ async fn run_git_command_async(
/// Run git under a hard deadline. `Ok(None)` means the deadline passed and the
/// child was killed, so a timed-out fetch never lingers.
///
/// On unix git runs in its own process group and the whole group is killed at
/// the deadline: git hands the network to a transport child (`git-remote-http`,
/// `ssh`) that survives a SIGKILL to git alone and would otherwise keep the
/// stalled connection open indefinitely. `kill_on_drop` still covers the
/// leader everywhere else.
/// git runs contained ([`crate::process_tree::contained_output`]) and the whole
/// tree is killed at the deadline or when the call is cancelled: git hands the
/// network to a transport child (`git-remote-http`, `ssh`) that survives a
/// SIGKILL to git alone and would otherwise keep the stalled connection open
/// indefinitely.
async fn run_git_command_bounded(
working_dir: &Path,
args: &[String],
@@ -906,49 +906,18 @@ async fn run_git_command_bounded(
"git is not installed or not in PATH",
));
};
cmd.args(args)
.current_dir(working_dir)
.stdin(std::process::Stdio::null())
.stdout(std::process::Stdio::piped())
.stderr(std::process::Stdio::piped())
.kill_on_drop(true);
#[cfg(unix)]
cmd.process_group(0);
let child = match cmd.spawn() {
Ok(child) => child,
Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
return Err(ToolError::not_available(
"git is not installed or not in PATH",
));
}
Err(e) => {
return Err(ToolError::execution_failed(format!(
"Failed to run git: {e}"
)));
}
};
let process_group = child.id();
match tokio::time::timeout(timeout, child.wait_with_output()).await {
cmd.args(args).current_dir(working_dir);
match tokio::time::timeout(timeout, crate::process_tree::contained_output(&mut cmd)).await {
Ok(Ok(output)) => Ok(Some(output)),
Ok(Err(e)) if e.kind() == std::io::ErrorKind::NotFound => Err(ToolError::not_available(
"git is not installed or not in PATH",
)),
Ok(Err(e)) => Err(ToolError::execution_failed(format!(
"Failed to run git: {e}"
))),
Err(_) => {
#[cfg(unix)]
if let Some(pgid) = process_group
.and_then(|id| libc::pid_t::try_from(id).ok())
.filter(|pgid| *pgid > 0)
{
// SAFETY: kill(2) dereferences no pointers; a negative pid
// targets the group this call created with process_group(0).
unsafe {
libc::kill(-pgid, libc::SIGKILL);
}
}
#[cfg(not(unix))]
let _ = process_group;
Ok(None)
}
// The elapsed deadline dropped the contained run, which killed git's
// whole tree.
Err(_) => Ok(None),
}
}
@@ -1486,4 +1455,27 @@ mod tests {
assert!(output.status.success());
String::from_utf8_lossy(&output.stdout).trim().to_string()
}
/// A cancelled call kills git's whole tree, not only at the deadline.
#[cfg(unix)]
#[tokio::test]
async fn cancelled_bounded_git_run_kills_what_git_started() {
if !git_available() {
return;
}
let tmp = tempdir().expect("tempdir");
init_git_repo(tmp.path());
let pid_file = tmp.path().join("hang.pid");
let args = [
"-c".to_string(),
"alias.hang=!sleep 300 & echo $! > hang.pid; wait".to_string(),
"hang".to_string(),
];
let run = run_git_command_bounded(tmp.path(), &args, std::time::Duration::from_secs(600));
let grandchild = crate::process_tree::drop_once_pid_written(run, &pid_file).await;
assert!(
crate::process_tree::wait_for_pid_exit(grandchild, std::time::Duration::from_secs(5)),
"a process git started outlived the cancelled call"
);
}
}
+34 -4
View File
@@ -156,10 +156,15 @@ pub async fn execute_js_execution_tool(
cmd.env("NODE_USE_ENV_PROXY", "1");
}
let output = tokio::time::timeout(Duration::from_secs(120), cmd.output())
.await
.map_err(|_| ToolError::Timeout { seconds: 120 })
.and_then(|res| res.map_err(|e| ToolError::execution_failed(e.to_string())))?;
// Contained: a timeout or a cancelled call ends the interpreter and
// anything it started (`setInterval`, a server), not only this future.
let output = tokio::time::timeout(
Duration::from_secs(120),
crate::process_tree::contained_output(&mut cmd),
)
.await
.map_err(|_| ToolError::Timeout { seconds: 120 })
.and_then(|res| res.map_err(|e| ToolError::execution_failed(e.to_string())))?;
let stdout = String::from_utf8_lossy(&output.stdout).to_string();
let stderr = String::from_utf8_lossy(&output.stderr).to_string();
@@ -272,6 +277,31 @@ mod tests {
);
}
/// A timed-out or cancelled call drops the future; the interpreter and
/// what it started must end with it instead of running on.
#[cfg(unix)]
#[tokio::test]
async fn dropped_js_execution_kills_the_interpreter_tree() {
if !node_present() {
return;
}
let tmp = tempdir().expect("tempdir");
let code = "const { spawn } = require('child_process');\n\
const child = spawn('sleep', ['300'], { stdio: 'ignore' });\n\
require('fs').writeFileSync('grandchild.pid', String(child.pid));\n\
setInterval(() => {}, 1000);";
let input = json!({ "code": code });
let grandchild = crate::process_tree::drop_once_pid_written(
execute_js_execution_tool(&input, tmp.path()),
&tmp.path().join("grandchild.pid"),
)
.await;
assert!(
crate::process_tree::wait_for_pid_exit(grandchild, Duration::from_secs(5)),
"a process started by the dropped script is still running"
);
}
#[tokio::test]
async fn execute_js_surfaces_runtime_error_with_nonzero_exit() {
if !node_present() {
+30 -27
View File
@@ -25,7 +25,6 @@ use std::time::Duration;
use async_trait::async_trait;
use serde_json::Value;
use tokio::io::AsyncWriteExt;
use super::spec::{
ApprovalRequirement, ToolCapability, ToolContext, ToolError, ToolResult, ToolSpec,
@@ -300,32 +299,17 @@ async fn run_plugin_child_raw(
let input_bytes = serde_json::to_vec(&input)
.map_err(|e| ToolError::invalid_input(format!("failed to serialize input: {e}")))?;
cmd.stdin(std::process::Stdio::piped());
cmd.stdout(std::process::Stdio::piped());
cmd.stderr(std::process::Stdio::piped());
let mut child = cmd
.spawn()
.map_err(|e| ToolError::execution_failed(format!("failed to spawn {label}: {e}")))?;
let stdin_writer = child.stdin.take().map(|mut stdin| {
tokio::spawn(async move {
if stdin.write_all(&input_bytes).await.is_ok() {
let _ = stdin.shutdown().await;
}
})
});
let output = tokio::time::timeout(PLUGIN_EXECUTION_TIMEOUT, child.wait_with_output())
.await
.map_err(|_| ToolError::Timeout {
seconds: PLUGIN_EXECUTION_TIMEOUT.as_secs(),
})?
.map_err(|e| ToolError::execution_failed(format!("process error: {e}")))?;
if let Some(stdin_writer) = stdin_writer {
let _ = stdin_writer.await;
}
// Contained: a timed-out or cancelled plugin takes everything it started
// down with it, not just the interpreter that `kill_on_drop` would reach.
let output = tokio::time::timeout(
PLUGIN_EXECUTION_TIMEOUT,
crate::process_tree::contained_output_with_input(cmd, input_bytes),
)
.await
.map_err(|_| ToolError::Timeout {
seconds: PLUGIN_EXECUTION_TIMEOUT.as_secs(),
})?
.map_err(|e| ToolError::execution_failed(format!("failed to run {label}: {e}")))?;
if output.status.success() {
let stdout = String::from_utf8_lossy(&output.stdout).to_string();
@@ -728,6 +712,25 @@ echo hello
assert!(result.content.len() > 64 * 1024);
}
/// A cancelled (or timed-out) plugin call kills what the plugin started,
/// not just the interpreter.
#[cfg(unix)]
#[tokio::test]
async fn dropped_plugin_call_kills_the_plugin_process_tree() {
let tmp = TempDir::new().unwrap();
let pid_file = tmp.path().join("grandchild.pid");
let mut cmd = tokio::process::Command::new("/bin/sh");
cmd.arg("-c")
.arg("sleep 300 & echo $! > grandchild.pid; wait")
.current_dir(tmp.path());
let run = run_plugin_child_raw(&mut cmd, "hanging plugin", serde_json::json!({}));
let grandchild = crate::process_tree::drop_once_pid_written(run, &pid_file).await;
assert!(
crate::process_tree::wait_for_pid_exit(grandchild, Duration::from_secs(5)),
"a process started by the cancelled plugin is still running"
);
}
#[test]
fn plugin_deadlock_child_process() {
if std::env::var_os(DEADLOCK_CHILD_ENV).is_none() {
+54 -24
View File
@@ -1512,23 +1512,34 @@ impl BackgroundShell {
}
fn take_delta(&mut self) -> (String, String, usize, usize, usize, usize) {
if let Some(snapshot) = self.bounded_output_snapshot(false).ok().flatten() {
let changed = snapshot.total_bytes != self.stdout_cursor;
self.stdout_cursor = snapshot.total_bytes;
if changed {
self.last_output_at = Instant::now();
self.last_observed_output_len = snapshot.total_bytes;
let delta_len = snapshot.content.len();
return (
snapshot.content,
String::new(),
delta_len,
0,
snapshot.total_bytes,
0,
);
// Only the bytes after the cursor: returning the whole retained tail
// made every `wait` poll repeat output the caller already holds.
let bounded_delta = self.bounded_output.as_ref().and_then(|output| {
let output = output.lock().unwrap_or_else(|error| error.into_inner());
let total = output.total_bytes();
output
.delta_since(self.stdout_cursor)
.ok()
.map(|delta| (delta, total))
});
if let Some(((mut delta, omitted), total_bytes)) = bounded_delta {
let delta_len = total_bytes.saturating_sub(self.stdout_cursor);
self.stdout_cursor = total_bytes;
if delta_len == 0 {
return (String::new(), String::new(), 0, 0, total_bytes, 0);
}
return (String::new(), String::new(), 0, 0, snapshot.total_bytes, 0);
self.last_output_at = Instant::now();
self.last_observed_output_len = total_bytes;
if omitted > 0
&& let Some(output) = self.bounded_output.as_ref()
{
let notice = output
.lock()
.unwrap_or_else(|error| error.into_inner())
.omitted_notice(omitted);
delta.insert_str(0, &notice);
}
return (delta, String::new(), delta_len, 0, total_bytes, 0);
}
let (stdout_delta, stdout_total) =
take_delta_from_buffer(&self.stdout_buffer, &mut self.stdout_cursor);
@@ -2390,9 +2401,14 @@ impl ShellManager {
}
install_parent_death_signal(&mut cmd);
if stdin_data.is_some() {
cmd.stdin(Stdio::piped());
}
// Without input, stdin is closed rather than inherited: an unexpected
// read (`cat`, a prompt) gets EOF instead of blocking on, or reading,
// the operator's terminal until the timeout.
cmd.stdin(if stdin_data.is_some() {
Stdio::piped()
} else {
Stdio::null()
});
child_env::apply_to_command(&mut cmd, child_env::string_map_env(&exec_env.env));
remove_readonly_redirect_env(&mut cmd, &exec_env.env);
@@ -3644,13 +3660,21 @@ impl ShellManager {
/// Age alone is not a bound: it only fired from `list_jobs()`, so a session
/// that never opened the jobs panel evicted nothing, and 500 finished
/// records inside one hour were all retained regardless of size (#5472).
///
/// Age counts from when a job finished, not when it started: a job that
/// ran longer than `max_age` would otherwise be dropped the moment it
/// exited, before its completion was delivered. Undelivered completions
/// age out too: jobs launched over the Runtime API, or owned by a session
/// that is no longer active, are never drained.
pub fn cleanup(&mut self, max_age: Duration) {
self.processes.retain(|_, shell| {
if shell.status == ShellStatus::Running {
true
} else {
shell.started_at.elapsed() < max_age
return true;
}
shell.mark_finished();
shell
.finished_at
.is_none_or(|finished| finished.elapsed() < max_age)
});
self.enforce_finished_job_bounds();
}
@@ -3683,7 +3707,8 @@ impl ShellManager {
(
id.clone(),
shell.completion_reported,
shell.started_at,
// Oldest by finish, like the age rule in `cleanup`.
shell.finished_at.unwrap_or(shell.started_at),
shell.retained_output_bytes(),
)
})
@@ -4870,7 +4895,12 @@ async fn execute_foreground_via_background(
manager.attach_heavy_permit(&task_id, permit)?;
}
if stdin_data.is_some() {
// A foreground pipe command gets EOF on stdin: an unexpected read (`cat`,
// `read x`, a confirmation prompt) then fails at once instead of blocking
// until the timeout kills it. A TTY keeps its terminal input. A command
// later moved to /jobs keeps the closed stdin; interactive input needs
// `background: true` from the start.
if stdin_data.is_some() || !tty {
let mut manager = context
.shell_manager
.lock()
+102 -3
View File
@@ -218,6 +218,53 @@ impl BoundedOutputAccumulator {
self.total_bytes
}
/// Decoded output appended after `cursor` (a previous `total_bytes`), for
/// incremental readers such as `wait`. The second value counts bytes of
/// that range the memory bound already discarded; the complete output is
/// still in the spill file.
pub(super) fn delta_since(&self, cursor: usize) -> io::Result<(String, usize)> {
if let Some(error) = self.stream_error.as_ref() {
return Err(io::Error::other(error.clone()));
}
let new_bytes = self.total_bytes.saturating_sub(cursor);
let available = new_bytes.min(self.tail.len());
let mut bytes = self
.tail
.range(self.tail.len() - available..)
.copied()
.collect::<Vec<_>>();
// A cursor is always a character boundary; a clipped tail's front may
// not be, so drop any continuation bytes it starts with.
let partial = bytes
.iter()
.take_while(|byte| (**byte & 0xC0) == 0x80)
.count();
bytes.drain(..partial);
let omitted = new_bytes - available + partial;
Ok((String::from_utf8_lossy(&bytes).into_owned(), omitted))
}
/// First line of a delta that lost `omitted` bytes to the memory bound:
/// how many, and where the complete output is (or why it is not kept).
pub(super) fn omitted_notice(&self, omitted: usize) -> String {
let location = self
.full_output_path
.as_deref()
.or_else(|| self.temp.as_ref().map(tempfile::NamedTempFile::path));
match (location, self.spill_unavailable.as_deref()) {
(Some(path), _) => format!(
"[{omitted} bytes of earlier output not retained in memory. Full output: {}]\n",
path.display()
),
(None, Some(reason)) => format!(
"[{omitted} bytes of earlier output not retained in memory. Full output was not persisted: {reason}]\n"
),
(None, None) => {
format!("[{omitted} bytes of earlier output not retained in memory]\n")
}
}
}
pub(super) fn snapshot(&mut self, finalize: bool) -> io::Result<BoundedOutputSnapshot> {
if let Some(error) = self.stream_error.as_ref() {
return Err(io::Error::other(error.clone()));
@@ -562,9 +609,9 @@ pub(super) fn tail_text(text: &str, max_chars: usize) -> String {
#[cfg(test)]
mod tests {
use super::{
BOUNDED_OUTPUT_MAX_BYTES, BOUNDED_OUTPUT_MAX_LINES, BoundedOutputAccumulator,
RAW_STREAM_MAX_BYTES, RawOutputBuffer, SharedRawOutput, tail_from_buffer,
take_delta_from_buffer,
BOUNDED_OUTPUT_MAX_BYTES, BOUNDED_OUTPUT_MAX_LINES, BOUNDED_OUTPUT_RETAIN_BYTES,
BoundedOutputAccumulator, RAW_STREAM_MAX_BYTES, RawOutputBuffer, SharedRawOutput,
tail_from_buffer, take_delta_from_buffer,
};
use std::sync::{Arc, Mutex};
@@ -778,6 +825,58 @@ mod tests {
std::fs::remove_file(path).expect("remove full output");
}
#[test]
fn bounded_output_delta_since_returns_only_new_bytes() {
let mut output = BoundedOutputAccumulator::new_in(None);
output.append(b"first\n").expect("append");
let cursor = output.total_bytes();
assert_eq!(output.delta_since(0).expect("delta").0, "first\n");
output.append(b"second\n").expect("append");
assert_eq!(
output.delta_since(cursor).expect("delta"),
("second\n".to_string(), 0)
);
assert_eq!(
output.delta_since(output.total_bytes()).expect("delta"),
(String::new(), 0)
);
// Past the memory bound: the dropped span is counted, and a clipped
// multi-byte character at the front is not emitted as garbage.
let mut output = BoundedOutputAccumulator::new_in(None);
// One ASCII byte first puts the retained tail's front mid-character.
output.append(b"a").expect("append");
output
.append("é".repeat(BOUNDED_OUTPUT_MAX_BYTES).as_bytes())
.expect("append");
let total = output.total_bytes();
let (delta, omitted) = output.delta_since(0).expect("delta");
assert!(!delta.contains('\u{FFFD}'));
assert_eq!(delta.len() + omitted, total);
assert!(delta.len() <= BOUNDED_OUTPUT_RETAIN_BYTES);
}
/// The notice for dropped delta bytes says where the full output is.
#[test]
fn bounded_output_omitted_notice_points_at_the_spill_file() {
let output = BoundedOutputAccumulator::new_in(None);
let spill = output
.temp
.as_ref()
.expect("spill file")
.path()
.display()
.to_string();
let notice = output.omitted_notice(42);
assert!(notice.starts_with("[42 bytes"), "{notice}");
assert!(notice.contains(&spill), "{notice}");
let missing = tempfile::tempdir().expect("tempdir").path().join("gone");
let unspilled = BoundedOutputAccumulator::new_in(Some(&missing));
let notice = unspilled.omitted_notice(7);
assert!(notice.contains("not persisted"), "{notice}");
}
#[test]
fn bounded_output_huge_terminal_line_matches_upstream_notice() {
let mut source = vec![b'x'; BOUNDED_OUTPUT_MAX_BYTES + 1_024];
+199 -1
View File
@@ -4053,7 +4053,8 @@ fn test_list_jobs_cleans_up_completed_old_processes() {
// Both the completed job and any tracking state should be present.
assert!(!manager.processes.is_empty());
// cleanup(ZERO) removes all completed processes immediately.
// cleanup(ZERO) removes every delivered completed process immediately.
manager.drain_finished_jobs_with_evidence();
manager.cleanup(Duration::ZERO);
assert!(
manager.processes.is_empty(),
@@ -4061,6 +4062,203 @@ fn test_list_jobs_cleans_up_completed_old_processes() {
);
}
/// A job that ran longer than the retention age must survive until its
/// completion is delivered: age counts from the finish, not the start.
#[test]
fn cleanup_ages_jobs_from_finish() {
let tmp = tempdir().expect("tempdir");
let mut manager = ShellManager::new(tmp.path().to_path_buf());
let long_run = FINISHED_SHELL_MAX_AGE + Duration::from_secs(600);
manager.seed_finished_record_for_test("long-job", long_run);
// Started 70 minutes ago, finished just now, not yet delivered.
manager.list_jobs();
assert!(
manager.inspect_job("long-job").is_ok(),
"a long job must not be evicted the moment it finishes"
);
let delivered = manager.drain_finished_jobs_with_evidence();
assert_eq!(delivered.len(), 1, "the completion must still be delivered");
assert_eq!(delivered[0].event.task_id, "long-job");
// Delivered and recently finished: still inside the retention window.
manager.cleanup(FINISHED_SHELL_MAX_AGE);
assert!(manager.inspect_job("long-job").is_ok());
// Delivered and finished longer ago than the window: evicted.
let shell = manager.processes.get_mut("long-job").expect("record");
shell.finished_at = Instant::now().checked_sub(long_run);
manager.cleanup(FINISHED_SHELL_MAX_AGE);
assert!(manager.inspect_job("long-job").is_err());
}
/// Nothing ever drains a completion owned by a Runtime API scope or by a
/// session that is no longer active, so an undelivered job must still age out
/// (counted from when it finished) instead of staying until the count cap.
#[test]
fn cleanup_ages_out_undelivered_completions_after_finish() {
let tmp = tempdir().expect("tempdir");
let mut manager = ShellManager::new(tmp.path().to_path_buf());
let max_age = Duration::from_secs(30);
manager.seed_finished_record_for_test("api-job", Duration::from_secs(50));
let shell = manager.processes.get_mut("api-job").expect("record");
shell.owner_session_id = "api:thread-1".to_string();
shell.finished_at = Instant::now().checked_sub(Duration::from_secs(40));
assert!(!shell.completion_reported);
manager.cleanup(max_age);
assert!(
manager.inspect_job("api-job").is_err(),
"an undelivered completion finished longer ago than max_age must age out"
);
}
/// The count ceiling evicts the records that finished longest ago, the same
/// clock the age rule uses: a long job that just finished is not "oldest".
#[test]
fn finished_job_bounds_evict_by_finish_time() {
let tmp = tempdir().expect("tempdir");
let mut manager = ShellManager::new(tmp.path().to_path_buf());
let now = Instant::now();
for index in 0..MAX_FINISHED_SHELL_RECORDS {
let id = format!("short-{index:03}");
// Started 20 s ago, finished 10 s ago.
manager.seed_finished_record_for_test(id.clone(), Duration::from_secs(20));
let shell = manager.processes.get_mut(&id).expect("record");
shell.finished_at = now.checked_sub(Duration::from_secs(10));
shell.completion_reported = true;
}
// Started 60 s ago, finished just now.
manager.seed_finished_record_for_test("long-job", Duration::from_secs(60));
manager
.processes
.get_mut("long-job")
.expect("record")
.completion_reported = true;
manager.cleanup(FINISHED_SHELL_MAX_AGE);
assert_eq!(manager.tracked_job_count(), MAX_FINISHED_SHELL_RECORDS);
assert!(
manager.inspect_job("long-job").is_ok(),
"the most recently finished job was evicted first"
);
}
/// The synchronous path closes stdin when there is no input: `cat` gets EOF
/// instead of reading, or blocking on, Codewhale's own stdin.
#[cfg(unix)]
#[test]
fn sync_command_without_input_gets_eof_not_inherited_stdin() {
let tmp = tempdir().expect("tempdir");
let mut manager = ShellManager::new(tmp.path().to_path_buf());
let started = Instant::now();
let result = manager
.execute_with_options_env(
"cat; echo after-eof",
None,
15_000,
false,
None,
false,
None,
HashMap::new(),
)
.expect("run");
assert!(
started.elapsed() < Duration::from_secs(10),
"stdin read blocked for {:?}",
started.elapsed()
);
assert!(result.stdout.contains("after-eof"), "{}", result.stdout);
}
/// A backgrounded lowercase `bash` job: each read returns only output the
/// caller has not seen, not the whole retained tail again.
#[cfg(unix)]
#[test]
fn bounded_job_delta_returns_only_new_output() {
let tmp = tempdir().expect("tempdir");
let mut manager = ShellManager::new(tmp.path().to_path_buf());
let started = manager
.execute_with_options_env_for_owner_and_work(
"printf 'first-chunk\\n'; while [ ! -e go ]; do sleep 0.05; done; printf 'second-chunk\\n'",
None,
60_000,
true,
None,
false,
None,
HashMap::new(),
None,
String::new(),
None,
None,
None,
None,
false,
(1, BASH_MAX_TIMEOUT_MS),
)
.expect("spawn bounded job");
let task_id = started.task_id.expect("task id");
assert!(
manager.processes[&task_id].bounded_output.is_some(),
"fixture must exercise the bounded accumulator"
);
let deadline = Instant::now() + Duration::from_secs(10);
let mut seen = String::new();
while !seen.contains("first-chunk") {
assert!(Instant::now() < deadline, "first chunk never arrived");
let delta = manager
.get_output_delta(&task_id, false, 0)
.expect("first delta");
seen.push_str(&delta.result.stdout);
std::thread::sleep(Duration::from_millis(20));
}
assert_eq!(seen.matches("first-chunk").count(), 1, "{seen}");
// The fixture writes its second chunk only now, so the first delta cannot
// have raced past it.
std::fs::write(tmp.path().join("go"), "").expect("release fixture");
let rest = manager
.get_output_delta(&task_id, true, 10_000)
.expect("remaining delta");
assert_ne!(rest.result.status, ShellStatus::Running);
assert!(
rest.result.stdout.contains("second-chunk"),
"{}",
rest.result.stdout
);
assert!(
!rest.result.stdout.contains("first-chunk"),
"a delta must not repeat output already returned: {:?}",
rest.result.stdout
);
}
/// A foreground command that reads stdin gets EOF instead of blocking until
/// the timeout kills it.
#[cfg(unix)]
#[tokio::test]
async fn foreground_command_reading_stdin_gets_eof() {
let tmp = tempdir().expect("tempdir");
let ctx = ToolContext::new(tmp.path());
let started = Instant::now();
let result = BashTool::new("Bash")
.execute(
json!({"command": "cat; echo after-eof", "timeout_ms": 15_000}),
&ctx,
)
.await
.expect("execute");
assert!(
started.elapsed() < Duration::from_secs(10),
"stdin read blocked for {:?}",
started.elapsed()
);
assert!(result.content.contains("after-eof"), "{}", result.content);
}
/// Regression for #1691: a `git commit -m "feat: complete sub-pages"` shell
/// command must reach the OS shell with its quoted message intact (one argv
/// slot), never split into `feat:` / `complete` / `sub-pages"`.
+54 -9
View File
@@ -674,26 +674,35 @@ impl TasksTool {
let started = Instant::now();
let mut cmd = build_gate_command(&command, &cwd);
let output =
tokio::time::timeout(std::time::Duration::from_millis(timeout_ms), cmd.output()).await;
// Contained: when the timeout elapses the gate's whole process tree
// is killed, instead of leaving it running behind a "timeout" result,
// and what it wrote until then still reaches the log.
let output = crate::process_tree::contained_output_until(
&mut cmd,
tokio::time::sleep(std::time::Duration::from_millis(timeout_ms)),
)
.await;
let duration_ms = u64::try_from(started.elapsed().as_millis()).unwrap_or(u64::MAX);
let (exit_code, stdout, stderr, timed_out, spawn_error) = match output {
Ok(Ok(out)) => (
out.status.code(),
String::from_utf8_lossy(&out.stdout).to_string(),
String::from_utf8_lossy(&out.stderr).to_string(),
false,
Ok(run) => (
if run.stopped {
None
} else {
run.output.status.code()
},
String::from_utf8_lossy(&run.output.stdout).to_string(),
String::from_utf8_lossy(&run.output.stderr).to_string(),
run.stopped,
None,
),
Ok(Err(err)) => (
Err(err) => (
None,
String::new(),
String::new(),
false,
Some(err.to_string()),
),
Err(_) => (None, String::new(), String::new(), true, None),
};
let full_log = format!(
@@ -1789,6 +1798,42 @@ mod tests {
);
}
#[cfg(unix)]
#[tokio::test]
async fn timed_out_gate_run_kills_the_gate_process_tree() {
let workspace = tempfile::tempdir().expect("workspace");
let context = ToolContext::new(workspace.path())
.with_shell_policy(crate::worker_profile::ShellPolicy::Full);
let result = TasksTool::new("tasks")
.execute(
json!({
"action": "gate_run",
"gate": "test",
"command": "echo gate-started; sleep 300 & echo $! > gate-child.pid; wait",
"timeout_ms": 5_000
}),
&context,
)
.await
.expect("gate runs");
let metadata = result.metadata.expect("metadata");
assert_eq!(metadata["timed_out"], true, "{metadata}");
// What the gate wrote before the timeout is kept.
assert!(
result.content.contains("gate-started"),
"{}",
result.content
);
let child = crate::process_tree::read_pid_file(
&workspace.path().join("gate-child.pid"),
std::time::Duration::from_secs(5),
);
assert!(
crate::process_tree::wait_for_pid_exit(child, std::time::Duration::from_secs(5)),
"the timed-out gate's process is still running"
);
}
#[test]
fn gate_command_uses_non_login_shell_invocation() {
// A login shell would source profile files that can re-export
+24 -4
View File
@@ -595,15 +595,17 @@ fn prune_output(input: &str) -> String {
.char_indices()
.find(|(index, _)| *index >= head)
.map_or(input.len(), |(index, _)| index);
// The earliest boundary that still fits: the tail is where a build's
// error and the command's completion marker are.
let tail_start = input
.char_indices()
.rev()
.find(|(index, _)| input.len() - *index <= tail)
.map_or(0, |(index, _)| index);
.map(|(index, _)| index)
.find(|index| *index >= head_end && input.len() - *index <= tail)
.unwrap_or(input.len());
format!(
"{}\n… [output truncated: {} bytes omitted] …\n{}",
&input[..head_end],
input.len() - OUTPUT_LIMIT,
tail_start - head_end,
&input[tail_start..]
)
}
@@ -1109,6 +1111,24 @@ mod tests {
session_result(&mut session, done, timed_out)
}
#[test]
fn prune_output_keeps_the_tail_and_counts_what_it_dropped() {
let input = format!(
"first-line\n{}\nerror: the last line é\n",
"x".repeat(OUTPUT_LIMIT * 2)
);
let pruned = prune_output(&input);
let (head, rest) = pruned
.split_once("\n… [output truncated: ")
.expect("truncation marker");
let (omitted, tail) = rest.split_once(" bytes omitted] …\n").expect("count");
let omitted: usize = omitted.parse().expect("omitted count");
assert!(head.starts_with("first-line"));
assert!(tail.ends_with("error: the last line é\n"), "{tail:?}");
assert!(tail.len() > OUTPUT_LIMIT / 2, "tail kept only {tail:?}");
assert_eq!(head.len() + omitted + tail.len(), input.len());
}
#[test]
#[cfg(unix)]
fn cd_persists_between_runs() {
+126 -10
View File
@@ -4,6 +4,7 @@
//! approval policy as the other code-executing tools.
use std::path::Path;
use std::time::Duration;
use async_trait::async_trait;
use serde::{Deserialize, Serialize};
@@ -27,6 +28,10 @@ struct RunTestsOutput {
stdout: String,
stderr: String,
command: String,
/// The run hit [`RUN_TESTS_TIMEOUT`] and was killed; stdout/stderr are
/// what it wrote until then.
#[serde(default, skip_serializing_if = "std::ops::Not::not")]
timed_out: bool,
}
#[async_trait]
@@ -100,17 +105,26 @@ impl ToolSpec for RunTestsTool {
}
let command_str = format_command(&workdir, &args);
let output = run_cargo(&workdir, &args)?;
let run = run_cargo(&workdir, &args, context, RUN_TESTS_TIMEOUT).await?;
let output = run.output;
let mut stderr = String::from_utf8_lossy(&output.stderr).into_owned();
if run.stopped {
stderr.push_str(&format!(
"\n[run_tests: stopped after {} s; the output above is everything cargo wrote before it was killed]",
RUN_TESTS_TIMEOUT.as_secs()
));
}
// The whole output: the end of a cargo run is where the failures and
// the `test result:` line are. Size is the engine's one recoverable
// budget (#6508), so the failure summary sees everything.
run_tests_result(RunTestsOutput {
success: output.status.success(),
success: !run.stopped && output.status.success(),
exit_code: output.status.code().unwrap_or(-1),
stdout: String::from_utf8_lossy(&output.stdout).into_owned(),
stderr: String::from_utf8_lossy(&output.stderr).into_owned(),
stderr,
command: command_str,
timed_out: run.stopped,
})
}
}
@@ -134,8 +148,21 @@ fn run_tests_result(result: RunTestsOutput) -> Result<ToolResult, ToolError> {
// === Helpers ===
fn run_cargo(workspace: &Path, args: &[String]) -> Result<std::process::Output, ToolError> {
let Some(mut cmd) = crate::dependencies::Cargo::command() else {
/// Ceiling for one `cargo test` run. Long suites fit; a hung test does not
/// hold the turn forever.
const RUN_TESTS_TIMEOUT: Duration = Duration::from_secs(30 * 60);
/// Run cargo without blocking a runtime worker. Stop and the timeout both end
/// the whole process tree (cargo and the test binaries it started). A timeout
/// still returns the output written so far (`stopped`): it is the only
/// evidence of which test hung.
async fn run_cargo(
workspace: &Path,
args: &[String],
context: &ToolContext,
timeout: Duration,
) -> Result<crate::process_tree::ContainedOutput, ToolError> {
let Some(mut cmd) = crate::dependencies::Cargo::tokio_command() else {
return Err(ToolError::not_available(
"cargo is not installed or not in PATH",
));
@@ -143,8 +170,20 @@ fn run_cargo(workspace: &Path, args: &[String]) -> Result<std::process::Output,
cmd.args(args).current_dir(workspace);
// `cargo test` builds and runs workspace code; do not hand it parent
// credentials.
crate::child_env::apply_to_command(&mut cmd, std::iter::empty::<(&str, &str)>());
cmd.output().map_err(|e| {
crate::child_env::apply_to_tokio_command(&mut cmd, std::iter::empty::<(&str, &str)>());
let run = crate::process_tree::contained_output_until(&mut cmd, tokio::time::sleep(timeout));
let cancelled = async {
match context.cancel_token.as_ref() {
Some(token) => token.cancelled().await,
None => std::future::pending::<()>().await,
}
};
// Dropping `run` on cancel kills the tree.
let output = tokio::select! {
output = run => output,
() = cancelled => return Err(ToolError::cancelled("cargo test cancelled")),
};
output.map_err(|e| {
if e.kind() == std::io::ErrorKind::NotFound {
ToolError::not_available("cargo is not installed or not in PATH")
} else {
@@ -214,6 +253,24 @@ mod tests {
);
}
/// Stop must reach `cargo test`: the call returns as cancelled instead of
/// blocking a runtime worker until cargo exits on its own.
#[tokio::test]
async fn run_tests_honors_cancellation() {
if !cargo_available() {
return;
}
let tmp = tempdir().expect("tempdir");
let token = tokio_util::sync::CancellationToken::new();
token.cancel();
let ctx = ToolContext::new(tmp.path()).with_cancel_token(token);
let result = RunTestsTool.execute(json!({}), &ctx).await;
assert!(
matches!(result, Err(ToolError::Cancelled { .. })),
"cancelled run_tests must not run to completion: {result:?}"
);
}
#[tokio::test]
async fn run_tests_succeeds_on_fresh_project() {
if !cargo_available() {
@@ -305,6 +362,7 @@ mod tests {
stdout: stdout.clone(),
stderr: String::new(),
command: "(cd /repo && cargo test)".to_string(),
timed_out: false,
})
.expect("result");
@@ -378,8 +436,8 @@ mod tests {
}
#[cfg(unix)]
#[test]
fn run_cargo_does_not_inherit_parent_secret_env() {
#[tokio::test]
async fn run_cargo_does_not_inherit_parent_secret_env() {
use crate::test_support::{EnvVarGuard, lock_test_env};
use std::os::unix::fs::PermissionsExt;
if !cargo_available() {
@@ -405,7 +463,17 @@ mod tests {
let _target = EnvVarGuard::set("CARGO_TARGET_DIR", "/tmp/codewhale-fixture-target");
let workspace = tempdir().expect("workspace");
let output = run_cargo(workspace.path(), &["envprobe".to_string()]).expect("cargo runs");
let ctx = ToolContext::new(workspace.path());
let run = run_cargo(
workspace.path(),
&["envprobe".to_string()],
&ctx,
RUN_TESTS_TIMEOUT,
)
.await
.expect("cargo runs");
assert!(!run.stopped);
let output = run.output;
let stdout = String::from_utf8_lossy(&output.stdout);
assert!(
output.status.success(),
@@ -417,4 +485,52 @@ mod tests {
"secret=unset target=/tmp/codewhale-fixture-target"
);
}
/// A run that hits the timeout is killed, tree and all, and still returns
/// what it wrote: the only evidence of which test hung.
#[cfg(unix)]
#[tokio::test]
async fn timed_out_run_cargo_keeps_partial_output_and_kills_the_tree() {
use crate::test_support::{EnvVarGuard, lock_test_env};
use std::os::unix::fs::PermissionsExt;
if !cargo_available() {
return;
}
let _env_lock = lock_test_env();
let bin = tempdir().expect("bin dir");
let probe = bin.path().join("cargo-hangprobe");
fs::write(
&probe,
"#!/bin/sh\necho 'test slow_case ... started'\nsleep 300 &\necho $! > hang.pid\nwait\n",
)
.expect("write probe");
fs::set_permissions(&probe, fs::Permissions::from_mode(0o755)).expect("chmod probe");
let path = std::env::var_os("PATH").unwrap_or_default();
let mut paths = vec![bin.path().to_path_buf()];
paths.extend(std::env::split_paths(&path));
let _path = EnvVarGuard::set("PATH", std::env::join_paths(paths).expect("join PATH"));
let workspace = tempdir().expect("workspace");
let pid_file = workspace.path().join("hang.pid");
let ctx = ToolContext::new(workspace.path());
let run = run_cargo(
workspace.path(),
&["hangprobe".to_string()],
&ctx,
Duration::from_secs(10),
)
.await
.expect("a timed-out run still returns its output");
assert!(run.stopped);
assert!(
String::from_utf8_lossy(&run.output.stdout).contains("test slow_case ... started"),
"{:?}",
run.output
);
let hung = crate::process_tree::read_pid_file(&pid_file, Duration::from_secs(5));
assert!(
crate::process_tree::wait_for_pid_exit(hung, Duration::from_secs(5)),
"the timed-out run's process is still running"
);
}
}
@@ -245,7 +245,14 @@ completion_sound = "off"
click(&mut reopened, "Confirm");
wait(&mut reopened, "y/Enter");
click(&mut reopened, "Confirm");
wait(&mut reopened, "deleted");
// The review already says "Nothing was deleted"; wait for the durable
// mutation and its receipt after the armed review has closed.
reopened
.wait_for(
|frame| !path.exists() && frame.contains("deleted") && !frame.contains("y/Enter"),
WAIT,
)
.unwrap_or_else(|error| panic!("deletion did not complete at {cols}x{rows}: {error}"));
assert!(
!path.exists(),
"confirmed mouse control deletes the reviewed definition"