test(run): synchronize process shutdown tests to fix flaky CI (#1908)

This commit is contained in:
l0ng-ai authored and GitHub committed 2026-11-07 11:25:54 +08:00
1 parent 352efc02a7
commit 68342b32bf
1 file changed
+62 -30
+62 -30
View File
@@ -23,38 +23,61 @@ fn wait_exit(events: &Receiver<IdeEvent>, id: RunId, timeout: Duration) -> Optio
let deadline = Instant::now() + timeout;
loop {
let left = deadline.saturating_duration_since(Instant::now());
match events.recv_timeout(left) {
Ok(IdeEvent::RunExited { run, code }) if run == id => return code,
Ok(_) => {}
Err(_) => return None,
match events.recv_timeout(left).expect("RunExited event") {
IdeEvent::RunExited { run, code } if run == id => return code,
_ => {}
}
}
}
fn wait_for(what: &str, mut ready: impl FnMut() -> bool) {
let deadline = Instant::now() + Duration::from_secs(10);
while !ready() {
assert!(Instant::now() < deadline, "timed out waiting for {what}");
std::thread::sleep(Duration::from_millis(20));
}
}
fn assert_still_running(events: &Receiver<IdeEvent>, timeout: Duration) {
let event = events.recv_timeout(timeout);
assert!(
matches!(event, Err(mpsc::RecvTimeoutError::Timeout)),
"run exited before shutdown was released: {event:?}"
);
}
#[test]
fn a_stopped_run_keeps_its_port_until_the_group_is_gone() {
let tmp = tempfile::tempdir().unwrap();
let (sink, events) = channel_sink();
let pm = ProcessManager::new();
// The `mvn`-starts-a-JVM shape: the direct child dies on SIGINT while the process it
// started ignores it and keeps its socket open for a moment.
// started ignores it and keeps its socket open until the test releases it.
let script = tmp.path().join("hold.py");
std::fs::write(
&script,
"import signal, socket, sys, time\n\
"import pathlib, signal, socket, sys, time\n\
signal.signal(signal.SIGINT, signal.SIG_IGN)\n\
s = socket.socket(); s.bind(('127.0.0.1', 0)); s.listen()\n\
open(sys.argv[1], 'w').write(str(s.getsockname()[1]))\n\
time.sleep(2)\n",
while not pathlib.Path(sys.argv[2]).exists(): time.sleep(0.01)\n",
)
.unwrap();
let port_file = tmp.path().join("port");
let release_file = tmp.path().join("release");
let config = RunConfig {
id: "boot".into(),
name: "boot".into(),
kind: "shell".into(),
command: "/bin/sh".into(),
args: vec!["-c".into(), format!("python3 {} {} & sleep 60", script.display(), port_file.display())],
args: vec![
"-c".into(),
"python3 \"$1\" \"$2\" \"$3\" & sleep 60".into(),
"sh".into(),
script.display().to_string(),
port_file.display().to_string(),
release_file.display().to_string(),
],
cwd: None,
env: BTreeMap::new(),
detected: false,
@@ -64,18 +87,24 @@ fn a_stopped_run_keeps_its_port_until_the_group_is_gone() {
extras: Default::default(),
};
let info = pm.start(tmp.path(), &config, &StartOptions::default(), sink, Box::new(|_| {})).expect("start");
let deadline = Instant::now() + Duration::from_secs(10);
while std::fs::read_to_string(&port_file).is_err() && Instant::now() < deadline {
std::thread::sleep(Duration::from_millis(20));
}
wait_for("the listening port", || {
std::fs::read_to_string(&port_file).ok().is_some_and(|text| text.trim().parse::<u16>().is_ok())
});
let port: u16 = std::fs::read_to_string(&port_file).expect("port").trim().parse().unwrap();
let stopped = Instant::now();
pm.stop(info.id).unwrap();
let pid = info.pid.expect("child pid") as libc::pid_t;
wait_for("the direct child to exit", || unsafe { libc::kill(pid, 0) != 0 });
let exit = events.recv_timeout(Duration::from_secs(1));
let port_held = std::net::TcpStream::connect(("127.0.0.1", port)).is_ok();
std::fs::write(&release_file, "").unwrap();
assert!(
matches!(exit, Err(mpsc::RecvTimeoutError::Timeout)),
"run exited while its descendant held the port: {exit:?}"
);
assert!(port_held, "the descendant still holds its port");
let code = wait_exit(&events, info.id, Duration::from_secs(20));
let elapsed = stopped.elapsed();
assert!(code.is_some(), "the run reports its exit");
assert!(elapsed >= Duration::from_millis(1500), "exit came {elapsed:?} after Stop, before the port was free");
assert!(
std::net::TcpStream::connect(("127.0.0.1", port)).is_err(),
"the port is still bound when the run reports finished"
@@ -87,15 +116,16 @@ fn a_first_stop_does_not_kill_a_program_that_is_still_shutting_down() {
let tmp = tempfile::tempdir().unwrap();
let (sink, events) = channel_sink();
let pm = ProcessManager::new();
// Ignores SIGINT, then exits by itself after 2 s, like a slow Spring Boot shutdown: a first
// Stop waits for it instead of killing it (exit code 137).
// Ignores SIGINT until stdin releases the shutdown, like a slow Spring Boot shutdown.
// A first Stop must wait even past the grace period instead of killing it (exit code 137).
let script = tmp.path().join("slow.py");
let ready_file = tmp.path().join("ready");
std::fs::write(
&script,
"import signal, sys, time\n\
"import pathlib, signal, sys\n\
signal.signal(signal.SIGINT, signal.SIG_IGN)\n\
print('ready', flush=True)\n\
time.sleep(2)\n",
pathlib.Path(sys.argv[1]).touch()\n\
sys.stdin.readline()\n",
)
.unwrap();
let config = RunConfig {
@@ -103,7 +133,7 @@ fn a_first_stop_does_not_kill_a_program_that_is_still_shutting_down() {
name: "slow".into(),
kind: "shell".into(),
command: "python3".into(),
args: vec![script.display().to_string()],
args: vec![script.display().to_string(), ready_file.display().to_string()],
cwd: None,
env: BTreeMap::new(),
detected: false,
@@ -113,12 +143,12 @@ fn a_first_stop_does_not_kill_a_program_that_is_still_shutting_down() {
extras: Default::default(),
};
let info = pm.start(tmp.path(), &config, &StartOptions::default(), sink, Box::new(|_| {})).expect("start");
std::thread::sleep(Duration::from_millis(500)); // let the program install its handler
let start = Instant::now();
pm.stop(info.id).unwrap();
wait_for("the SIGINT handler", || ready_file.exists());
pm.stop_with_grace(info.id, Duration::from_millis(100)).unwrap();
assert_still_running(&events, Duration::from_secs(1));
pm.write_stdin(info.id, "finish shutdown\n").unwrap();
let code = wait_exit(&events, info.id, Duration::from_secs(10));
assert_eq!(code, Some(0), "the program exited by itself, not by SIGKILL");
assert!(start.elapsed() >= Duration::from_millis(1500), "stop waited for the shutdown, not a timer");
}
#[test]
@@ -127,11 +157,13 @@ fn a_second_stop_kills_a_program_that_ignores_the_first() {
let (sink, events) = channel_sink();
let pm = ProcessManager::new();
let script = tmp.path().join("stuck.py");
let ready_file = tmp.path().join("ready");
std::fs::write(
&script,
"import signal, time\n\
"import pathlib, signal, sys\n\
signal.signal(signal.SIGINT, signal.SIG_IGN)\n\
time.sleep(60)\n",
pathlib.Path(sys.argv[1]).touch()\n\
sys.stdin.readline()\n",
)
.unwrap();
let config = RunConfig {
@@ -139,7 +171,7 @@ fn a_second_stop_kills_a_program_that_ignores_the_first() {
name: "stuck".into(),
kind: "shell".into(),
command: "python3".into(),
args: vec![script.display().to_string()],
args: vec![script.display().to_string(), ready_file.display().to_string()],
cwd: None,
env: BTreeMap::new(),
detected: false,
@@ -149,9 +181,9 @@ fn a_second_stop_kills_a_program_that_ignores_the_first() {
extras: Default::default(),
};
let info = pm.start(tmp.path(), &config, &StartOptions::default(), sink, Box::new(|_| {})).expect("start");
std::thread::sleep(Duration::from_millis(500));
wait_for("the SIGINT handler", || ready_file.exists());
pm.stop(info.id).unwrap();
std::thread::sleep(Duration::from_millis(200));
assert_still_running(&events, Duration::from_millis(200));
pm.stop(info.id).unwrap();
let code = wait_exit(&events, info.id, Duration::from_secs(10));
assert_eq!(code, Some(137), "the second Stop SIGKILLs");