Run setup/cleanup commands in a shell (#990)

README.md

@@ -108,7 +108,7 @@ hyperfine -S 'my-command > output-${HYPERFINE_ITERATION}.log'
 
 ### Intermediate shell
 
-By default, commands are executed directly, without an intermediate shell (`--shell=none`).
+By default, benchmarked commands, including warmup runs, are executed directly, without an intermediate shell (`--shell=none`).
 Arguments are split using shell-like quoting, so quoted arguments containing spaces are supported.
 Shell syntax such as pipes, redirections, environment-variable expansion, `*`, and `~` is not interpreted.
 This avoids shell startup overhead and the noise from correcting for it, especially for fast commands
@@ -125,10 +125,11 @@ You can also select a specific shell with `--shell <SHELL>`:
 hyperfine --shell zsh 'for i in {1..10000}; do echo test; done'
 ```
 
-The shell setting applies to all commands, including `--setup`, `--prepare`, `--conclude`, and `--cleanup`.
-If any of these commands need shell syntax, enable a shell explicitly.
+The `--setup`, `--prepare`, `--conclude`, and `--cleanup` commands always run through a shell.
+They use the shell selected by `--shell`, or the platform shell (`sh` on Unix, `cmd.exe` on Windows)
+when `--shell` is omitted or set to `none`.
 
-When a shell is enabled, hyperfine *corrects for the shell spawning time*. It runs the shell with an
+When benchmarked commands use a shell, hyperfine *corrects for the shell spawning time*. It runs the shell with an
 empty command multiple times to measure its startup time, then subtracts this time from each
 time measurement.
 

doc/hyperfine.1

@@ -194,14 +194,16 @@ possible parameter combinations.
 .HP
 \fB\-\-shell\fR \fISHELL\fP
 .IP
-Set the shell to use for executing commands (default: none). This can be
-the name or the path to the shell executable, or a full command line
+Set the shell to use for executing benchmarked commands, including warmup runs
+(default: none). This can be the name or the path to the shell executable,
+or a full command line
 like "bash \fB\-\-norc\fR". It can also be set to "default" to explicitly
 select the platform shell (sh on Unix, cmd.exe on Windows). Finally, this can also be
-set to "none" to disable the shell. In this case, commands will be
+set to "none" to disable the shell. In this case, benchmarked commands will be
 executed directly. They can still have arguments, but more complex
 things like "sleep 0.1; sleep 0.2" are not possible without a shell.
-This option also applies to setup, prepare, conclude, and cleanup commands.
+Setup, prepare, conclude, and cleanup commands always use a shell: the selected
+shell, or the platform shell when this option is omitted or set to "none".
 .HP
 \fB\-S\fR
 .IP

src/benchmark/executor.rs

@@ -5,9 +5,7 @@ use std::process::ExitStatus;
 use crate::benchmark::measurement::Measurement;
 use crate::benchmark::measurement::Measurements;
 use crate::command::Command;
-use crate::options::{
-    CmdFailureAction, CommandInputPolicy, CommandOutputPolicy, Options, OutputStyleOption, Shell,
-};
+use crate::options::{CmdFailureAction, CommandOutputPolicy, Options, OutputStyleOption, Shell};
 use crate::output::progress_bar::get_progress_bar;
 use crate::quantity::{second, Information, Time, Zero};
 use crate::timer::execute_and_measure;
@@ -42,6 +40,21 @@ pub trait Executor {
         output_policy: &CommandOutputPolicy,
     ) -> Result<Measurement>;
 
+    /// Run a lifecycle command through a shell, always treating failures as errors.
+    fn run_intermediate_command_and_measure(
+        &self,
+        command: &Command<'_>,
+        iteration: BenchmarkIteration,
+        output_policy: &CommandOutputPolicy,
+    ) -> Result<Measurement> {
+        self.run_command_and_measure(
+            command,
+            iteration,
+            Some(CmdFailureAction::RaiseError),
+            output_policy,
+        )
+    }
+
     /// Perform a calibration of this executor. For example,
     /// when running commands through a shell, we need to
     /// measure the shell spawning time separately in order
@@ -58,13 +71,13 @@ pub trait Executor {
 fn run_command_and_measure_common(
     mut command: std::process::Command,
     iteration: BenchmarkIteration,
-    command_failure_action: CmdFailureAction,
-    command_input_policy: &CommandInputPolicy,
+    command_failure_action: Option<CmdFailureAction>,
+    options: &Options,
     command_output_policy: &CommandOutputPolicy,
     command_name: &str,
     collect_hardware_counters: bool,
 ) -> Result<Measurement> {
-    let stdin = command_input_policy.get_stdin()?;
+    let stdin = options.command_input_policy.get_stdin()?;
     let (stdout, stderr) = command_output_policy.get_stdout_stderr()?;
     command.stdin(stdin).stdout(stdout).stderr(stderr);
 
@@ -83,10 +96,14 @@ fn run_command_and_measure_common(
     if !measurement.exit_status.success() {
         use crate::util::exit_code::extract_exit_code;
 
-        let should_fail = match command_failure_action {
+        let show_ignore_failure_hint = command_failure_action.is_none();
+        let should_fail = match command_failure_action
+            .as_ref()
+            .unwrap_or(&options.command_failure_action)
+        {
             CmdFailureAction::RaiseError => true,
             CmdFailureAction::IgnoreAllFailures => false,
-            CmdFailureAction::IgnoreSpecificFailures(ref codes) => {
+            CmdFailureAction::IgnoreSpecificFailures(codes) => {
                 // Only fail if the exit code is not in the list of codes to ignore
                 if let Some(exit_code) = extract_exit_code(measurement.exit_status) {
                     !codes.contains(&exit_code)
@@ -105,15 +122,17 @@ fn run_command_and_measure_common(
                 BenchmarkIteration::Benchmark(0) => "the first benchmark run".to_string(),
                 BenchmarkIteration::Benchmark(i) => format!("benchmark iteration {i}"),
             };
-            bail!(
-                "{cause} in {when}. Use the '-i'/'--ignore-failure' option if you want to ignore this. \
-                Alternatively, use the '--show-output' option to debug what went wrong.",
-                cause=measurement.exit_status.code().map_or(
-                    "The process has been terminated by a signal".into(),
-                    |c| format!("Command terminated with non-zero exit code {c}")
-
-                ),
+            let cause = measurement.exit_status.code().map_or_else(
+                || "was terminated by a signal".to_owned(),
+                |code| format!("terminated with non-zero exit code {code}"),
             );
+            let hint = if show_ignore_failure_hint {
+                " Use the '-i'/'--ignore-failure' option if you want to ignore this. \
+                 Alternatively, use the '--show-output' option to debug what went wrong."
+            } else {
+                " Use the '--show-output' option to debug what went wrong."
+            };
+            bail!("Command '{command_name}' {cause} in {when}.{hint}");
         }
     }
 
@@ -141,14 +160,26 @@ impl Executor for RawExecutor<'_> {
         run_command_and_measure_common(
             command.get_command()?,
             iteration,
-            command_failure_action.unwrap_or_else(|| self.options.command_failure_action.clone()),
-            &self.options.command_input_policy,
+            command_failure_action,
+            self.options,
             output_policy,
             &command.get_command_line(),
             matches!(iteration, BenchmarkIteration::Benchmark(_)),
         )
     }
 
+    fn run_intermediate_command_and_measure(
+        &self,
+        command: &Command<'_>,
+        iteration: BenchmarkIteration,
+        output_policy: &CommandOutputPolicy,
+    ) -> Result<Measurement> {
+        // Keep shell startup in the hook's duration: raw benchmarks have no
+        // calibrated shell overhead to add back when estimating the run count.
+        ShellExecutor::new(&Shell::platform_default(), self.options)
+            .run_intermediate_command_and_measure(command, iteration, output_policy)
+    }
+
     fn calibrate(&mut self) -> Result<()> {
         Ok(())
     }
@@ -197,8 +228,8 @@ impl Executor for ShellExecutor<'_> {
         let mut measurement = run_command_and_measure_common(
             command_builder,
             iteration,
-            command_failure_action.unwrap_or_else(|| self.options.command_failure_action.clone()),
-            &self.options.command_input_policy,
+            command_failure_action,
+            self.options,
             output_policy,
             &command.get_command_line(),
             false,

src/benchmark/mod.rs

@@ -11,9 +11,7 @@ use crate::benchmark::benchmark_result::Parameter;
 use crate::benchmark::executor::BenchmarkIteration;
 use crate::benchmark::measurement::{Measurement, Measurements};
 use crate::command::Command;
-use crate::options::{
-    CmdFailureAction, CommandOutputPolicy, ExecutorKind, Options, OutputStyleOption,
-};
+use crate::options::{CommandOutputPolicy, ExecutorKind, Options, OutputStyleOption};
 use crate::outlier_detection::OUTLIER_THRESHOLD;
 use crate::output::console_writeln;
 use crate::output::progress_bar::{
@@ -55,22 +53,24 @@ impl<'a> Benchmark<'a> {
         }
     }
 
-    /// Run setup, cleanup, or preparation commands
+    /// Run setup, cleanup, preparation, or conclusion commands
     fn run_intermediate_command(
         &self,
         command: &Command<'_>,
-        error_output: &'static str,
+        kind: &'static str,
         output_policy: &CommandOutputPolicy,
         iteration: executor::BenchmarkIteration,
     ) -> Result<Measurement> {
         self.executor
-            .run_command_and_measure(
-                command,
-                iteration,
-                Some(CmdFailureAction::RaiseError),
-                output_policy,
-            )
-            .map_err(|_| anyhow!(error_output))
+            .run_intermediate_command_and_measure(command, iteration, output_policy)
+            .map_err(|error| {
+                let hint = if cfg!(windows) {
+                    "Append ' || exit /b 0' to the command if this failure can be ignored."
+                } else {
+                    "Append ' || true' to the command if this failure can be ignored."
+                };
+                anyhow!("The {kind} command failed: {error:#} {hint}")
+            })
     }
 
     /// Run the command specified by `--setup`.
@@ -85,14 +85,11 @@ impl<'a> Benchmark<'a> {
             .as_ref()
             .map(|setup_command| Command::new_parametrized(None, setup_command, parameters));
 
-        let error_output = "The setup command terminated with a non-zero exit code. \
-                            Append ' || true' to the command if you are sure that this can be ignored.";
-
         Ok(command
             .map(|cmd| {
                 self.run_intermediate_command(
                     &cmd,
-                    error_output,
+                    "setup",
                     output_policy,
                     BenchmarkIteration::NonBenchmarkRun,
                 )
@@ -113,14 +110,11 @@ impl<'a> Benchmark<'a> {
             .as_ref()
             .map(|cleanup_command| Command::new_parametrized(None, cleanup_command, parameters));
 
-        let error_output = "The cleanup command terminated with a non-zero exit code. \
-                            Append ' || true' to the command if you are sure that this can be ignored.";
-
         Ok(command
             .map(|cmd| {
                 self.run_intermediate_command(
                     &cmd,
-                    error_output,
+                    "cleanup",
                     output_policy,
                     BenchmarkIteration::NonBenchmarkRun,
                 )
@@ -136,10 +130,7 @@ impl<'a> Benchmark<'a> {
         output_policy: &CommandOutputPolicy,
         iteration: executor::BenchmarkIteration,
     ) -> Result<Measurement> {
-        let error_output = "The preparation command terminated with a non-zero exit code. \
-                            Append ' || true' to the command if you are sure that this can be ignored.";
-
-        self.run_intermediate_command(command, error_output, output_policy, iteration)
+        self.run_intermediate_command(command, "preparation", output_policy, iteration)
     }
 
     /// Run the command specified by `--conclude`.
@@ -149,10 +140,7 @@ impl<'a> Benchmark<'a> {
         output_policy: &CommandOutputPolicy,
         iteration: executor::BenchmarkIteration,
     ) -> Result<Measurement> {
-        let error_output = "The conclusion command terminated with a non-zero exit code. \
-                            Append ' || true' to the command if you are sure that this can be ignored.";
-
-        self.run_intermediate_command(command, error_output, output_policy, iteration)
+        self.run_intermediate_command(command, "conclusion", output_policy, iteration)
     }
 
     fn validate_measurement(&self, measurement: &Measurement) -> Result<()> {

src/cli.rs

@@ -236,14 +236,15 @@ fn build_command() -> Command {
                 .value_name("SHELL")
                 .overrides_with("shell")
                 .value_hint(ValueHint::CommandString)
-                .help("Set the shell to use for executing commands (default: none). This can be the \
-                       name or the path to the shell executable, or a full command line \
+                .help("Set the shell to use for executing benchmarked commands, including warmup runs \
+                       (default: none). This can be the name or the path to the shell executable, or a full command line \
                        like \"bash --norc\". It can also be set to \"default\" to explicitly select \
                        the platform shell (sh on Unix, cmd.exe on Windows). It can also be set to \
-                       \"none\" to disable the shell. In this case, commands will be executed \
+                       \"none\" to disable the shell. In this case, benchmarked commands will be executed \
                        directly. They can still have arguments, but more complex things like \
-                       \"sleep 0.1; sleep 0.2\" are not possible without a shell. This option also \
-                       applies to setup, prepare, conclude, and cleanup commands.")
+                       \"sleep 0.1; sleep 0.2\" are not possible without a shell. Setup, prepare, conclude, \
+                       and cleanup commands always use a shell: the selected shell, or the platform \
+                       shell when this option is omitted or set to \"none\".")
         )
         .arg(
             Arg::new("default-shell")

tests/integration_tests.rs

@@ -275,7 +275,7 @@ fn fails_for_unknown_command() {
         .assert()
         .failure()
         .stderr(predicate::str::contains(
-            "Command terminated with non-zero exit code",
+            "Command 'some-nonexisting-program-b5d9574198b7e4b12a71fa4747c0a577' terminated with non-zero exit code",
         ));
 }
 
@@ -298,12 +298,15 @@ fn fails_for_failing_command_without_shell() {
     hyperfine()
         .arg("--shell=none")
         .arg("--runs=1")
-        .arg("false")
+        .arg("--command-name=named benchmark")
+        .args(["--parameter-list", "status", "0,1"])
+        .arg("test {status} -eq 0")
         .assert()
         .failure()
         .stderr(predicate::str::contains(
-            "Command terminated with non-zero exit code",
-        ));
+            "Command 'test 1 -eq 0' terminated with non-zero exit code 1 in the first benchmark run.",
+        ))
+        .stderr(predicate::str::contains("--ignore-failure"));
 }
 
 #[test]
@@ -315,9 +318,7 @@ fn fails_for_unknown_setup_command() {
         .arg("echo test")
         .assert()
         .failure()
-        .stderr(predicate::str::contains(
-            "The setup command terminated with a non-zero exit code.",
-        ));
+        .stderr(predicate::str::contains("The setup command failed:"));
 }
 
 #[test]
@@ -329,9 +330,7 @@ fn fails_for_unknown_cleanup_command() {
         .arg("echo test")
         .assert()
         .failure()
-        .stderr(predicate::str::contains(
-            "The cleanup command terminated with a non-zero exit code.",
-        ));
+        .stderr(predicate::str::contains("The cleanup command failed:"));
 }
 
 #[test]
@@ -342,9 +341,7 @@ fn fails_for_unknown_prepare_command() {
         .arg("echo test")
         .assert()
         .failure()
-        .stderr(predicate::str::contains(
-            "The preparation command terminated with a non-zero exit code.",
-        ));
+        .stderr(predicate::str::contains("The preparation command failed:"));
 }
 
 #[test]
@@ -355,9 +352,7 @@ fn fails_for_unknown_conclude_command() {
         .arg("echo test")
         .assert()
         .failure()
-        .stderr(predicate::str::contains(
-            "The conclusion command terminated with a non-zero exit code.",
-        ));
+        .stderr(predicate::str::contains("The conclusion command failed:"));
 }
 
 #[cfg(unix)]
@@ -368,7 +363,7 @@ fn can_run_failing_commands_with_ignore_failure_option() {
         .assert()
         .failure()
         .stderr(predicate::str::contains(
-            "Command terminated with non-zero exit code",
+            "Command 'false' terminated with non-zero exit code",
         ));
 
     hyperfine()
@@ -400,7 +395,7 @@ fn can_ignore_specific_exit_codes() {
         .assert()
         .failure()
         .stderr(predicate::str::contains(
-            "Command terminated with non-zero exit code 2",
+            "Command 'exit 2' terminated with non-zero exit code 2",
         ));
 }
 
@@ -441,7 +436,7 @@ fn can_ignore_multiple_exit_codes() {
         .assert()
         .failure()
         .stderr(predicate::str::contains(
-            "Command terminated with non-zero exit code 4",
+            "Command 'exit 4' terminated with non-zero exit code 4",
         ));
 }
 
@@ -477,9 +472,14 @@ fn runs_commands_using_user_defined_shell() {
         .arg("--show-output")
         .arg("--shell")
         .arg("echo 'custom_shell' '--shell-arg'")
+        .arg("--setup=echo setup")
+        .arg("--prepare=echo prepare")
+        .arg("--conclude=echo conclude")
+        .arg("--cleanup=echo cleanup")
         .arg("echo benchmark")
         .assert()
         .success()
+        .stdout(predicate::str::contains("custom_shell --shell-arg").count(5))
         .stdout(
             predicate::str::contains("custom_shell --shell-arg -c echo benchmark").or(
                 predicate::str::contains("custom_shell --shell-arg /C echo benchmark"),