diff --git a/README.md b/README.md index afe7147..827972d 100644 --- a/README.md +++ b/README.md @@ -155,6 +155,31 @@ Otherwise, inline the function into the benchmarked command: hyperfine -S 'my_function() { sleep 1; }; my_function' ``` +### Environment variables + +Use `--env` to set environment variables for benchmark commands and their respective setup, prepare, conclude, +and cleanup commands. Assignments are space-separated and support shell-style quoting, +but no shell expansion: + +```sh +hyperfine --env 'OMP_NUM_THREADS=8 CFLAGS="-O3 -march=native"' 'my_command' +``` + +Specify once for all commands or once per benchmark command: + +```sh +hyperfine \ + --env 'OMP_NUM_THREADS=8' './benchmark-cpp' \ + --env 'RAYON_NUM_THREADS=8' './benchmark-rust' +``` + +Use an empty `--env=''` for no overrides. Values can contain parameter placeholders. +For example, the following benchmarks `my_command` with `OMP_NUM_THREADS` ranging from 1 to 8: + +```sh +hyperfine -P threads 1 8 --env 'OMP_NUM_THREADS={threads}' 'my_command' +``` + ### Choosing metrics and units By default, hyperfine displays wall-clock time and memory usage diff --git a/doc/hyperfine.1 b/doc/hyperfine.1 index 8ef77d5..e4ded43 100644 --- a/doc/hyperfine.1 +++ b/doc/hyperfine.1 @@ -192,6 +192,17 @@ This performs benchmarks for 'g++ \-O2 main.cpp' and 'clang++ \-O2 main.cpp'. The option can be specified multiple times to run benchmarks for all possible parameter combinations. .HP +\fB\-\-env\fR \fIASSIGNMENTS\fP +.IP +Set environment variables for benchmark commands and their respective setup, prepare, conclude, +and cleanup commands. Assignments are space-separated and support shell-style quoting, +but no shell expansion, e.g. \fB\-\-env\fR 'OMP_NUM_THREADS=8 CFLAGS="\-O3 \-march=native"'. +Specify once for all commands or once per benchmark command, like \fB\-\-prepare\fR. +Use an empty string for no overrides. Values can contain parameter placeholders. +.IP +For example, \fBhyperfine \-P threads 1 8 \-\-env\fR 'OMP_NUM_THREADS={threads}' 'my_command' +benchmarks 'my_command' with OMP_NUM_THREADS ranging from 1 to 8. +.HP \fB\-\-shell\fR \fISHELL\fP .IP Set the shell to use for executing benchmarked commands, including warmup runs diff --git a/src/benchmark/benchmark_result.rs b/src/benchmark/benchmark_result.rs index 35bd694..3ac3d41 100644 --- a/src/benchmark/benchmark_result.rs +++ b/src/benchmark/benchmark_result.rs @@ -18,15 +18,13 @@ pub struct BenchmarkResult { /// regardless of any custom name. pub command: String, - /// The custom name after parameter substitution, if it differs from `command`. - /// For example, `--command-name="wait {duration}s"` with `duration=1` - /// produces `Some("wait 1s")`. Without a custom name, or when the expanded - /// name equals `command`, this is `None`. + /// The custom or automatically generated name after parameter substitution, + /// if it differs from `command`. Automatic names include environment overrides. #[serde(skip_serializing_if = "Option::is_none")] pub name: Option, - /// The custom name (or `command`), followed by parameters not used in the - /// command template, in command-line order. For example, `sleep {duration}` + /// The name, followed by parameters not used in the command or environment + /// templates, in command-line order. For example, `sleep {duration}` /// with `duration=1`, `branch=main`, and the custom name `wait {duration}s` /// produces `wait 1s (branch = main)`. Without the custom name, this is /// `sleep 1 (branch = main)`. Used in summaries and table exports. @@ -40,6 +38,10 @@ pub struct BenchmarkResult { /// Parameter values for this benchmark #[serde(skip_serializing_if = "BTreeMap::is_empty")] pub parameters: BTreeMap, + + /// Explicit environment overrides after parameter substitution. + #[serde(skip_serializing_if = "BTreeMap::is_empty")] + pub environment: BTreeMap, } impl BenchmarkResult { diff --git a/src/benchmark/executor.rs b/src/benchmark/executor.rs index cc24d88..23d239b 100644 --- a/src/benchmark/executor.rs +++ b/src/benchmark/executor.rs @@ -215,6 +215,7 @@ impl Executor for ShellExecutor<'_> { ) -> Result { let on_windows_cmd = cfg!(windows) && *self.shell == Shell::Default("cmd.exe"); let mut command_builder = self.shell.command(); + command_builder.envs(command.get_environment()); command_builder.arg(if on_windows_cmd { "/C" } else { "-c" }); // Windows needs special treatment for its behavior on parsing cmd arguments diff --git a/src/benchmark/mod.rs b/src/benchmark/mod.rs index 251b1f1..dd01985 100644 --- a/src/benchmark/mod.rs +++ b/src/benchmark/mod.rs @@ -61,8 +61,9 @@ impl<'a> Benchmark<'a> { output_policy: &CommandOutputPolicy, iteration: executor::BenchmarkIteration, ) -> Result { + let command = command.clone().with_environment_from(self.command); self.executor - .run_intermediate_command_and_measure(command, iteration, output_policy) + .run_intermediate_command_and_measure(&command, iteration, output_policy) .map_err(|error| { let hint = if cfg!(windows) { "Append ' || exit /b 0' to the command if this failure can be ignored." @@ -443,6 +444,11 @@ impl<'a> Benchmark<'a> { Ok(BenchmarkResult { command, name, + environment: self + .command + .get_environment() + .map(|(name, value)| (name.to_owned(), value)) + .collect(), display_name: self.command.get_name_with_unused_parameters(), measurements, parameters: self diff --git a/src/cli.rs b/src/cli.rs index 0f714d9..e05bbdf 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -119,6 +119,28 @@ fn build_command() -> Command { .help("Perform exactly NUM runs for each command. If this option is not specified, \ hyperfine automatically determines the number of runs."), ) + .arg( + Arg::new("env") + .long("env") + .action(ArgAction::Append) + .num_args(1) + .value_name("ASSIGNMENTS") + .help("Set environment variables for benchmark commands and their respective setup, \ + prepare, conclude, and cleanup commands. Assignments are space-separated and \ + support shell-style quoting, but no shell expansion:\n\n \ + hyperfine \\\n \ + --env 'OMP_NUM_THREADS=8 CFLAGS=\"-O3 -march=native\"' \\\n \ + 'my_command'\n\n\ + Specify once for all commands or once per benchmark command:\n\n \ + hyperfine \\\n \ + --env 'OMP_NUM_THREADS=8' './benchmark-cpp' \\\n \ + --env 'RAYON_NUM_THREADS=8' './benchmark-rust'\n\n\ + Use an empty --env='' for no overrides. Values can contain parameter placeholders. \ + For example, the following benchmarks 'my_command' with OMP_NUM_THREADS ranging \ + from 1 to 8:\n\n \ + hyperfine -P threads 1 8 \\\n \ + --env 'OMP_NUM_THREADS={threads}' 'my_command'"), + ) .arg( Arg::new("setup") .long("setup") diff --git a/src/command.rs b/src/command.rs index eefe711..f694b45 100644 --- a/src/command.rs +++ b/src/command.rs @@ -14,9 +14,11 @@ use crate::{ use clap::{parser::ValuesRef, ArgMatches}; -use anyhow::{bail, Context, Result}; +use anyhow::{bail, ensure, Context, Result}; use rust_decimal::Decimal; +type Environment = Vec<(String, String)>; + /// A command that should be benchmarked. #[derive(Debug, Clone, PartialEq, Eq)] pub struct Command<'a> { @@ -28,6 +30,9 @@ pub struct Command<'a> { /// Zero or more parameter values. parameters: Vec>, + + /// Environment overrides, before parameter substitution. + environment: Environment, } impl<'a> Command<'a> { @@ -36,6 +41,7 @@ impl<'a> Command<'a> { name, expression, parameters: Vec::new(), + environment: Vec::new(), } } @@ -48,14 +54,26 @@ impl<'a> Command<'a> { name, expression, parameters: parameters.into_iter().collect(), + environment: Vec::new(), } } pub fn get_name(&self) -> String { - self.name.map_or_else( - || self.get_command_line(), - |name| self.replace_parameters_in(name), - ) + match self.name { + Some(name) => self.replace_parameters_in(name), + None => { + let environment = self + .get_environment() + .map(|(name, value)| format!("{name}={}", shell_words::quote(&value))) + .collect::>() + .join(" "); + if environment.is_empty() { + self.get_command_line() + } else { + format!("{environment} {}", self.get_command_line()) + } + } + } } pub fn get_name_with_unused_parameters(&self) -> String { @@ -87,6 +105,7 @@ impl<'a> Command<'a> { if let Some(program_name) = tokens.next() { let mut command_builder = std::process::Command::new(program_name); command_builder.args(tokens); + command_builder.envs(self.get_environment()); Ok(command_builder) } else { bail!("Can not execute empty command") @@ -97,8 +116,24 @@ impl<'a> Command<'a> { &self.parameters } + pub fn get_environment(&self) -> impl Iterator { + self.environment + .iter() + .map(move |(name, value)| (name.as_str(), self.replace_parameters_in(value))) + } + + pub fn with_environment_from(mut self, command: &Command<'_>) -> Self { + self.environment = command.environment.clone(); + self + } + pub fn is_parameter_unused(&self, parameter: &str) -> bool { - !self.expression.contains(&format!("{{{parameter}}}")) + let placeholder = format!("{{{parameter}}}"); + !self.expression.contains(&placeholder) + && !self + .environment + .iter() + .any(|(_, value)| value.contains(&placeholder)) } pub fn get_unused_parameters(&self) -> impl Iterator { @@ -137,8 +172,58 @@ impl<'a> Command<'a> { /// A collection of commands that should be benchmarked pub struct Commands<'a>(Vec>); +fn parse_environment(value: &str) -> Result { + let mut environment = Environment::new(); + for assignment in shell_words::split(value).context("Failed to parse '--env' assignments")? { + let (name, value) = assignment.split_once('=').with_context(|| { + format!( + "Invalid value '{assignment}' for '--env': expected an assignment like NAME=value or NAME=\"quoted value\"." + ) + })?; + ensure!( + !name.is_empty() + && name.bytes().enumerate().all(|(index, byte)| { + byte == b'_' + || byte.is_ascii_alphabetic() + || (index > 0 && byte.is_ascii_digit()) + }), + "Invalid environment variable name '{name}' in '--env'" + ); + // Keep only the last assignment, including case variants on Windows. + environment.retain(|(key, _)| { + if cfg!(windows) { + !key.eq_ignore_ascii_case(name) + } else { + key != name + } + }); + environment.push((name.to_owned(), value.to_owned())); + } + Ok(environment) +} + impl<'a> Commands<'a> { pub fn from_cli_arguments(matches: &'a ArgMatches) -> Result> { + let mut commands = Self::build_from_cli_arguments(matches)?; + if let Some(values) = matches.get_many::("env") { + let environments = values + .map(|value| parse_environment(value)) + .collect::>>()?; + let num_commands = commands.num_commands(); + ensure!( + environments.len() == 1 || environments.len() == num_commands, + "The '--env' option has to be provided just once or N times, where N={num_commands} is the \ + number of benchmark commands." + ); + for (index, command) in commands.0.iter_mut().enumerate() { + command.environment = + environments[if environments.len() == 1 { 0 } else { index }].clone(); + } + } + Ok(commands) + } + + fn build_from_cli_arguments(matches: &'a ArgMatches) -> Result> { let command_names = matches.get_many::("command-name"); let command_strings = matches .get_many::("command") diff --git a/src/export/csv.rs b/src/export/csv.rs index f42bb1b..7ebdde4 100644 --- a/src/export/csv.rs +++ b/src/export/csv.rs @@ -91,6 +91,7 @@ fn test_csv() { let results = vec![ BenchmarkResult { + environment: BTreeMap::new(), command: String::from("echo command_a"), name: Some(String::from("command_a")), display_name: String::from("command_a"), @@ -141,6 +142,7 @@ fn test_csv() { }, }, BenchmarkResult { + environment: BTreeMap::new(), command: String::from("command_b"), name: None, display_name: String::from("command_b"), diff --git a/src/export/tests.rs b/src/export/tests.rs index 3191c5c..b3f70a8 100644 --- a/src/export/tests.rs +++ b/src/export/tests.rs @@ -35,6 +35,7 @@ fn get_output(results: &[BenchmarkResult], unit: Option Result<()> { { for command in commands.iter() { let command_line = command.get_command_line(); + if shell_syntax::starts_with_assignment(&command_line) { + bail!( + "Cannot run '{command_line}' directly because it starts with a variable assignment.\n\ + Use the --env option to set environment variables, for example:\n\n \ + hyperfine --env 'OMP_NUM_THREADS=8 CFLAGS=\"-O3 -march=native\"' 'my_command'\n\n\ + Alternatively, use -S / --shell=default to run the command in a shell." + ); + } if let Some(operator) = shell_syntax::first_unquoted_operator(&command_line) { bail!( concat!( diff --git a/src/shell_syntax.rs b/src/shell_syntax.rs index caba560..043b8b5 100644 --- a/src/shell_syntax.rs +++ b/src/shell_syntax.rs @@ -1,8 +1,32 @@ -//! Conservative detection of shell operators in commands run without a shell. +//! Conservative detection of shell syntax in commands run without a shell. use std::iter::Peekable; use std::str::CharIndices; +/// Detects the variable assignment at the start of a command like +/// `OMP_NUM_THREADS=8 my_command`. +pub fn starts_with_assignment(command: &str) -> bool { + let Some(token) = tokenize(command).next() else { + return false; + }; + let mut bytes = token.bytes(); + if !bytes + .next() + .is_some_and(|byte| byte.is_ascii_alphabetic() || byte == b'_') + { + return false; + } + for byte in bytes { + if byte == b'=' { + return true; + } + if !byte.is_ascii_alphanumeric() && byte != b'_' { + return false; + } + } + false +} + /// Find the first standalone operator that is neither quoted nor escaped. /// /// Operators attached to another word (such as `hello>out`), expansions, and @@ -90,7 +114,30 @@ impl<'a> Iterator for Tokens<'a> { #[cfg(test)] mod tests { - use super::{first_unquoted_operator, tokenize}; + use super::{first_unquoted_operator, starts_with_assignment, tokenize}; + + #[test] + fn detects_leading_assignments() { + assert!(starts_with_assignment("A=1 command")); + assert!(starts_with_assignment("A=$HOME command")); + assert!(starts_with_assignment("A= command")); + assert!(starts_with_assignment("_A2='hello world' B=2 command")); + assert!(starts_with_assignment("A=1")); + assert!(starts_with_assignment(" # comment\n A=1 command")); + + assert!(!starts_with_assignment("")); + assert!(!starts_with_assignment("# A=1 command")); + assert!(!starts_with_assignment("command A=1")); + assert!(!starts_with_assignment("env A=1 command")); + assert!(!starts_with_assignment("'A=1' command")); + assert!(!starts_with_assignment("\"A\"=1 command")); + assert!(!starts_with_assignment(r"A\=1 command")); + assert!(!starts_with_assignment("./A=1 command")); + assert!(!starts_with_assignment("1A=1 command")); + assert!(!starts_with_assignment("A-B=1 command")); + assert!(!starts_with_assignment("=1 command")); + assert!(!starts_with_assignment("A")); + } #[test] fn tokenizer_preserves_words_quotes_and_escapes() { diff --git a/tests/integration_tests.rs b/tests/integration_tests.rs index b4574e9..ce86ced 100644 --- a/tests/integration_tests.rs +++ b/tests/integration_tests.rs @@ -53,6 +53,66 @@ fn one_run_is_supported() { .success(); } +#[test] +fn environment_basic_usage() { + let command = if cfg!(windows) { + "echo %HYPERFINE_TEST%" + } else { + "echo \"$HYPERFINE_TEST\"" + }; + hyperfine() + .args(["--shell=default", "--runs=1", "--show-output"]) + .arg("--command-name=env-test") + .args(["--env", r#"HYPERFINE_TEST="hello, world""#]) + .arg(command) + .assert() + .success() + .stdout(predicate::str::contains("hello, world")); +} + +#[test] +fn environment_parameter_expansion() { + let command = if cfg!(windows) { + "echo %HYPERFINE_TEST%" + } else { + "echo \"$HYPERFINE_TEST\"" + }; + hyperfine() + .args(["--shell=default", "--runs=1", "--show-output"]) + .arg("--command-name=env-test") + .args(["-P", "value", "1", "2"]) + .args(["--env", "HYPERFINE_TEST=hello-{value}"]) + .arg(command) + .assert() + .success() + .stdout(predicate::str::contains("hello-1")) + .stdout(predicate::str::contains("hello-2")); +} + +#[test] +fn rejects_invalid_environment() { + hyperfine() + .args(["--env", "OMP_NUM_THREADS"]) + .arg("my_command") + .assert() + .failure() + .stderr(predicate::str::contains( + "Invalid value 'OMP_NUM_THREADS' for '--env': expected an assignment like NAME=value or NAME=\"quoted value\".", + )); +} + +#[test] +fn requires_explicit_shell_for_environment_assignments() { + // Without an explicit shell choice, report the leading variable assignment. + hyperfine() + .args(["--runs=1", "SOME_ENV_VAR=$HOME my_command"]) + .assert() + .failure() + .stderr(predicate::str::contains( + "starts with a variable assignment", + )); +} + /// Regression test: hyperfine must not panic when writing to a closed /// stdout pipe (e.g. `hyperfine ... | head -n 1` or a downstream process /// that terminates early). A `BrokenPipe` error should result in a quiet