From d4749b015f49bae61a2cfbe3652d515fe81ca050 Mon Sep 17 00:00:00 2001 From: Jan-Eric Nitschke <47750513+JanEricNitschke@users.noreply.github.com> Date: Sun, 23 Jun 2024 18:38:07 +0200 Subject: [PATCH] Fix prepare and conclude behaviour and add tests. --- src/benchmark/relative_speed.rs | 14 +++ src/options.rs | 16 ++- tests/execution_order_tests.rs | 197 ++++++++++++++++++++++++++++++++ tests/integration_tests.rs | 85 ++++++++++++++ 4 files changed, 306 insertions(+), 6 deletions(-) diff --git a/src/benchmark/relative_speed.rs b/src/benchmark/relative_speed.rs index b7c5a05..7449168 100644 --- a/src/benchmark/relative_speed.rs +++ b/src/benchmark/relative_speed.rs @@ -155,6 +155,20 @@ fn test_compute_relative_speed() { assert_relative_eq!(2.5, annotated_results[2].relative_speed); } +#[test] +fn test_compute_relative_speed_with_reference() { + use approx::assert_relative_eq; + + let results = vec![create_result("cmd2", 2.0), create_result("cmd3", 5.0)]; + let reference = create_result("cmd2", 4.0); + + let annotated_results = + compute_with_check_from_reference(&results, &reference, SortOrder::Command).unwrap(); + + assert_relative_eq!(2.0, annotated_results[0].relative_speed); + assert_relative_eq!(1.25, annotated_results[1].relative_speed); +} + #[test] fn test_compute_relative_speed_for_zero_times() { let results = vec![create_result("cmd1", 1.0), create_result("cmd2", 0.0)]; diff --git a/src/options.rs b/src/options.rs index 5652b0a..72b1e1b 100644 --- a/src/options.rs +++ b/src/options.rs @@ -437,21 +437,25 @@ impl Options { } pub fn validate_against_command_list(&self, commands: &Commands) -> Result<()> { + let num_commands = commands.num_commands() + + if self.reference_command.is_some() { + 1 + } else { + 0 + }; if let Some(preparation_command) = &self.preparation_command { ensure!( - preparation_command.len() <= 1 - || commands.num_commands() == preparation_command.len(), + preparation_command.len() <= 1 || num_commands == preparation_command.len(), "The '--prepare' option has to be provided just once or N times, where N is the \ - number of benchmark commands." + number of benchmark commands including a potential reference." ); } if let Some(conclusion_command) = &self.conclusion_command { ensure!( - conclusion_command.len() <= 1 - || commands.num_commands() == conclusion_command.len(), + conclusion_command.len() <= 1 || num_commands == conclusion_command.len(), "The '--conclude' option has to be provided just once or N times, where N is the \ - number of benchmark commands." + number of benchmark commands including a potential reference." ); } diff --git a/tests/execution_order_tests.rs b/tests/execution_order_tests.rs index 5404157..794cd22 100644 --- a/tests/execution_order_tests.rs +++ b/tests/execution_order_tests.rs @@ -54,6 +54,11 @@ impl ExecutionOrderTest { self.command(output) } + fn reference(&mut self, output: &str) -> &mut Self { + self.arg("--reference"); + self.command(output) + } + fn conclude(&mut self, output: &str) -> &mut Self { self.arg("--conclude"); self.command(output) @@ -364,3 +369,195 @@ fn multiple_parameter_values() { .expect_output("command 3 b") .run(); } + +#[test] +fn reference_is_executed_first() { + ExecutionOrderTest::new() + .arg("--runs=1") + .reference("reference") + .command("command 1") + .command("command 2") + .expect_output("reference") + .expect_output("command 1") + .expect_output("command 2") + .run(); +} + +#[test] +fn reference_is_executed_first_parameter_value() { + ExecutionOrderTest::new() + .arg("--runs=2") + .reference("reference") + .arg("--parameter-list") + .arg("number") + .arg("1,2,3") + .command("command {number}") + .expect_output("reference") + .expect_output("reference") + .expect_output("command 1") + .expect_output("command 1") + .expect_output("command 2") + .expect_output("command 2") + .expect_output("command 3") + .expect_output("command 3") + .run(); +} + +#[test] +fn reference_is_executed_separately_from_commands() { + ExecutionOrderTest::new() + .arg("--runs=1") + .reference("command 1") + .command("command 1") + .command("command 2") + .expect_output("command 1") + .expect_output("command 1") + .expect_output("command 2") + .run(); +} + +#[test] +fn setup_prepare_reference_conclude_cleanup_combined() { + ExecutionOrderTest::new() + .arg("--warmup=1") + .arg("--runs=2") + .setup("setup") + .prepare("prepare") + .reference("reference") + .command("command1") + .command("command2") + .conclude("conclude") + .cleanup("cleanup") + // reference + .expect_output("setup") + .expect_output("prepare") + .expect_output("reference") + .expect_output("conclude") + .expect_output("prepare") + .expect_output("reference") + .expect_output("conclude") + .expect_output("prepare") + .expect_output("reference") + .expect_output("conclude") + .expect_output("cleanup") + // 1 + .expect_output("setup") + .expect_output("prepare") + .expect_output("command1") + .expect_output("conclude") + .expect_output("prepare") + .expect_output("command1") + .expect_output("conclude") + .expect_output("prepare") + .expect_output("command1") + .expect_output("conclude") + .expect_output("cleanup") + // 2 + .expect_output("setup") + .expect_output("prepare") + .expect_output("command2") + .expect_output("conclude") + .expect_output("prepare") + .expect_output("command2") + .expect_output("conclude") + .expect_output("prepare") + .expect_output("command2") + .expect_output("conclude") + .expect_output("cleanup") + .run(); +} + +#[test] +fn setup_separete_prepare_separate_conclude_cleanup_combined() { + ExecutionOrderTest::new() + .arg("--warmup=1") + .arg("--runs=2") + .setup("setup") + .cleanup("cleanup") + .prepare("prepare1") + .command("command1") + .conclude("conclude1") + .prepare("prepare2") + .command("command2") + .conclude("conclude2") + // 1 + .expect_output("setup") + .expect_output("prepare1") + .expect_output("command1") + .expect_output("conclude1") + .expect_output("prepare1") + .expect_output("command1") + .expect_output("conclude1") + .expect_output("prepare1") + .expect_output("command1") + .expect_output("conclude1") + .expect_output("cleanup") + // 2 + .expect_output("setup") + .expect_output("prepare2") + .expect_output("command2") + .expect_output("conclude2") + .expect_output("prepare2") + .expect_output("command2") + .expect_output("conclude2") + .expect_output("prepare2") + .expect_output("command2") + .expect_output("conclude2") + .expect_output("cleanup") + .run(); +} + +#[test] +fn setup_separete_prepare_reference_separate_conclude_cleanup_combined() { + ExecutionOrderTest::new() + .arg("--warmup=1") + .arg("--runs=2") + .setup("setup") + .cleanup("cleanup") + .prepare("prepareref") + .reference("reference") + .conclude("concluderef") + .prepare("prepare1") + .command("command1") + .conclude("conclude1") + .prepare("prepare2") + .command("command2") + .conclude("conclude2") + // reference + .expect_output("setup") + .expect_output("prepareref") + .expect_output("reference") + .expect_output("concluderef") + .expect_output("prepareref") + .expect_output("reference") + .expect_output("concluderef") + .expect_output("prepareref") + .expect_output("reference") + .expect_output("concluderef") + .expect_output("cleanup") + // 1 + .expect_output("setup") + .expect_output("prepare1") + .expect_output("command1") + .expect_output("conclude1") + .expect_output("prepare1") + .expect_output("command1") + .expect_output("conclude1") + .expect_output("prepare1") + .expect_output("command1") + .expect_output("conclude1") + .expect_output("cleanup") + // 2 + .expect_output("setup") + .expect_output("prepare2") + .expect_output("command2") + .expect_output("conclude2") + .expect_output("prepare2") + .expect_output("command2") + .expect_output("conclude2") + .expect_output("prepare2") + .expect_output("command2") + .expect_output("conclude2") + .expect_output("cleanup") + .run(); +} diff --git a/tests/integration_tests.rs b/tests/integration_tests.rs index 2bbde41..3a58237 100644 --- a/tests/integration_tests.rs +++ b/tests/integration_tests.rs @@ -61,6 +61,17 @@ fn fails_with_wrong_number_of_prepare_options() { .assert() .success(); + hyperfine() + .arg("--runs=1") + .arg("--prepare=echo ref") + .arg("--prepare=echo a") + .arg("--prepare=echo b") + .arg("--reference=echo ref") + .arg("echo a") + .arg("echo b") + .assert() + .success(); + hyperfine() .arg("--runs=1") .arg("--prepare=echo a") @@ -73,6 +84,19 @@ fn fails_with_wrong_number_of_prepare_options() { .stderr(predicate::str::contains( "The '--prepare' option has to be provided", )); + + hyperfine() + .arg("--runs=1") + .arg("--prepare=echo a") + .arg("--prepare=echo b") + .arg("--reference=echo ref") + .arg("echo a") + .arg("echo b") + .assert() + .failure() + .stderr(predicate::str::contains( + "The '--prepare' option has to be provided", + )); } #[test] @@ -86,6 +110,17 @@ fn fails_with_wrong_number_of_conclude_options() { .assert() .success(); + hyperfine() + .arg("--runs=1") + .arg("--conclude=echo ref") + .arg("--conclude=echo a") + .arg("--conclude=echo b") + .arg("--reference=echo ref") + .arg("echo a") + .arg("echo b") + .assert() + .success(); + hyperfine() .arg("--runs=1") .arg("--conclude=echo a") @@ -98,6 +133,19 @@ fn fails_with_wrong_number_of_conclude_options() { .stderr(predicate::str::contains( "The '--conclude' option has to be provided", )); + + hyperfine() + .arg("--runs=1") + .arg("--conclude=echo a") + .arg("--conclude=echo b") + .arg("--reference=echo ref") + .arg("echo a") + .arg("echo b") + .assert() + .failure() + .stderr(predicate::str::contains( + "The '--conclude' option has to be provided", + )); } #[test] @@ -413,6 +461,20 @@ fn shows_benchmark_comparison_with_relative_times() { ); } +#[test] +fn shows_benchmark_comparison_relative_to_reference() { + hyperfine_debug() + .arg("--reference=sleep 2.0") + .arg("sleep 1.0") + .arg("sleep 3.0") + .assert() + .success() + .stdout( + predicate::str::contains("2.00 ± 0.00 times slower") + .and(predicate::str::contains("1.50 ± 0.00 times faster")), + ); +} + #[test] fn performs_all_benchmarks_in_parameter_scan() { hyperfine_debug() @@ -434,6 +496,29 @@ fn performs_all_benchmarks_in_parameter_scan() { ); } +#[test] +fn performs_reference_and_all_benchmarks_in_parameter_scan() { + hyperfine_debug() + .arg("--reference=sleep 25") + .arg("--parameter-scan") + .arg("time") + .arg("30") + .arg("45") + .arg("--parameter-step-size") + .arg("5") + .arg("sleep {time}") + .assert() + .success() + .stdout( + predicate::str::contains("Benchmark 1: sleep 25") + .and(predicate::str::contains("Benchmark 2: sleep 30")) + .and(predicate::str::contains("Benchmark 3: sleep 35")) + .and(predicate::str::contains("Benchmark 4: sleep 40")) + .and(predicate::str::contains("Benchmark 5: sleep 45")) + .and(predicate::str::contains("Benchmark 6: sleep 50").not()), + ); +} + #[test] fn intermediate_results_are_not_exported_to_stdout() { hyperfine_debug()