From 1b3ec922facade74de71d8617b86268c89b1bbce Mon Sep 17 00:00:00 2001 From: David Peter Date: Mon, 5 Oct 2026 16:59:35 +0200 Subject: [PATCH] Export metric summaries and adapt analysis scripts --- scripts/advanced_statistics.py | 25 +-- scripts/plot_benchmark_comparison.py | 2 +- scripts/plot_histogram.py | 4 +- scripts/plot_parametrized.py | 9 +- scripts/plot_progression.py | 6 +- scripts/plot_whisker.py | 4 +- scripts/welch_ttest.py | 2 +- src/export/json.rs | 248 ++++++++++++++++++++++++++- src/quantity/mod.rs | 18 ++ 9 files changed, 292 insertions(+), 26 deletions(-) diff --git a/scripts/advanced_statistics.py b/scripts/advanced_statistics.py index 5ad3579..310bd53 100755 --- a/scripts/advanced_statistics.py +++ b/scripts/advanced_statistics.py @@ -50,11 +50,9 @@ unit_str = str(unit) with open(args.file) as f: results = json.load(f)["results"] -commands = [b["command"] for b in results] -times = [b["times"] for b in results] - -for command, ts in zip(commands, times): - ts = [t * unit.factor() for t in ts] +for result in results: + summary = result["summary"]["time_wall_clock"] + ts = [m["time_wall_clock"]["value"] * unit.factor() for m in result["measurements"]] p05 = np.percentile(ts, 5) p25 = np.percentile(ts, 25) @@ -63,13 +61,16 @@ for command, ts in zip(commands, times): iqr = p75 - p25 - print(f"Command '{command}'") - print(f" runs: {len(ts):8d}") - print(f" mean: {np.mean(ts):8.3f} {unit_str}") - print(f" stddev: {np.std(ts, ddof=1):8.3f} {unit_str}") - print(f" median: {np.median(ts):8.3f} {unit_str}") - print(f" min: {np.min(ts):8.3f} {unit_str}") - print(f" max: {np.max(ts):8.3f} {unit_str}") + print(f"Command '{result['command']}'") + print(f" runs: {summary['count']:8d}") + print(f" mean: {summary['mean'] * unit.factor():8.3f} {unit_str}") + if summary["stddev"] is None: + print(f" stddev: {'N/A':>8}") + else: + print(f" stddev: {summary['stddev'] * unit.factor():8.3f} {unit_str}") + print(f" median: {summary['median'] * unit.factor():8.3f} {unit_str}") + print(f" min: {summary['min'] * unit.factor():8.3f} {unit_str}") + print(f" max: {summary['max'] * unit.factor():8.3f} {unit_str}") print() print(" percentiles:") print(f" P_05 .. P_95: {p05:.3f} {unit_str} .. {p95:.3f} {unit_str}") diff --git a/scripts/plot_benchmark_comparison.py b/scripts/plot_benchmark_comparison.py index 48d49a8..e30f15d 100755 --- a/scripts/plot_benchmark_comparison.py +++ b/scripts/plot_benchmark_comparison.py @@ -51,7 +51,7 @@ for i, filename in enumerate(args.files): assert ( commands == benchmark_commands ), f"Unexpected commands in {filename}: {benchmark_commands}, expected: {commands}" - data.append([round(b["mean"], 2) for b in results]) + data.append([b["summary"]["time_wall_clock"]["mean"] for b in results]) if args.benchmark_names: inputs.append(args.benchmark_names[i]) else: diff --git a/scripts/plot_histogram.py b/scripts/plot_histogram.py index cc11677..4ce1bd1 100755 --- a/scripts/plot_histogram.py +++ b/scripts/plot_histogram.py @@ -67,7 +67,9 @@ if args.labels: labels = args.labels.split(",") else: labels = [b["command"] for b in results] -all_times = [b["times"] for b in results] +all_times = [ + [m["time_wall_clock"]["value"] for m in b["measurements"]] for b in results +] t_min = float(args.t_min) if args.t_min else np.min(list(map(np.min, all_times))) t_max = float(args.t_max) if args.t_max else np.max(list(map(np.max, all_times))) diff --git a/scripts/plot_parametrized.py b/scripts/plot_parametrized.py index 8ab4bc8..0db1cb6 100755 --- a/scripts/plot_parametrized.py +++ b/scripts/plot_parametrized.py @@ -71,7 +71,7 @@ def unique_parameter(benchmark): f"benchmarks must have exactly one parameter, but found multiple: {sorted(params_dict)}" ) [(name, value)] = params_dict.items() - return (name, float(value)) + return (name, float(value["value"])) parameter_name = None @@ -87,8 +87,11 @@ for filename in args.file: ) parameter_name = this_parameter_name - times_mean = [b["mean"] for b in results] - times_stddev = [b["stddev"] for b in results] + summaries = [b["summary"]["time_wall_clock"] for b in results] + times_mean = [s["mean"] for s in summaries] + times_stddev = [ + s["stddev"] if s["stddev"] is not None else float("nan") for s in summaries + ] plt.errorbar(x=parameter_values, y=times_mean, yerr=times_stddev, capsize=2) diff --git a/scripts/plot_progression.py b/scripts/plot_progression.py index 61b88eb..1932934 100755 --- a/scripts/plot_progression.py +++ b/scripts/plot_progression.py @@ -53,7 +53,7 @@ with open(args.file) as f: for result in results: label = result["command"] - times = result["times"] + times = [m["time_wall_clock"]["value"] for m in result["measurements"]] num = len(times) nums = range(num) @@ -63,7 +63,9 @@ for result in results: if not args.no_moving_average: moving_average_width = ( - num // 5 if args.moving_average_width is None else args.moving_average_width + max(1, num // 5) + if args.moving_average_width is None + else args.moving_average_width ) average = moving_average(times, moving_average_width) diff --git a/scripts/plot_whisker.py b/scripts/plot_whisker.py index 5aeb74d..ab27a23 100755 --- a/scripts/plot_whisker.py +++ b/scripts/plot_whisker.py @@ -38,10 +38,10 @@ if args.labels: labels = args.labels.split(",") else: labels = [b["command"] for b in results] -times = [b["times"] for b in results] +times = [[m["time_wall_clock"]["value"] for m in b["measurements"]] for b in results] if args.sort_by == "median": - medians = [b["median"] for b in results] + medians = [b["summary"]["time_wall_clock"]["median"] for b in results] indices = sorted(range(len(labels)), key=lambda k: medians[k]) labels = [labels[i] for i in indices] times = [times[i] for i in indices] diff --git a/scripts/welch_ttest.py b/scripts/welch_ttest.py index 6183848..4d87d8a 100755 --- a/scripts/welch_ttest.py +++ b/scripts/welch_ttest.py @@ -27,7 +27,7 @@ if len(results) != 2: sys.exit(1) a, b = (x["command"] for x in results[:2]) -X, Y = (x["times"] for x in results[:2]) +X, Y = ([m["time_wall_clock"]["value"] for m in x["measurements"]] for x in results[:2]) print(f"Command 1: {a}") print(f"Command 2: {b}\n") diff --git a/src/export/json.rs b/src/export/json.rs index 31a01ee..2abae92 100644 --- a/src/export/json.rs +++ b/src/export/json.rs @@ -1,20 +1,105 @@ +use std::ops::{Add, AddAssign, Div}; + use serde::*; use serde_json::to_vec_pretty; use super::Exporter; use crate::benchmark::benchmark_result::BenchmarkResult; +use crate::benchmark::measurement::Measurements; use crate::options::SortOrder; -use crate::quantity::TimeUnit; +use crate::quantity::statistics::{max, mean, median, min, standard_deviation, UnsafeRawValue}; +use crate::quantity::{byte, second, QuantityInUnit, Ratio, TimeUnit, Zero}; use anyhow::Result; mod metadata; use metadata::Metadata; -#[derive(Serialize, Debug)] +#[derive(Serialize)] struct HyperfineSummary<'a> { metadata: &'a Metadata, - results: &'a [BenchmarkResult], + results: Vec>, +} + +#[derive(Serialize)] +struct JsonBenchmarkResult<'a> { + #[serde(flatten)] + result: &'a BenchmarkResult, + summary: BenchmarkSummary, +} + +#[derive(Serialize)] +struct BenchmarkSummary { + time_wall_clock: StatisticalSummary, + time_user: StatisticalSummary, + time_system: StatisticalSummary, + #[serde(skip_serializing_if = "Option::is_none")] + peak_memory_usage: Option, +} + +impl BenchmarkSummary { + fn from_measurements(measurements: &Measurements) -> Self { + let values = &measurements.measurements; + let mut memory_values = values.iter().filter_map(|m| m.peak_memory_usage).peekable(); + Self { + time_wall_clock: StatisticalSummary::from_values( + values.iter().map(|m| m.time_wall_clock), + second, + ), + time_user: StatisticalSummary::from_values(values.iter().map(|m| m.time_user), second), + time_system: StatisticalSummary::from_values( + values.iter().map(|m| m.time_system), + second, + ), + peak_memory_usage: memory_values + .peek() + .is_some() + .then(|| StatisticalSummary::from_values(memory_values, byte)), + } + } +} + +#[derive(Serialize)] +struct StatisticalSummary { + unit: &'static str, + count: usize, + mean: f64, + stddev: Option, + median: f64, + min: f64, + max: f64, +} + +impl StatisticalSummary { + fn from_values(values: impl IntoIterator, _unit: U) -> Self + where + Q: Copy + + PartialOrd + + Add + + AddAssign + + Zero + + Div + + UnsafeRawValue + + QuantityInUnit, + P: Into, + U: uom::si::Unit, + { + let values: Vec<_> = values.into_iter().collect(); + assert!( + !values.is_empty(), + "Statistical summaries require at least one measurement" + ); + Self { + unit: U::singular(), + count: values.len(), + mean: mean(values.iter().copied()).value_in_unit(), + stddev: (values.len() > 1) + .then(|| standard_deviation(values.iter().copied()).value_in_unit()), + median: median(values.iter().copied()).value_in_unit(), + min: min(values.iter().copied()).value_in_unit(), + max: max(values.iter().copied()).value_in_unit(), + } + } } #[derive(Default)] @@ -32,7 +117,13 @@ impl Exporter for JsonExporter { ) -> Result> { let mut output = to_vec_pretty(&HyperfineSummary { metadata: &self.metadata, - results, + results: results + .iter() + .map(|result| JsonBenchmarkResult { + result, + summary: BenchmarkSummary::from_measurements(&result.measurements), + }) + .collect(), }); if let Ok(ref mut content) = output { content.push(b'\n'); @@ -42,6 +133,36 @@ impl Exporter for JsonExporter { } } +#[test] +fn test_statistical_summary_unit_conversion() { + use crate::quantity::{kibibyte, millisecond, Information, Time}; + + let time = StatisticalSummary::from_values( + [Time::new::(1.0), Time::new::(3.0)], + millisecond, + ); + let information = StatisticalSummary::from_values( + [ + Information::new::(1024.0), + Information::new::(3072.0), + ], + kibibyte, + ); + + for (summary, unit, scale) in [ + (time, "millisecond", 1000.0), + (information, "kibibyte", 1.0), + ] { + assert_eq!(summary.unit, unit); + assert_eq!(summary.count, 2); + approx::assert_relative_eq!(summary.mean, 2.0 * scale); + approx::assert_relative_eq!(summary.median, 2.0 * scale); + approx::assert_relative_eq!(summary.min, scale); + approx::assert_relative_eq!(summary.max, 3.0 * scale); + approx::assert_relative_eq!(summary.stddev.unwrap(), 2.0_f64.sqrt() * scale); + } +} + #[test] fn test_json_optional_memory() { use crate::benchmark::measurement::{Measurement, Measurements}; @@ -82,6 +203,17 @@ fn test_json_optional_memory() { }, "results": [{ "command": "example", + "summary": { + "time_wall_clock": { + "unit": "second", + "count": 1, + "mean": 0.0, + "stddev": null, + "median": 0.0, + "min": 0.0, + "max": 0.0 + } + }, "measurements": [{ "time_wall_clock": {"value": 0.0, "unit": "second"}, "time_user": {"value": 0.0, "unit": "second"}, @@ -90,9 +222,117 @@ fn test_json_optional_memory() { }] }] }); + for field in ["time_user", "time_system"] { + expected["results"][0]["summary"][field] = + expected["results"][0]["summary"]["time_wall_clock"].clone(); + } if let Some(memory) = expected_memory { + let value = memory["value"].clone(); + expected["results"][0]["summary"]["peak_memory_usage"] = json!({ + "unit": "byte", "count": 1, "mean": value, + "stddev": null, "median": value, "min": value, "max": value + }); expected["results"][0]["measurements"][0]["peak_memory_usage"] = memory; } assert_eq!(actual, expected); } } + +#[test] +fn test_json_summaries_for_parameterized_results() { + use crate::benchmark::benchmark_result::Parameter; + use crate::benchmark::measurement::Measurement; + use crate::quantity::{second, Time}; + use serde_json::json; + + let results: Vec<_> = vec![("small", vec![1.0, 3.0]), ("large", vec![10.0])] + .into_iter() + .map(|(size, times)| BenchmarkResult { + command: "example".into(), + parameters: [( + "size".into(), + Parameter { + value: size.into(), + is_unused: true, + }, + )] + .into(), + measurements: Measurements::new( + times + .into_iter() + .map(|time| Measurement { + time_wall_clock: Time::new::(time), + ..Measurement::default() + }) + .collect(), + ), + }) + .collect(); + let output = JsonExporter::default() + .serialize(&results, None, SortOrder::Command, None) + .unwrap(); + let actual: serde_json::Value = serde_json::from_slice(&output).unwrap(); + let results = actual["results"].as_array().unwrap(); + assert_eq!(results.len(), 2); + assert_eq!(results[0]["parameters"]["size"]["value"], "small"); + assert_eq!(results[1]["parameters"]["size"]["value"], "large"); + assert_eq!(results[0]["measurements"].as_array().unwrap().len(), 2); + assert_eq!(results[1]["measurements"].as_array().unwrap().len(), 1); + assert_eq!( + results[0]["summary"]["time_wall_clock"], + json!({ + "unit": "second", "count": 2, "mean": 2.0, + "stddev": 2.0_f64.sqrt(), "median": 2.0, "min": 1.0, "max": 3.0 + }) + ); + assert_eq!( + results[1]["summary"]["time_wall_clock"], + json!({ + "unit": "second", "count": 1, "mean": 10.0, + "stddev": null, "median": 10.0, "min": 10.0, "max": 10.0 + }) + ); +} + +#[test] +fn test_json_summary_metrics_and_missing_samples() { + use crate::benchmark::measurement::Measurement; + use crate::quantity::{byte, second, Information, Time}; + use serde_json::json; + + let measurements = Measurements::new( + vec![(1.0, Some(1024.0)), (2.0, None), (3.0, Some(3072.0))] + .into_iter() + .map(|(time, memory)| Measurement { + time_wall_clock: Time::new::(time), + time_user: Time::new::(time * 2.0), + time_system: Time::new::(time * 3.0), + peak_memory_usage: memory.map(Information::new::), + ..Measurement::default() + }) + .collect(), + ); + let actual = serde_json::to_value(BenchmarkSummary::from_measurements(&measurements)).unwrap(); + assert_eq!( + actual, + json!({ + "time_wall_clock": { + "unit": "second", "count": 3, "mean": 2.0, "stddev": 1.0, + "median": 2.0, "min": 1.0, "max": 3.0 + }, + "time_user": { + "unit": "second", "count": 3, "mean": 4.0, "stddev": 2.0, + "median": 4.0, "min": 2.0, "max": 6.0 + }, + "time_system": { + "unit": "second", "count": 3, "mean": 6.0, "stddev": 3.0, + "median": 6.0, "min": 3.0, "max": 9.0 + }, + "peak_memory_usage": { + "unit": "byte", "count": 2, "mean": 2048.0, + "stddev": 1024.0 * 2.0_f64.sqrt(), + "median": 2048.0, "min": 1024.0, "max": 3072.0 + } + }) + ); +} diff --git a/src/quantity/mod.rs b/src/quantity/mod.rs index 867ab20..8eea62b 100644 --- a/src/quantity/mod.rs +++ b/src/quantity/mod.rs @@ -16,6 +16,24 @@ pub use units::{InformationUnit, IsUnit, TimeUnit}; pub mod statistics; mod units; +pub trait QuantityInUnit { + fn value_in_unit(self) -> f64; +} + +impl> QuantityInUnit for Time { + fn value_in_unit(self) -> f64 { + self.get::() + } +} + +impl> QuantityInUnit + for Information +{ + fn value_in_unit(self) -> f64 { + self.get::() + } +} + pub trait FormatQuantity { type Unit;