sharkdp/hyperfine
1
0
Fork 0
mirror of https://github.com/sharkdp/hyperfine.git synced 2026-10-12 02:22:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

Export metric summaries and adapt analysis scripts

Browse Source
This commit is contained in:
David Peter
2026-10-05 16:59:35 +02:00
parent 6d0ae4f87c
commit 1b3ec922fa
9 changed files with 292 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/advanced_statistics.py
+13 -12
View File
@@ -50,11 +50,9 @@ unit_str = str(unit)
with open(args.file) as f:
results = json.load(f)["results"]
commands = [b["command"] for b in results]
times = [b["times"] for b in results]
for command, ts in zip(commands, times):
ts = [t * unit.factor() for t in ts]
for result in results:
summary = result["summary"]["time_wall_clock"]
ts = [m["time_wall_clock"]["value"] * unit.factor() for m in result["measurements"]]
p05 = np.percentile(ts, 5)
p25 = np.percentile(ts, 25)
@@ -63,13 +61,16 @@ for command, ts in zip(commands, times):
iqr = p75 - p25
print(f"Command '{command}'")
print(f" runs: {len(ts):8d}")
print(f" mean: {np.mean(ts):8.3f} {unit_str}")
print(f" stddev: {np.std(ts, ddof=1):8.3f} {unit_str}")
print(f" median: {np.median(ts):8.3f} {unit_str}")
print(f" min: {np.min(ts):8.3f} {unit_str}")
print(f" max: {np.max(ts):8.3f} {unit_str}")
print(f"Command '{result['command']}'")
print(f" runs: {summary['count']:8d}")
print(f" mean: {summary['mean'] * unit.factor():8.3f} {unit_str}")
if summary["stddev"] is None:
print(f" stddev: {'N/A':>8}")
else:
print(f" stddev: {summary['stddev'] * unit.factor():8.3f} {unit_str}")
print(f" median: {summary['median'] * unit.factor():8.3f} {unit_str}")
print(f" min: {summary['min'] * unit.factor():8.3f} {unit_str}")
print(f" max: {summary['max'] * unit.factor():8.3f} {unit_str}")
print()
print(" percentiles:")
print(f" P_05 .. P_95: {p05:.3f} {unit_str} .. {p95:.3f} {unit_str}")
scripts/plot_benchmark_comparison.py
+1 -1
View File
@@ -51,7 +51,7 @@ for i, filename in enumerate(args.files):
assert (
commands == benchmark_commands
), f"Unexpected commands in {filename}: {benchmark_commands}, expected: {commands}"
data.append([round(b["mean"], 2) for b in results])
data.append([b["summary"]["time_wall_clock"]["mean"] for b in results])
if args.benchmark_names:
inputs.append(args.benchmark_names[i])
else:
scripts/plot_histogram.py
+3 -1
View File
@@ -67,7 +67,9 @@ if args.labels:
labels = args.labels.split(",")
else:
labels = [b["command"] for b in results]
all_times = [b["times"] for b in results]
all_times = [
[m["time_wall_clock"]["value"] for m in b["measurements"]] for b in results
]
t_min = float(args.t_min) if args.t_min else np.min(list(map(np.min, all_times)))
t_max = float(args.t_max) if args.t_max else np.max(list(map(np.max, all_times)))
scripts/plot_parametrized.py
+6 -3
View File
@@ -71,7 +71,7 @@ def unique_parameter(benchmark):
f"benchmarks must have exactly one parameter, but found multiple: {sorted(params_dict)}"
)
[(name, value)] = params_dict.items()
return (name, float(value))
return (name, float(value["value"]))
parameter_name = None
@@ -87,8 +87,11 @@ for filename in args.file:
)
parameter_name = this_parameter_name
times_mean = [b["mean"] for b in results]
times_stddev = [b["stddev"] for b in results]
summaries = [b["summary"]["time_wall_clock"] for b in results]
times_mean = [s["mean"] for s in summaries]
times_stddev = [
s["stddev"] if s["stddev"] is not None else float("nan") for s in summaries
]
plt.errorbar(x=parameter_values, y=times_mean, yerr=times_stddev, capsize=2)
scripts/plot_progression.py
+4 -2
View File
@@ -53,7 +53,7 @@ with open(args.file) as f:
for result in results:
label = result["command"]
times = result["times"]
times = [m["time_wall_clock"]["value"] for m in result["measurements"]]
num = len(times)
nums = range(num)
@@ -63,7 +63,9 @@ for result in results:
if not args.no_moving_average:
moving_average_width = (
num // 5 if args.moving_average_width is None else args.moving_average_width
max(1, num // 5)
if args.moving_average_width is None
else args.moving_average_width
)
average = moving_average(times, moving_average_width)
scripts/plot_whisker.py
+2 -2
View File
@@ -38,10 +38,10 @@ if args.labels:
labels = args.labels.split(",")
else:
labels = [b["command"] for b in results]
times = [b["times"] for b in results]
times = [[m["time_wall_clock"]["value"] for m in b["measurements"]] for b in results]
if args.sort_by == "median":
medians = [b["median"] for b in results]
medians = [b["summary"]["time_wall_clock"]["median"] for b in results]
indices = sorted(range(len(labels)), key=lambda k: medians[k])
labels = [labels[i] for i in indices]
times = [times[i] for i in indices]
scripts/welch_ttest.py
+1 -1
View File
@@ -27,7 +27,7 @@ if len(results) != 2:
sys.exit(1)
a, b = (x["command"] for x in results[:2])
X, Y = (x["times"] for x in results[:2])
X, Y = ([m["time_wall_clock"]["value"] for m in x["measurements"]] for x in results[:2])
print(f"Command 1: {a}")
print(f"Command 2: {b}\n")
src/export/json.rs
+244 -4
View File
@@ -1,20 +1,105 @@
use std::ops::{Add, AddAssign, Div};
use serde::*;
use serde_json::to_vec_pretty;
use super::Exporter;
use crate::benchmark::benchmark_result::BenchmarkResult;
use crate::benchmark::measurement::Measurements;
use crate::options::SortOrder;
use crate::quantity::TimeUnit;
use crate::quantity::statistics::{max, mean, median, min, standard_deviation, UnsafeRawValue};
use crate::quantity::{byte, second, QuantityInUnit, Ratio, TimeUnit, Zero};
use anyhow::Result;
mod metadata;
use metadata::Metadata;
#[derive(Serialize, Debug)]
#[derive(Serialize)]
struct HyperfineSummary<'a> {
metadata: &'a Metadata,
results: &'a [BenchmarkResult],
results: Vec<JsonBenchmarkResult<'a>>,
}
#[derive(Serialize)]
struct JsonBenchmarkResult<'a> {
#[serde(flatten)]
result: &'a BenchmarkResult,
summary: BenchmarkSummary,
}
#[derive(Serialize)]
struct BenchmarkSummary {
time_wall_clock: StatisticalSummary,
time_user: StatisticalSummary,
time_system: StatisticalSummary,
#[serde(skip_serializing_if = "Option::is_none")]
peak_memory_usage: Option<StatisticalSummary>,
}
impl BenchmarkSummary {
fn from_measurements(measurements: &Measurements) -> Self {
let values = &measurements.measurements;
let mut memory_values = values.iter().filter_map(|m| m.peak_memory_usage).peekable();
Self {
time_wall_clock: StatisticalSummary::from_values(
values.iter().map(|m| m.time_wall_clock),
second,
),
time_user: StatisticalSummary::from_values(values.iter().map(|m| m.time_user), second),
time_system: StatisticalSummary::from_values(
values.iter().map(|m| m.time_system),
second,
),
peak_memory_usage: memory_values
.peek()
.is_some()
.then(|| StatisticalSummary::from_values(memory_values, byte)),
}
}
}
#[derive(Serialize)]
struct StatisticalSummary {
unit: &'static str,
count: usize,
mean: f64,
stddev: Option<f64>,
median: f64,
min: f64,
max: f64,
}
impl StatisticalSummary {
fn from_values<Q, P, U>(values: impl IntoIterator<Item = Q>, _unit: U) -> Self
where
Q: Copy
+ PartialOrd
+ Add<Output = Q>
+ AddAssign
+ Zero
+ Div<Ratio, Output = P>
+ UnsafeRawValue
+ QuantityInUnit<U>,
P: Into<Q>,
U: uom::si::Unit,
{
let values: Vec<_> = values.into_iter().collect();
assert!(
!values.is_empty(),
"Statistical summaries require at least one measurement"
);
Self {
unit: U::singular(),
count: values.len(),
mean: mean(values.iter().copied()).value_in_unit(),
stddev: (values.len() > 1)
.then(|| standard_deviation(values.iter().copied()).value_in_unit()),
median: median(values.iter().copied()).value_in_unit(),
min: min(values.iter().copied()).value_in_unit(),
max: max(values.iter().copied()).value_in_unit(),
}
}
}
#[derive(Default)]
@@ -32,7 +117,13 @@ impl Exporter for JsonExporter {
) -> Result<Vec<u8>> {
let mut output = to_vec_pretty(&HyperfineSummary {
metadata: &self.metadata,
results,
results: results
.iter()
.map(|result| JsonBenchmarkResult {
result,
summary: BenchmarkSummary::from_measurements(&result.measurements),
})
.collect(),
});
if let Ok(ref mut content) = output {
content.push(b'\n');
@@ -42,6 +133,36 @@ impl Exporter for JsonExporter {
}
}
#[test]
fn test_statistical_summary_unit_conversion() {
use crate::quantity::{kibibyte, millisecond, Information, Time};
let time = StatisticalSummary::from_values(
[Time::new::<second>(1.0), Time::new::<second>(3.0)],
millisecond,
);
let information = StatisticalSummary::from_values(
[
Information::new::<byte>(1024.0),
Information::new::<byte>(3072.0),
],
kibibyte,
);
for (summary, unit, scale) in [
(time, "millisecond", 1000.0),
(information, "kibibyte", 1.0),
] {
assert_eq!(summary.unit, unit);
assert_eq!(summary.count, 2);
approx::assert_relative_eq!(summary.mean, 2.0 * scale);
approx::assert_relative_eq!(summary.median, 2.0 * scale);
approx::assert_relative_eq!(summary.min, scale);
approx::assert_relative_eq!(summary.max, 3.0 * scale);
approx::assert_relative_eq!(summary.stddev.unwrap(), 2.0_f64.sqrt() * scale);
}
}
#[test]
fn test_json_optional_memory() {
use crate::benchmark::measurement::{Measurement, Measurements};
@@ -82,6 +203,17 @@ fn test_json_optional_memory() {
},
"results": [{
"command": "example",
"summary": {
"time_wall_clock": {
"unit": "second",
"count": 1,
"mean": 0.0,
"stddev": null,
"median": 0.0,
"min": 0.0,
"max": 0.0
}
},
"measurements": [{
"time_wall_clock": {"value": 0.0, "unit": "second"},
"time_user": {"value": 0.0, "unit": "second"},
@@ -90,9 +222,117 @@ fn test_json_optional_memory() {
}]
}]
});
for field in ["time_user", "time_system"] {
expected["results"][0]["summary"][field] =
expected["results"][0]["summary"]["time_wall_clock"].clone();
}
if let Some(memory) = expected_memory {
let value = memory["value"].clone();
expected["results"][0]["summary"]["peak_memory_usage"] = json!({
"unit": "byte", "count": 1, "mean": value,
"stddev": null, "median": value, "min": value, "max": value
});
expected["results"][0]["measurements"][0]["peak_memory_usage"] = memory;
}
assert_eq!(actual, expected);
}
}
#[test]
fn test_json_summaries_for_parameterized_results() {
use crate::benchmark::benchmark_result::Parameter;
use crate::benchmark::measurement::Measurement;
use crate::quantity::{second, Time};
use serde_json::json;
let results: Vec<_> = vec![("small", vec![1.0, 3.0]), ("large", vec![10.0])]
.into_iter()
.map(|(size, times)| BenchmarkResult {
command: "example".into(),
parameters: [(
"size".into(),
Parameter {
value: size.into(),
is_unused: true,
},
)]
.into(),
measurements: Measurements::new(
times
.into_iter()
.map(|time| Measurement {
time_wall_clock: Time::new::<second>(time),
..Measurement::default()
})
.collect(),
),
})
.collect();
let output = JsonExporter::default()
.serialize(&results, None, SortOrder::Command, None)
.unwrap();
let actual: serde_json::Value = serde_json::from_slice(&output).unwrap();
let results = actual["results"].as_array().unwrap();
assert_eq!(results.len(), 2);
assert_eq!(results[0]["parameters"]["size"]["value"], "small");
assert_eq!(results[1]["parameters"]["size"]["value"], "large");
assert_eq!(results[0]["measurements"].as_array().unwrap().len(), 2);
assert_eq!(results[1]["measurements"].as_array().unwrap().len(), 1);
assert_eq!(
results[0]["summary"]["time_wall_clock"],
json!({
"unit": "second", "count": 2, "mean": 2.0,
"stddev": 2.0_f64.sqrt(), "median": 2.0, "min": 1.0, "max": 3.0
})
);
assert_eq!(
results[1]["summary"]["time_wall_clock"],
json!({
"unit": "second", "count": 1, "mean": 10.0,
"stddev": null, "median": 10.0, "min": 10.0, "max": 10.0
})
);
}
#[test]
fn test_json_summary_metrics_and_missing_samples() {
use crate::benchmark::measurement::Measurement;
use crate::quantity::{byte, second, Information, Time};
use serde_json::json;
let measurements = Measurements::new(
vec![(1.0, Some(1024.0)), (2.0, None), (3.0, Some(3072.0))]
.into_iter()
.map(|(time, memory)| Measurement {
time_wall_clock: Time::new::<second>(time),
time_user: Time::new::<second>(time * 2.0),
time_system: Time::new::<second>(time * 3.0),
peak_memory_usage: memory.map(Information::new::<byte>),
..Measurement::default()
})
.collect(),
);
let actual = serde_json::to_value(BenchmarkSummary::from_measurements(&measurements)).unwrap();
assert_eq!(
actual,
json!({
"time_wall_clock": {
"unit": "second", "count": 3, "mean": 2.0, "stddev": 1.0,
"median": 2.0, "min": 1.0, "max": 3.0
},
"time_user": {
"unit": "second", "count": 3, "mean": 4.0, "stddev": 2.0,
"median": 4.0, "min": 2.0, "max": 6.0
},
"time_system": {
"unit": "second", "count": 3, "mean": 6.0, "stddev": 3.0,
"median": 6.0, "min": 3.0, "max": 9.0
},
"peak_memory_usage": {
"unit": "byte", "count": 2, "mean": 2048.0,
"stddev": 1024.0 * 2.0_f64.sqrt(),
"median": 2048.0, "min": 1024.0, "max": 3072.0
}
})
);
}
src/quantity/mod.rs
+18
View File
@@ -16,6 +16,24 @@ pub use units::{InformationUnit, IsUnit, TimeUnit};
pub mod statistics;
mod units;
pub trait QuantityInUnit<U> {
fn value_in_unit(self) -> f64;
}
impl<U: uom::si::time::Unit + uom::Conversion<f64, T = f64>> QuantityInUnit<U> for Time {
fn value_in_unit(self) -> f64 {
self.get::<U>()
}
}
impl<U: uom::si::information::Unit + uom::Conversion<f64, T = f64>> QuantityInUnit<U>
for Information
{
fn value_in_unit(self) -> f64 {
self.get::<U>()
}
}
pub trait FormatQuantity {
type Unit;