From bdbfae9907cafea6788282cd1cf973b794989f8e Mon Sep 17 00:00:00 2001 From: David Peter Date: Tue, 6 Oct 2026 15:50:19 +0200 Subject: [PATCH] Alternative performance metrics in the terminal (#986) --- CHANGELOG.md | 10 + README.md | 85 ++- doc/hyperfine.1 | 109 ++-- scripts/README.md | 4 +- scripts/advanced_statistics.py | 72 +-- scripts/plot_benchmark_comparison.py | 15 +- scripts/plot_histogram.py | 14 +- scripts/plot_parametrized.py | 20 +- scripts/plot_progression.py | 14 +- scripts/plot_utils.py | 66 ++- scripts/plot_whisker.py | 16 +- scripts/welch_ttest.py | 25 +- src/benchmark/benchmark_result.rs | 6 - src/benchmark/executor.rs | 3 + src/benchmark/measurement.rs | 69 +-- src/benchmark/mod.rs | 159 +++--- src/benchmark/relative_speed.rs | 237 -------- src/benchmark/scheduler.rs | 154 +----- src/cli.rs | 119 ++--- src/command.rs | 4 +- src/error.rs | 2 + src/export/asciidoc.rs | 13 +- src/export/csv.rs | 134 +++-- src/export/json.rs | 211 ++------ src/export/markup.rs | 162 +++--- src/export/mod.rs | 37 +- src/export/tests.rs | 263 ++++----- src/main.rs | 8 +- src/metric.rs | 505 ++++++++++++++++++ src/options.rs | 144 ++--- src/output/mod.rs | 1 + src/output/progress_bar.rs | 37 +- src/output/report.rs | 197 +++++++ src/output/warnings.rs | 31 +- src/quantity/mod.rs | 181 ------- src/quantity/statistics.rs | 5 - src/quantity/units.rs | 166 ------ src/timer/mod.rs | 1 + tests/execution_order_tests.rs | 17 +- tests/integration_tests.rs | 428 +++++++-------- .../integration_tests__json_export_basic.snap | 77 ++- ...n_export_parameterized_with_reference.snap | 74 ++- ...gration_tests__json_export_single_run.snap | 33 +- 43 files changed, 1872 insertions(+), 2056 deletions(-) delete mode 100644 src/benchmark/relative_speed.rs create mode 100644 src/metric.rs create mode 100644 src/output/report.rs delete mode 100644 src/quantity/units.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index a5ce7af..1b7389d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Unreleased +## Features + +- Add support for alternative performance metrics (peak memory usage, CPU cycles, instructions, cache misses, branch misses, ...). By default, hyperfine will now display wall-clock time and peak memory usage, + but users can use the new `--metrics` option to select different performance metrics. +- The terminal output now shows an overview of all measured performance metrics, including relative changes. + ## Breaking changes - Benchmarked commands are now always executed directly, without an intermediate shell (`--shell=none`). Use `-S` (an alias for `--shell=default`) to restore the previous behavior (`sh` on Unix, `cmd.exe` on Windows), or select a shell with `--shell `. @@ -46,6 +52,10 @@ ``` Times are always exported in seconds and memory in bytes. +- `--time-unit`/`-u` has been removed. Units can now be selected using the `--metrics METRIC[:UNIT],…` option, e.g. `--metrics time_wall_clock:ms,memory_peak_resident:MiB`. +- `--sort` has been removed. It complicated hyperfine significantly and doesn't make too much sense with the new output format. +- `--reference` and `--reference-name` have been removed (for now). The first command is always considered as the reference command. +- The format of Markdown, AsciiDoc, org-mode and CSV exports has also been changed. CSV now includes columns with the name and the unit of the primary metric. # v1.21.0 diff --git a/README.md b/README.md index eba7e44..6b2535f 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ A command-line benchmarking tool. * Support for arbitrary shell commands. * Constant feedback about the benchmark progress and current estimates. * Warmup runs can be executed before the actual benchmark. -* Cache-clearing commands can be set up before each timing run. +* Cache-clearing commands can be set up before each benchmark run. * Statistical outlier detection to detect interference from other programs and caching effects. * Export results to various formats: CSV, JSON, Markdown, AsciiDoc. * Parameterized benchmarks (e.g. vary the number of threads). @@ -41,11 +41,14 @@ number of runs, you can use the `-r`/`--runs` option: hyperfine --runs 5 'sleep 0.3' ``` -If you want to compare the runtimes of different programs, you can pass multiple commands: +If you want to compare different programs, you can pass multiple commands: ```sh hyperfine 'hexdump file' 'xxd file' ``` +Commands run and appear in input order. The first command is the reference for all comparisons; +each subsequent result shows its change from that reference. + ### Warmup runs and preparation commands For programs that perform a lot of disk I/O, the benchmarking results can be heavily influenced @@ -58,7 +61,7 @@ hyperfine -S --warmup 3 'grep -R TODO *' ``` Conversely, if you want to run the benchmark for a cold cache, you can use the `-p`/`--prepare` -option to run a special command before *each* timing run. For example, to clear Linux filesystem caches, +option to run a special command before *each* benchmark run. For example, to clear Linux filesystem caches, you can run ```sh sync; echo 3 | sudo tee /proc/sys/vm/drop_caches @@ -90,7 +93,7 @@ hyperfine -L compiler g++,clang++ '{compiler} -O2 main.cpp' ``` A common use case is comparing the same command across multiple Git branches. Use `--setup` -to switch branches once before each set of timing runs, so the branch switch is not part of +to switch branches once before each set of benchmark runs, so the branch switch is not part of the measured command: ```sh hyperfine \ @@ -129,7 +132,7 @@ If any of these commands need shell syntax, enable a shell explicitly. When a shell is enabled, hyperfine *corrects for the shell spawning time*. It runs the shell with an empty command multiple times to measure its startup time, then subtracts this time from each -measurement. +time measurement. ### Shell functions @@ -148,38 +151,40 @@ Otherwise, inline the function into the benchmarked command: hyperfine -S 'my_function() { sleep 1; }; my_function' ``` -### Exporting results +### Choosing metrics and units -Hyperfine has multiple options for exporting benchmark results to CSV, JSON, Markdown and other -formats (see `--help` text for details). +By default, hyperfine displays wall-clock time and memory usage +(`time_wall_clock,memory_peak_resident`). Use `--metrics` with a +comma-separated list to select a different set of metrics. Each +metric can have an optional unit after it: -#### Markdown +```sh +hyperfine \ + --metrics memory_peak_resident:MiB,time_wall_clock:ms,instructions \ + './baseline' './candidate' +``` -You can use the `--export-markdown ` option to create tables like the following: - -| Command | Mean [s] | Min [s] | Max [s] | Relative | -|:---|---:|---:|---:|---:| -| `find . -iregex '.*[0-9]\.jpg$'` | 2.275 ± 0.046 | 2.243 | 2.397 | 9.79 ± 0.22 | -| `find . -iname '*[0-9].jpg'` | 1.427 ± 0.026 | 1.405 | 1.468 | 6.14 ± 0.13 | -| `fd -HI '.*[0-9]\.jpg$'` | 0.232 ± 0.002 | 0.230 | 0.236 | 1.00 | - -#### JSON - -The JSON export includes the following metrics for each measured run (excluding warmup runs): +The following metrics are available: - **`time_wall_clock`**: Time from start to finish, including time spent waiting, in seconds. -- **`time_user`**: CPU time spent running the program's code, summed across threads, in seconds. +- **`time_user`**: CPU time spent executing application and library code in user mode, + summed across threads, in seconds. - Linux/macOS: Includes child-process time when parents wait for their children to finish. - Windows: Includes the command and its child processes. -- **`time_system`**: CPU time spent running operating-system code for the program, for example +- **`time_system`**: CPU time spent executing kernel code on the program's behalf, for example to read files, summed across threads, in seconds. - Linux/macOS: Includes child-process time when parents wait for their children to finish. - Windows: Includes the command and its child processes. +- **`time_cpu`**: Total CPU time, calculated as `time_user + time_system` for each run, in seconds. + It excludes time spent sleeping or waiting without executing. CPU time is summed across + threads, so it can exceed wall-clock time: four threads running on four cores for one second + can consume roughly four CPU-seconds. + - **`memory_peak_resident`**: Peak memory held in physical RAM, in bytes. - Linux/macOS: Peak resident set size (RSS). This is the largest per-process peak among the @@ -215,7 +220,39 @@ The JSON export includes the following metrics for each measured run (excluding - Linux: Same scope as `cpu_cycles`. - macOS/Windows: Currently not supported. -Hardware counters are not available if a `--shell` is used. +Note that hardware counters are not available if a `--shell` is used. + +Time measurements support units `ns`, `us`, `ms`, `s`, `min`, and `h`. Memory measurements +support `B`, `kB`, `MB`, `GB`, `TB`, `KiB`, `MiB`, `GiB`, and `TiB`. Hardware counters +support `count` (1), `k` (thousand), `M` (million), and `B` (billion). + +You can also use a preset to select a group of metrics: + +| Preset | Metrics | +|:---|:---| +| `--metrics=default` | `time_wall_clock,memory_peak_resident` | +| `--metrics=time` | `time_wall_clock,time_cpu,time_user,time_system` | +| `--metrics=all` | All available metrics | + +### Exporting results + +Hyperfine can export results to CSV, JSON, Markdown, AsciiDoc, and org-mode. Non-JSON formats +contain only the primary metric. + +#### Markdown + +You can use the `--export-markdown ` option to create tables like the following: + +| Command | Mean Wall Time [s] | Change | Factor | +|:---|---:|---:|:---| +| `find . -iregex '.*[0-9]\.jpg$'` | 2.275 ± 0.046 | | | +| `find . -iname '*[0-9].jpg'` | 1.427 ± 0.026 | -37.3% | (1.6x faster) | +| `fd -HI '.*[0-9]\.jpg$'` | 0.232 ± 0.002 | -89.8% | (9.8x faster) | + +#### JSON + +The JSON export includes all available metrics listed in [Choosing metrics and units](#choosing-metrics-and-units) +for each measured run (excluding warmup runs). The JSON output is useful if you want to analyze the benchmark results in more detail. The [`scripts/`](https://github.com/sharkdp/hyperfine/tree/master/scripts) folder includes a lot @@ -229,7 +266,7 @@ multiple benchmarks: ### Detailed benchmark flowchart -The following chart explains the execution order of various timing runs when using options +The following chart explains the execution order of various benchmark runs when using options like `--warmup`, `--prepare `, `--setup ` or `--cleanup `: ![](doc/execution-order.png) diff --git a/doc/hyperfine.1 b/doc/hyperfine.1 index e986238..e2b8308 100644 --- a/doc/hyperfine.1 +++ b/doc/hyperfine.1 @@ -15,10 +15,6 @@ hyperfine \- command\-line benchmarking tool .IR NUM ] .RB [ \-\-setup .IR CMD ] -.RB [ \-\-reference -.IR CMD ] -.RB [ \-\-reference\-name -.IR NAME ] .RB [ \-\-prepare .IR CMD ] .RB [ \-\-conclude @@ -38,10 +34,8 @@ hyperfine \- command\-line benchmarking tool .IR SHELL ] .RB [ \-\-style .IR TYPE ] -.RB [ \-\-sort -.IR METHOD ] -.RB [ \-\-time-unit -.IR UNIT ] +.RB [ \-\-metrics +.IR METRIC[:UNIT],...|default|time|all ] .RB [ \-\-export\-asciidoc .IR FILE ] .RB [ \-\-export\-csv @@ -75,7 +69,7 @@ A command\-line benchmarking tool which includes: * Warmup runs can be executed before the actual benchmark .RE .RS -* Cache-clearing commands can be set up before each timing run +* Cache-clearing commands can be set up before each benchmark run .RE .RS * Statistical outlier detection to detect interference from other programs and caching effects @@ -109,28 +103,14 @@ Perform exactly \fINUM\fP runs for each command. If this option is not specified .HP \fB\-s\fR, \fB\-\-setup\fR \fICMD...\fP .IP -Execute \fICMD\fP once before each set of timing runs. This is useful +Execute \fICMD\fP once before each set of benchmark runs. This is useful for compiling your software with the provided parameters, or to do any other work that should happen once before a series of benchmark runs, not every time as would happen with the \fB\-\-prepare\fR option. .HP -\fB\-\-reference\fR \fICMD\fP -.IP -Use \fICMD\fP as the reference for the relative comparison of results. Without -parameters, benchmark \fICMD\fP as a separate reference command. If this option -is not specified, the fastest command is used as the reference. -With \fB\-\-parameter\-scan\fR or \fB\-\-parameter\-list\fR, select an existing -benchmark by its exact printed name. The name must match exactly one benchmark. -.HP -\fB\-\-reference\-name\fR \fINAME\fP -.IP -Give a meaningful \fINAME\fP to the reference command. -Requires \fB\-\-reference\fR. -Cannot be used with parameterized benchmarks; use \fB\-\-command\-name\fR instead. -.HP \fB\-p\fR, \fB\-\-prepare\fR \fICMD...\fP .IP -Execute \fICMD\fP before each timing run. This is useful for clearing disk caches, +Execute \fICMD\fP before each benchmark run. This is useful for clearing disk caches, for example. The \fB\-\-prepare\fR option can be specified once for all commands or multiple times, once for each command. In the latter case, each preparation command will be @@ -138,7 +118,7 @@ run prior to the corresponding benchmark command. .HP \fB\-C\fR, \fB\-\-conclude\fR \fICMD...\fP .IP -Execute \fICMD\fP after each timing run. This is useful for clearing disk caches, +Execute \fICMD\fP after each benchmark run. This is useful for clearing disk caches, for example. The \fB\-\-conclude\fR option can be specified once for all commands or multiple times, once for each command. In the latter case, each conclusion command will be @@ -248,65 +228,62 @@ interactive output without any colors. Set this to 'color' to keep the colors without any interactive output. Set this to 'none' to disable all the output of the tool. .HP -\fB\-\-sort\fR \fIMETHOD\fP +\fB\-\-metrics\fR \fIMETRIC[:UNIT],...|default|time|all\fP .IP -Specify the sort order of the speed comparison summary and the -exported tables for markup formats (Markdown, AsciiDoc, org\-mode): +Select the metrics to display, in order, with an optional unit for each. +Default: \fBtime_wall_clock,memory_peak_resident\fR, or \fBtime_wall_clock\fR +on Windows, where peak resident memory is not yet supported. +.IP +Use \fBdefault\fR for the platform defaults, \fBtime\fR for +\fBtime_wall_clock,time_cpu,time_user,time_system\fR, or \fBall\fR for all available +metrics in the order listed below. +.IP +Time metrics \fBtime_wall_clock\fR, \fBtime_cpu\fR, \fBtime_user\fR, and \fBtime_system\fR +accept units \fBns\fR, \fBus\fR, \fBms\fR, \fBs\fR, \fBmin\fR, and \fBh\fR. +Peak memory, \fBmemory_peak_resident\fR, accepts \fBB\fR, \fBkB\fR, +\fBMB\fR, \fBGB\fR, \fBTB\fR, \fBKiB\fR, \fBMiB\fR, \fBGiB\fR, +and \fBTiB\fR. +.IP +Counter metrics \fBcpu_cycles\fR, \fBinstructions\fR, \fBcache_references\fR, +\fBcache_misses\fR, and \fBbranch_misses\fR accept \fBcount\fR (unscaled), +\fBk\fR (thousand), \fBM\fR (million), and \fBB\fR (billion). +.IP +If a unit is omitted, terminal and markup output automatically select a suitable +unit. .RS -.IP "auto (default)" -the speed comparison will be ordered by time and -the markup tables will be ordered by command (input order). -.IP "command" -order benchmarks in the way they were specified -.IP "mean\-time" -order benchmarks by mean runtime -.RE -.HP -\fB\-u\fR, \fB\-\-time\-unit\fR \fIUNIT\fP -.IP -Set the time unit to be used. If the option is not given, the time unit is -determined automatically. Possible values: +Example: .RS -.IP "microsecond" -Aliases: µs, us, microseconds. -.IP "millisecond" -Aliases: ms, milliseconds. -.IP "second" -Aliases: s, seconds. -.IP "minute" -Aliases: min, minutes. -.IP "hour" -Aliases: h, hours. +\fBhyperfine\fR \fB\-\-metrics\fR memory_peak_resident:MiB,time_wall_clock:ms,instructions:B './baseline' './candidate' +.RE .RE -.IP -This option affects the standard output as well as all export formats -except for CSV and JSON. .HP \fB\-\-export\-asciidoc\fR \fIFILE\fP .IP -Export the timing summary statistics as an AsciiDoc table to the given \fIFILE\fP. -The output time unit can be changed using the \fB\-\-time\-unit\fR option. +Export the primary metric's summary statistics as an AsciiDoc table to the given +\fIFILE\fP. Use its selected unit, or choose a unit automatically when omitted. .HP \fB\-\-export\-csv\fR \fIFILE\fP .IP -Export the timing summary statistics as CSV to the given \fIFILE\fP. If you need the -timing results for each individual run, use the JSON export format. -The output time unit is always seconds. +Export the primary metric's summary statistics as CSV to the given \fIFILE\fP, +identifying the metric and unit explicitly. Use the explicitly selected unit, +or the base unit (seconds, bytes, or unscaled counts) when omitted. .HP \fB\-\-export\-json\fR \fIFILE\fP .IP -Export the timing summary statistics and timings of individual runs as JSON to -the given \fIFILE\fP. The output time unit is always seconds. +Export summary statistics and individual samples for all collected metrics as +JSON to the given \fIFILE\fP. Include the primary metric, and always use base +units (seconds, bytes, or unscaled counts), regardless of display settings. .HP \fB\-\-export\-markdown\fR \fIFILE\fP .IP -Export the timing summary statistics as a Markdown table to the given \fIFILE\fP. -The output time unit can be changed using the \fB\-\-time\-unit\fR option. +Export the primary metric's summary statistics as a Markdown table to the given +\fIFILE\fP. Use its selected unit, or choose a unit automatically when omitted. .HP \fB\-\-export\-orgmode\fR \fIFILE\fP .IP -Export the timing summary statistics as an Emacs org\-mode table to the -given \fIFILE\fP. The output time unit can be changed using the \fB\-\-time\-unit\fR option. +Export the primary metric's summary statistics as an Emacs org\-mode table to +the given \fIFILE\fP. Use its selected unit, or choose a unit automatically +when omitted. .HP \fB\-\-show\-output\fR .IP diff --git a/scripts/README.md b/scripts/README.md index 0b635b4..fcf9c7d 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -7,9 +7,7 @@ hyperfine 'sleep 0.020' 'sleep 0.021' 'sleep 0.022' --export-json sleep.json uv run plot_whisker.py sleep.json ``` -All plotting scripts accept `--metric` to select a metric. Hardware counters -(`cpu_cycles`, `instructions`, `cache_references`, `cache_misses`, and `branch_misses`) -are plotted as unscaled counts when available in the JSON export. For example: +All plotting and analysis scripts use the JSON export's `primary_metric` by default. ```bash uv run plot_histogram.py benchmark.json --metric instructions diff --git a/scripts/advanced_statistics.py b/scripts/advanced_statistics.py index 99394eb..1a8ddda 100755 --- a/scripts/advanced_statistics.py +++ b/scripts/advanced_statistics.py @@ -7,74 +7,56 @@ # /// import argparse -import json -from enum import Enum import numpy as np - -class Unit(Enum): - SECOND = 1 - MILLISECOND = 2 - - def factor(self): - match self: - case Unit.SECOND: - return 1 - case Unit.MILLISECOND: - return 1e3 - - def __str__(self): - match self: - case Unit.SECOND: - return "s" - case Unit.MILLISECOND: - return "ms" - +from plot_utils import METRICS, add_metric_argument, load_results parser = argparse.ArgumentParser() parser.add_argument("file", help="JSON file with benchmark results") +add_metric_argument(parser) parser.add_argument( "--time-unit", - help="The unit of time.", - default="second", - action="store", + help="Display unit for time metrics (default: second).", choices=["second", "millisecond"], - dest="unit", ) args = parser.parse_args() -unit = Unit.MILLISECOND if args.unit == "millisecond" else Unit.SECOND -unit_str = str(unit) +metric, [results] = load_results(parser, [args.file], args.metric) +description = METRICS[metric] +unit, scale = description.unit, description.scale +if args.time_unit is not None: + if not metric.startswith("time_"): + parser.error("--time-unit can only be used with a time metric") + if args.time_unit == "millisecond": + unit, scale = "ms", 0.001 -with open(args.file) as f: - results = json.load(f)["results"] +def format_value(value): + suffix = f" {unit}" if unit else "" + return f"{value:.3f}{suffix}" + + +print(f"Metric: {description.label}\n") for result in results: - summary = result["summary"]["time_wall_clock"] - ts = [m["time_wall_clock"]["value"] * unit.factor() for m in result["measurements"]] + summary = result["summary"][metric] + values = [m[metric]["value"] / scale for m in result["measurements"]] - p05 = np.percentile(ts, 5) - p25 = np.percentile(ts, 25) - p75 = np.percentile(ts, 75) - p95 = np.percentile(ts, 95) + p05, p25, p75, p95 = np.percentile(values, [5, 25, 75, 95]) iqr = p75 - p25 print(f"Command '{result.get('name', result['command'])}'") print(f" runs: {summary['count']:8d}") - print(f" mean: {summary['mean'] * unit.factor():8.3f} {unit_str}") - if summary["stddev"] is None: - print(f" stddev: {'N/A':>8}") - else: - print(f" stddev: {summary['stddev'] * unit.factor():8.3f} {unit_str}") - print(f" median: {summary['median'] * unit.factor():8.3f} {unit_str}") - print(f" min: {summary['min'] * unit.factor():8.3f} {unit_str}") - print(f" max: {summary['max'] * unit.factor():8.3f} {unit_str}") + for statistic in ["mean", "stddev", "median", "min", "max"]: + value = summary[statistic] + formatted = "N/A" if value is None else format_value(value / scale) + print(f" {statistic + ':':<8}{formatted:>8}") print() print(" percentiles:") - print(f" P_05 .. P_95: {p05:.3f} {unit_str} .. {p95:.3f} {unit_str}") + print(f" P_05 .. P_95: {format_value(p05)} .. {format_value(p95)}") print( - f" P_25 .. P_75: {p25:.3f} {unit_str} .. {p75:.3f} {unit_str} (IQR = {iqr:.3f} {unit_str})" + f" P_25 .. P_75: {format_value(p25)} .. {format_value(p75)}" + f" (IQR = {format_value(iqr)})" ) print() diff --git a/scripts/plot_benchmark_comparison.py b/scripts/plot_benchmark_comparison.py index 44f66d9..a0e7459 100755 --- a/scripts/plot_benchmark_comparison.py +++ b/scripts/plot_benchmark_comparison.py @@ -14,13 +14,12 @@ Note all the input files must contain results for all commands. """ import argparse -import json import pathlib import matplotlib.pyplot as plt import numpy as np -from plot_utils import METRICS, add_metric_argument, validate_metric +from plot_utils import METRICS, add_metric_argument, load_results parser = argparse.ArgumentParser(description=__doc__) add_metric_argument(parser) @@ -34,7 +33,8 @@ parser.add_argument( parser.add_argument("-o", "--output", help="Save image to the given filename") args = parser.parse_args() -metric_label, metric_scale = METRICS[args.metric] +metric, datasets = load_results(parser, args.files, args.metric) +metric_info = METRICS[metric] commands = None data = [] @@ -45,10 +45,7 @@ if args.benchmark_names: "Number of benchmark names must match the number of input files." ) -for i, filename in enumerate(args.files): - with open(filename) as f: - results = json.load(f)["results"] - validate_metric(parser, results, args.metric) +for i, (filename, results) in enumerate(zip(args.files, datasets)): benchmark_commands = [b.get("name", b["command"]) for b in results] if commands is None: commands = benchmark_commands @@ -56,7 +53,7 @@ for i, filename in enumerate(args.files): assert commands == benchmark_commands, ( f"Unexpected commands in {filename}: {benchmark_commands}, expected: {commands}" ) - data.append([b["summary"][args.metric]["mean"] / metric_scale for b in results]) + data.append([b["summary"][metric]["mean"] / metric_info.scale for b in results]) if args.benchmark_names: inputs.append(args.benchmark_names[i]) else: @@ -79,7 +76,7 @@ ax.grid(visible=True, axis="y") if args.title: plt.title(args.title) plt.xlabel("Benchmark") -plt.ylabel(metric_label) +plt.ylabel(metric_info.axis_label) plt.legend(title="Command") if args.output: diff --git a/scripts/plot_histogram.py b/scripts/plot_histogram.py index c757ce2..46b15ce 100755 --- a/scripts/plot_histogram.py +++ b/scripts/plot_histogram.py @@ -11,12 +11,11 @@ """This program shows `hyperfine` benchmark results as a histogram.""" import argparse -import json import matplotlib.pyplot as plt import numpy as np -from plot_utils import METRICS, add_metric_argument, validate_metric +from plot_utils import METRICS, add_metric_argument, load_results parser = argparse.ArgumentParser(description=__doc__) add_metric_argument(parser) @@ -70,18 +69,15 @@ parser.add_argument( ) args = parser.parse_args() -metric_label, metric_scale = METRICS[args.metric] - -with open(args.file) as f: - results = json.load(f)["results"] -validate_metric(parser, results, args.metric) +metric, [results] = load_results(parser, [args.file], args.metric) +metric_info = METRICS[metric] if args.labels: labels = args.labels.split(",") else: labels = [b.get("name", b["command"]) for b in results] all_values = [ - [m[args.metric]["value"] / metric_scale for m in b["measurements"]] for b in results + [m[metric]["value"] / metric_info.scale for m in b["measurements"]] for b in results ] value_min = ( @@ -113,7 +109,7 @@ plt.legend( prop={"size": 10, "family": ["Source Code Pro", "Fira Mono", "Courier New"]}, ) -plt.xlabel(metric_label) +plt.xlabel(metric_info.axis_label) if args.title: plt.title(args.title) diff --git a/scripts/plot_parametrized.py b/scripts/plot_parametrized.py index 4973347..724f0b2 100755 --- a/scripts/plot_parametrized.py +++ b/scripts/plot_parametrized.py @@ -11,12 +11,11 @@ errorbar plot.""" import argparse -import json import sys import matplotlib.pyplot as plt -from plot_utils import METRICS, add_metric_argument, validate_metric +from plot_utils import METRICS, add_metric_argument, load_results parser = argparse.ArgumentParser(description=__doc__) add_metric_argument(parser) @@ -43,7 +42,8 @@ parser.add_argument( parser.add_argument("-o", "--output", help="Save image to the given filename.") args = parser.parse_args() -metric_label, metric_scale = METRICS[args.metric] +metric, datasets = load_results(parser, args.file, args.metric) +metric_info = METRICS[metric] if args.parameter_name is not None: sys.stderr.write( "warning: --parameter-name is deprecated; names are inferred from " @@ -84,11 +84,7 @@ def unique_parameter(benchmark): parameter_name = None -for filename in args.file: - with open(filename) as f: - results = json.load(f)["results"] - validate_metric(parser, results, args.metric) - +for results in datasets: (this_parameter_name, parameter_values) = extract_parameters(results) if parameter_name is not None and this_parameter_name != parameter_name: die( @@ -96,17 +92,17 @@ for filename in args.file: ) parameter_name = this_parameter_name - summaries = [b["summary"][args.metric] for b in results] - means = [s["mean"] / metric_scale for s in summaries] + summaries = [b["summary"][metric] for b in results] + means = [s["mean"] / metric_info.scale for s in summaries] stddevs = [ - s["stddev"] / metric_scale if s["stddev"] is not None else float("nan") + s["stddev"] / metric_info.scale if s["stddev"] is not None else float("nan") for s in summaries ] plt.errorbar(x=parameter_values, y=means, yerr=stddevs, capsize=2) plt.xlabel(parameter_name) -plt.ylabel(metric_label) +plt.ylabel(metric_info.axis_label) if args.log_y: plt.yscale("log") diff --git a/scripts/plot_progression.py b/scripts/plot_progression.py index e0edf67..c84c482 100755 --- a/scripts/plot_progression.py +++ b/scripts/plot_progression.py @@ -14,12 +14,11 @@ thermal throttling and similar effects. """ import argparse -import json import matplotlib.pyplot as plt import numpy as np -from plot_utils import METRICS, add_metric_argument, validate_metric +from plot_utils import METRICS, add_metric_argument, load_results def moving_average(values, num_runs): @@ -50,15 +49,12 @@ parser.add_argument( args = parser.parse_args() -metric_label, metric_scale = METRICS[args.metric] - -with open(args.file) as f: - results = json.load(f)["results"] -validate_metric(parser, results, args.metric) +metric, [results] = load_results(parser, [args.file], args.metric) +metric_info = METRICS[metric] for result in results: label = result.get("name", result["command"]) - values = [m[args.metric]["value"] / metric_scale for m in result["measurements"]] + values = [m[metric]["value"] / metric_info.scale for m in result["measurements"]] num = len(values) nums = range(num) @@ -86,7 +82,7 @@ for result in results: legend.append("moving average") plt.legend(legend) -plt.ylabel(metric_label) +plt.ylabel(metric_info.axis_label) if args.output: plt.savefig(args.output) diff --git a/scripts/plot_utils.py b/scripts/plot_utils.py index 842ded9..1eed2aa 100644 --- a/scripts/plot_utils.py +++ b/scripts/plot_utils.py @@ -1,15 +1,30 @@ -"""Shared metric options for the plotting scripts.""" +"""Shared metric selection and units for analysis and plotting scripts.""" + +import json +from typing import NamedTuple + + +class Metric(NamedTuple): + label: str + unit: str = "" + scale: float = 1 + + @property + def axis_label(self): + return f"{self.label} [{self.unit}]" if self.unit else self.label + METRICS = { - "time_wall_clock": ("Time [s]", 1), - "time_user": ("User CPU time [s]", 1), - "time_system": ("System CPU time [s]", 1), - "memory_peak_resident": ("Peak resident memory [MiB]", 1024**2), - "cpu_cycles": ("CPU cycles", 1), - "instructions": ("Completed CPU instructions", 1), - "cache_references": ("Cache references", 1), - "cache_misses": ("Cache misses", 1), - "branch_misses": ("Mispredicted branches", 1), + "time_wall_clock": Metric("Wall-clock time", "s"), + "time_cpu": Metric("Total CPU time", "s"), + "time_user": Metric("User CPU time", "s"), + "time_system": Metric("System CPU time", "s"), + "memory_peak_resident": Metric("Peak resident memory", "MiB", 1024**2), + "cpu_cycles": Metric("CPU cycles"), + "instructions": Metric("Completed CPU instructions"), + "cache_references": Metric("Cache references"), + "cache_misses": Metric("Cache misses"), + "branch_misses": Metric("Mispredicted branches"), } @@ -17,8 +32,8 @@ def add_metric_argument(parser): parser.add_argument( "--metric", choices=METRICS, - default="time_wall_clock", - help="Metric to plot (default: time_wall_clock). " + help="Metric to analyze (default: primary_metric from the JSON export, " + "or time_wall_clock if unspecified). " "Times are in seconds; memory is in MiB; hardware counters are unscaled counts.", ) @@ -30,3 +45,30 @@ def validate_metric(parser, results, metric): f"metric {metric!r} is unavailable for " f"{result.get('name', result['command'])!r}" ) + + +def load_results(parser, filenames, metric=None): + exports = [] + for filename in filenames: + with open(filename, encoding="utf-8") as f: + export = json.load(f) + if not export["results"]: + parser.error(f"no benchmark results in {filename}") + exports.append(export) + + if metric is None: + primary_metrics = { + export.get("primary_metric", "time_wall_clock") for export in exports + } + if len(primary_metrics) != 1: + parser.error( + "input files use different primary metrics; select one with --metric" + ) + metric = primary_metrics.pop() + if metric not in METRICS: + parser.error(f"unknown metric {metric!r}; select one with --metric") + + results = [export["results"] for export in exports] + for benchmarks in results: + validate_metric(parser, benchmarks, metric) + return metric, results diff --git a/scripts/plot_whisker.py b/scripts/plot_whisker.py index e53d375..d3c33b8 100755 --- a/scripts/plot_whisker.py +++ b/scripts/plot_whisker.py @@ -16,11 +16,10 @@ Quoting from the matplotlib documentation: """ import argparse -import json import matplotlib.pyplot as plt -from plot_utils import METRICS, add_metric_argument, validate_metric +from plot_utils import METRICS, add_metric_argument, load_results parser = argparse.ArgumentParser(description=__doc__) add_metric_argument(parser) @@ -33,22 +32,19 @@ parser.add_argument( parser.add_argument("-o", "--output", help="Save image to the given filename.") args = parser.parse_args() -metric_label, metric_scale = METRICS[args.metric] - -with open(args.file, encoding="utf-8") as f: - results = json.load(f)["results"] -validate_metric(parser, results, args.metric) +metric, [results] = load_results(parser, [args.file], args.metric) +metric_info = METRICS[metric] if args.labels: labels = args.labels.split(",") else: labels = [b.get("name", b["command"]) for b in results] values = [ - [m[args.metric]["value"] / metric_scale for m in b["measurements"]] for b in results + [m[metric]["value"] / metric_info.scale for m in b["measurements"]] for b in results ] if args.sort_by == "median": - medians = [b["summary"][args.metric]["median"] for b in results] + medians = [b["summary"][metric]["median"] for b in results] indices = sorted(range(len(labels)), key=lambda k: medians[k]) labels = [labels[i] for i in indices] values = [values[i] for i in indices] @@ -64,7 +60,7 @@ for patch, color in zip(boxplot["boxes"], colors): if args.title: plt.title(args.title) plt.legend(handles=boxplot["boxes"], labels=labels, loc="best", fontsize="medium") -plt.ylabel(metric_label) +plt.ylabel(metric_info.axis_label) plt.ylim(0, None) plt.xticks(list(range(1, len(labels) + 1)), labels, rotation=45) if args.output: diff --git a/scripts/welch_ttest.py b/scripts/welch_ttest.py index f6e48b3..1f76b37 100755 --- a/scripts/welch_ttest.py +++ b/scripts/welch_ttest.py @@ -7,32 +7,37 @@ # /// """This script performs Welch's t-test on a JSON export file with two -benchmark results to test for a difference in their population mean runtimes.""" +benchmark results to test for a difference in their population means.""" import argparse -import json -import sys +import math from scipy import stats +from plot_utils import METRICS, add_metric_argument, load_results + parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("file", help="JSON file with two benchmark results") +add_metric_argument(parser) args = parser.parse_args() -with open(args.file) as f: - results = json.load(f)["results"] +metric, [results] = load_results(parser, [args.file], args.metric) if len(results) != 2: - print("The input file has to contain exactly two benchmarks") - sys.exit(1) + parser.error("the input file must contain exactly two benchmarks") a, b = (x.get("name", x["command"]) for x in results[:2]) -X, Y = ([m["time_wall_clock"]["value"] for m in x["measurements"]] for x in results[:2]) +X, Y = ([m[metric]["value"] for m in x["measurements"]] for x in results[:2]) +if min(len(X), len(Y)) < 2: + parser.error("Welch's t-test requires at least two measurements per benchmark") +print(f"Metric: {METRICS[metric].label}") print(f"Command 1: {a}") print(f"Command 2: {b}\n") t, p = stats.ttest_ind(X, Y, equal_var=False) +if not math.isfinite(p): + parser.error("Welch's t-test is undefined for these measurements") th = 0.05 dispose = p < th print(f"t = {t:.3}, p = {p:.3}") @@ -40,9 +45,9 @@ print() if dispose: print( - f"A statistically significant difference in mean runtime was detected (p < {th})." + f"A statistically significant difference in mean value was detected (p < {th})." ) else: print( - f"No statistically significant difference in mean runtime was detected (p >= {th})." + f"No statistically significant difference in mean value was detected (p >= {th})." ) diff --git a/src/benchmark/benchmark_result.rs b/src/benchmark/benchmark_result.rs index 1d9f597..35bd694 100644 --- a/src/benchmark/benchmark_result.rs +++ b/src/benchmark/benchmark_result.rs @@ -3,7 +3,6 @@ use std::collections::BTreeMap; use serde::Serialize; use crate::benchmark::measurement::Measurements; -use crate::quantity::Time; /// Parameter value #[derive(Debug, Default, Clone, Serialize, PartialEq)] @@ -47,9 +46,4 @@ impl BenchmarkResult { pub fn get_name(&self) -> &str { self.name.as_deref().unwrap_or(&self.command) } - - /// The average wall clock time - pub fn mean_wall_clock_time(&self) -> Time { - self.measurements.time_wall_clock_mean() - } } diff --git a/src/benchmark/executor.rs b/src/benchmark/executor.rs index 762204b..a0e584c 100644 --- a/src/benchmark/executor.rs +++ b/src/benchmark/executor.rs @@ -220,6 +220,7 @@ impl Executor for ShellExecutor<'_> { ensure_non_negative(measurement.time_user - spawning_time.time_user); measurement.time_system = ensure_non_negative(measurement.time_system - spawning_time.time_system); + measurement.time_cpu = measurement.time_user + measurement.time_system; } Ok(measurement) @@ -278,6 +279,7 @@ impl Executor for ShellExecutor<'_> { self.shell_spawning_time = Some(Measurement { time_wall_clock: measurements.time_wall_clock_mean(), + time_cpu: measurements.time_user_mean() + measurements.time_system_mean(), time_user: measurements.time_user_mean(), time_system: measurements.time_system_mean(), memory_peak_resident: None, @@ -337,6 +339,7 @@ impl Executor for MockExecutor { Ok(Measurement { time_wall_clock: Self::extract_time(command.get_command_line()), + time_cpu: Time::zero(), time_user: Time::zero(), time_system: Time::zero(), memory_peak_resident: Some(Information::zero()), diff --git a/src/benchmark/measurement.rs b/src/benchmark/measurement.rs index d1ec75f..5c27a1f 100644 --- a/src/benchmark/measurement.rs +++ b/src/benchmark/measurement.rs @@ -2,7 +2,8 @@ use std::process::ExitStatus; use serde::Serialize; -use crate::quantity::statistics::{max, mean, median, min, modified_zscores, standard_deviation}; +use crate::metric::Metric; +use crate::quantity::statistics::{mean, modified_zscores_f64}; use crate::quantity::{serialize_information, serialize_time, Information, Time}; use crate::util::exit_code::extract_exit_code; @@ -77,6 +78,10 @@ pub struct Measurement { #[serde(serialize_with = "serialize_time")] pub time_wall_clock: Time, + /// Total CPU time (user and kernel mode) + #[serde(serialize_with = "serialize_time")] + pub time_cpu: Time, + /// Time spent in user mode #[serde(serialize_with = "serialize_time")] pub time_user: Time, @@ -131,35 +136,14 @@ impl Measurements { mean(self.wall_clock_times()) } - /// The standard deviation of all wall clock times. Not available if only one run has been performed - pub fn stddev(&self) -> Option