Refine JSON schema and command metadata

README.md

@@ -158,23 +158,16 @@ You can use the `--export-markdown <file>` option to create tables like the foll
 
 #### JSON
 
-The JSON export contains a top-level `"schema_version": 2` and a `results` array with one entry
-per benchmark. Earlier exports without `schema_version` use the legacy format (version 1).
-The schema version is independent of the hyperfine release version and changes when the format
-changes incompatibly; adding optional metrics does not require a new schema version.
+The JSON export includes the following metrics for each measured run (excluding warmup runs):
 
-Each result has a `measurements` array with one object per measured run (excluding warmup runs):
-
-| Field | Format | Meaning and platform differences |
+| Metric | Unit | Meaning and platform differences |
 |:---|:---|:---|
-| `time_wall_clock` | `{ "value": number, "unit": "second" }` | Elapsed time, including time spent waiting or descheduled. Available on all platforms. |
-| `time_user` | `{ "value": number, "unit": "second" }` | CPU time spent in user mode, accumulated across threads. On Unix, includes descendants whose usage propagates through parents waiting for them. On Windows, includes processes associated with the benchmark's job object. |
-| `time_system` | `{ "value": number, "unit": "second" }` | CPU time spent in kernel mode, with the same process accounting as `time_user`. |
-| `peak_memory_usage` | `{ "value": number, "unit": "byte" }` | Peak resident set size (RSS) on Linux and macOS; omitted on Windows. This is the largest per-process peak among the command and descendants whose usage is collected by waiting for them, **not the simultaneous total memory of the process tree**. Linux's KiB counter and macOS's byte counter are both exported in bytes. On macOS, this is RSS, not Apple's separate memory-footprint metric. |
-| `exit_code` | Integer or `null` | Process exit code; `0` indicates success. On Unix, termination by a signal is encoded as `128 + signal number`. `null` means no exit code could be determined. Failed runs are included when allowed by `--ignore-failure`. |
+| `time_wall_clock` | second | Elapsed time, including time spent waiting or descheduled. Available on all platforms. |
+| `time_user` | second | CPU time spent in user mode, accumulated across threads. On Unix, includes descendants whose usage propagates through parents waiting for them. On Windows, includes processes associated with the benchmark's job object. |
+| `time_system` | second | CPU time spent in kernel mode, with the same process accounting as `time_user`. |
+| `peak_memory_usage` | byte | Peak resident set size (RSS) on Linux and macOS; omitted on Windows. This is the largest per-process peak among the command and descendants whose usage is collected by waiting for them, **not the simultaneous total memory of the process tree**. Linux's KiB counter and macOS's byte counter are both exported in bytes. On macOS, this is RSS, not Apple's separate memory-footprint metric. |
 
-Metrics unavailable on a platform are omitted, rather than reported as zero. Consumers should
-accept missing metrics and additional fields as new metrics are introduced. Time values are always
+Metrics unavailable on a platform are omitted, rather than reported as zero. Time values are always
 exported in seconds, regardless of `--time-unit`. When using a shell, the estimated shell startup
 CPU and wall-clock times are subtracted; memory usage is not adjusted.
 

scripts/advanced_statistics.py

@@ -61,7 +61,7 @@ for result in results:
 
     iqr = p75 - p25
 
-    print(f"Command '{result['command']}'")
+    print(f"Command '{result.get('name', result['command'])}'")
     print(f"  runs:   {summary['count']:8d}")
     print(f"  mean:   {summary['mean'] * unit.factor():8.3f} {unit_str}")
     if summary["stddev"] is None:

scripts/plot_benchmark_comparison.py

@@ -44,7 +44,7 @@ if args.benchmark_names:
 for i, filename in enumerate(args.files):
     with open(filename) as f:
         results = json.load(f)["results"]
-    benchmark_commands = [b["command"] for b in results]
+    benchmark_commands = [b.get("name", b["command"]) for b in results]
     if commands is None:
         commands = benchmark_commands
     else:

scripts/plot_histogram.py

@@ -66,7 +66,7 @@ with open(args.file) as f:
 if args.labels:
     labels = args.labels.split(",")
 else:
-    labels = [b["command"] for b in results]
+    labels = [b.get("name", b["command"]) for b in results]
 all_times = [
     [m["time_wall_clock"]["value"] for m in b["measurements"]] for b in results
 ]

scripts/plot_progression.py

@@ -52,7 +52,7 @@ with open(args.file) as f:
     results = json.load(f)["results"]
 
 for result in results:
-    label = result["command"]
+    label = result.get("name", result["command"])
     times = [m["time_wall_clock"]["value"] for m in result["measurements"]]
     num = len(times)
     nums = range(num)
@@ -76,7 +76,7 @@ if args.title:
 
 legend = []
 for result in results:
-    legend.append(result["command"])
+    legend.append(result.get("name", result["command"]))
     if not args.no_moving_average:
         legend.append("moving average")
 plt.legend(legend)

scripts/plot_whisker.py

@@ -37,7 +37,7 @@ with open(args.file, encoding="utf-8") as f:
 if args.labels:
     labels = args.labels.split(",")
 else:
-    labels = [b["command"] for b in results]
+    labels = [b.get("name", b["command"]) for b in results]
 times = [[m["time_wall_clock"]["value"] for m in b["measurements"]] for b in results]
 
 if args.sort_by == "median":

scripts/welch_ttest.py

@@ -26,7 +26,7 @@ if len(results) != 2:
     print("The input file has to contain exactly two benchmarks")
     sys.exit(1)
 
-a, b = (x["command"] for x in results[:2])
+a, b = (x.get("name", x["command"]) for x in results[:2])
 X, Y = ([m["time_wall_clock"]["value"] for m in x["measurements"]] for x in results[:2])
 
 print(f"Command 1: {a}")

src/benchmark/benchmark_result.rs

@@ -9,18 +9,19 @@ use crate::quantity::Time;
 #[derive(Debug, Default, Clone, Serialize, PartialEq)]
 pub struct Parameter {
     pub value: String,
+    #[serde(skip_serializing)]
     pub is_unused: bool,
 }
 
 /// Meta data and performance metrics for a single benchmark
 #[derive(Debug, Default, Clone, Serialize, PartialEq)]
 pub struct BenchmarkResult {
-    /// The display name of the command being benchmarked
+    /// The expanded command line being benchmarked
     pub command: String,
 
-    /// The expanded command line, when it differs from the display name
+    /// The display name, when it differs from the command line
     #[serde(skip_serializing_if = "Option::is_none")]
-    pub command_raw: Option<String>,
+    pub name: Option<String>,
 
     /// Performance metric measurements and exit codes for each run
     #[serde(flatten)]
@@ -32,12 +33,16 @@ pub struct BenchmarkResult {
 }
 
 impl BenchmarkResult {
+    pub fn get_name(&self) -> &str {
+        self.name.as_deref().unwrap_or(&self.command)
+    }
+
     /// The average wall clock time
     pub fn mean_wall_clock_time(&self) -> Time {
         self.measurements.time_wall_clock_mean()
     }
 
-    /// The full command line of the program that is being benchmarked, possibly including a list of
+    /// The display name of the command, possibly including a list of
     /// parameters that were not used in the command line template.
     pub fn command_with_unused_parameters(&self) -> String {
         let parameters = self
@@ -54,6 +59,6 @@ impl BenchmarkResult {
             format!(" ({parameters})")
         };
 
-        format!("{}{}", self.command, parameters)
+        format!("{}{}", self.get_name(), parameters)
     }
 }

src/benchmark/mod.rs

@@ -475,13 +475,13 @@ impl<'a> Benchmark<'a> {
 
         self.run_cleanup_command(self.command.get_parameters().iter().cloned(), output_policy)?;
 
-        let command = self.command.get_name();
-        let command_raw = self.command.get_command_line();
-        let command_raw = (command_raw != command).then_some(command_raw);
+        let command = self.command.get_command_line();
+        let name = self.command.get_name();
+        let name = (name != command).then_some(name);
 
         Ok(BenchmarkResult {
             command,
-            command_raw,
+            name,
             measurements,
             parameters: self
                 .command

src/benchmark/relative_speed.rs

@@ -158,7 +158,7 @@ fn create_result(name: &str, mean: f64) -> BenchmarkResult {
 
     BenchmarkResult {
         command: name.into(),
-        command_raw: None,
+        name: None,
         measurements: Measurements {
             measurements: vec![Measurement {
                 time_wall_clock: Time::new::<second>(mean),

src/export/csv.rs

@@ -42,7 +42,7 @@ impl Exporter for CsvExporter {
         }
 
         for res in results {
-            let mut fields = vec![Cow::Borrowed(res.command.as_bytes())];
+            let mut fields = vec![Cow::Borrowed(res.get_name().as_bytes())];
             for f in &[
                 res.mean_wall_clock_time(),
                 res.measurements.stddev().unwrap_or_default(),
@@ -80,8 +80,8 @@ fn test_csv() {
 
     let results = vec![
         BenchmarkResult {
-            command: String::from("command_a"),
-            command_raw: None,
+            command: String::from("echo command_a"),
+            name: Some(String::from("command_a")),
             measurements: Measurements::new(vec![
                 Measurement {
                     time_wall_clock: Time::new::<second>(7.0),
@@ -126,7 +126,7 @@ fn test_csv() {
         },
         BenchmarkResult {
             command: String::from("command_b"),
-            command_raw: None,
+            name: None,
             measurements: Measurements::new(vec![
                 Measurement {
                     time_wall_clock: Time::new::<second>(17.0),

src/export/json.rs

@@ -15,8 +15,11 @@ use anyhow::Result;
 mod metadata;
 use metadata::Metadata;
 
+const JSON_SCHEMA_VERSION: u32 = 2;
+
 #[derive(Serialize)]
 struct HyperfineSummary<'a> {
+    schema_version: u32,
     metadata: &'a Metadata,
     results: Vec<JsonBenchmarkResult<'a>>,
 }
@@ -116,6 +119,7 @@ impl Exporter for JsonExporter {
         _reference_index: Option<usize>,
     ) -> Result<Vec<u8>> {
         let mut output = to_vec_pretty(&HyperfineSummary {
+            schema_version: JSON_SCHEMA_VERSION,
             metadata: &self.metadata,
             results: results
                 .iter()
@@ -182,7 +186,7 @@ fn test_json_optional_memory() {
     ] {
         let result = BenchmarkResult {
             command: "example".into(),
-            command_raw: None,
+            name: None,
             measurements: Measurements::new(vec![Measurement {
                 peak_memory_usage: memory,
                 ..Measurement::default()
@@ -196,8 +200,8 @@ fn test_json_optional_memory() {
             .unwrap();
         let actual: serde_json::Value = serde_json::from_slice(&output).unwrap();
         let mut expected = json!({
+            "schema_version": JSON_SCHEMA_VERSION,
             "metadata": {
-                "json_schema_version": metadata::JSON_SCHEMA_VERSION,
                 "hyperfine_version": env!("CARGO_PKG_VERSION"),
                 "start_time": "2000-02-29T12:34:56Z",
                 "platform": exporter.metadata.platform
@@ -250,12 +254,12 @@ fn test_json_summaries_for_parameterized_results() {
         .into_iter()
         .map(|(size, times)| BenchmarkResult {
             command: "example".into(),
-            command_raw: None,
+            name: None,
             parameters: [(
                 "size".into(),
                 Parameter {
                     value: size.into(),
-                    is_unused: true,
+                    is_unused: size == "small",
                 },
             )]
             .into(),
@@ -276,8 +280,8 @@ fn test_json_summaries_for_parameterized_results() {
     let actual: serde_json::Value = serde_json::from_slice(&output).unwrap();
     let results = actual["results"].as_array().unwrap();
     assert_eq!(results.len(), 2);
-    assert_eq!(results[0]["parameters"]["size"]["value"], "small");
-    assert_eq!(results[1]["parameters"]["size"]["value"], "large");
+    assert_eq!(results[0]["parameters"]["size"], json!({"value": "small"}));
+    assert_eq!(results[1]["parameters"]["size"], json!({"value": "large"}));
     assert_eq!(results[0]["measurements"].as_array().unwrap().len(), 2);
     assert_eq!(results[1]["measurements"].as_array().unwrap().len(), 1);
     assert_eq!(

src/export/json/metadata.rs

@@ -1,10 +1,7 @@
 use serde::Serialize;
 
-pub(super) const JSON_SCHEMA_VERSION: u32 = 2;
-
 #[derive(Debug, Serialize)]
 pub(super) struct Metadata {
-    json_schema_version: u32,
     hyperfine_version: &'static str,
     #[serde(skip_serializing_if = "Option::is_none")]
     pub start_time: Option<String>,
@@ -14,7 +11,6 @@ pub(super) struct Metadata {
 impl Default for Metadata {
     fn default() -> Self {
         Self {
-            json_schema_version: JSON_SCHEMA_VERSION,
             hyperfine_version: env!("CARGO_PKG_VERSION"),
             start_time: utc_now(),
             platform: Platform::detect(),

src/export/tests.rs

@@ -28,7 +28,7 @@ fn test_markup_export_auto_ms() {
     let results = [
         BenchmarkResult {
             command: String::from("sleep 0.1"),
-            command_raw: None,
+            name: None,
             measurements: Measurements::new(vec![
                 Measurement {
                     time_wall_clock: Time::new::<second>(0.09),
@@ -56,7 +56,7 @@ fn test_markup_export_auto_ms() {
         },
         BenchmarkResult {
             command: String::from("sleep 2"),
-            command_raw: None,
+            name: None,
             measurements: Measurements::new(vec![
                 Measurement {
                     time_wall_clock: Time::new::<second>(2.0),
@@ -129,7 +129,7 @@ fn test_markup_export_auto_s() {
     let results = [
         BenchmarkResult {
             command: String::from("sleep 2"),
-            command_raw: None,
+            name: None,
             measurements: Measurements::new(vec![
                 Measurement {
                     time_wall_clock: Time::new::<second>(2.1),
@@ -157,7 +157,7 @@ fn test_markup_export_auto_s() {
         },
         BenchmarkResult {
             command: String::from("sleep 0.1"),
-            command_raw: None,
+            name: None,
             measurements: Measurements::new(vec![
                 Measurement {
                     time_wall_clock: Time::new::<second>(0.1),
@@ -230,7 +230,7 @@ fn test_markup_export_manual_ms() {
     let timing_results = [
         BenchmarkResult {
             command: String::from("sleep 2"),
-            command_raw: None,
+            name: None,
             measurements: Measurements::new(vec![
                 Measurement {
                     time_wall_clock: Time::new::<second>(2.1),
@@ -258,7 +258,7 @@ fn test_markup_export_manual_ms() {
         },
         BenchmarkResult {
             command: String::from("sleep 0.1"),
-            command_raw: None,
+            name: None,
             measurements: Measurements::new(vec![
                 Measurement {
                     time_wall_clock: Time::new::<second>(0.1),
@@ -330,7 +330,7 @@ fn test_markup_export_manual_s() {
     let results = [
         BenchmarkResult {
             command: String::from("sleep 2"),
-            command_raw: None,
+            name: None,
             measurements: Measurements::new(vec![
                 Measurement {
                     time_wall_clock: Time::new::<second>(2.01),
@@ -358,7 +358,7 @@ fn test_markup_export_manual_s() {
         },
         BenchmarkResult {
             command: String::from("sleep 0.1"),
-            command_raw: None,
+            name: None,
             measurements: Measurements::new(vec![
                 Measurement {
                     time_wall_clock: Time::new::<second>(0.11),

tests/integration_tests.rs

@@ -43,12 +43,12 @@ fn one_run_is_supported() {
 }
 
 #[test]
-fn json_includes_raw_command_only_when_different() {
+fn json_includes_name_only_when_different() {
     for (args, expected) in [
         (vec!["sleep 0.01"], vec![("sleep 0.01", None)]),
         (
             vec!["--command-name=example", "sleep 0.01"],
-            vec![("example", Some("sleep 0.01"))],
+            vec![("sleep 0.01", Some("example"))],
         ),
         (
             vec!["--command-name=sleep 0.01", "sleep 0.01"],
@@ -63,8 +63,8 @@ fn json_includes_raw_command_only_when_different() {
                 "sleep {delay}",
             ],
             vec![
-                ("delay 0.01", Some("sleep 0.01")),
-                ("delay 0.02", Some("sleep 0.02")),
+                ("sleep 0.01", Some("delay 0.01")),
+                ("sleep 0.02", Some("delay 0.02")),
             ],
         ),
         (
@@ -87,11 +87,12 @@ fn json_includes_raw_command_only_when_different() {
         let export: serde_json::Value = serde_json::from_slice(&output.stdout).unwrap();
         let results = export["results"].as_array().unwrap();
         assert_eq!(results.len(), expected.len());
-        for (result, (command, command_raw)) in results.iter().zip(expected) {
+        for (result, (command, name)) in results.iter().zip(expected) {
             assert_eq!(result["command"], command);
+            assert!(result.get("command_raw").is_none());
             assert_eq!(
-                result.get("command_raw"),
-                command_raw.map(serde_json::Value::from).as_ref()
+                result.get("name"),
+                name.map(serde_json::Value::from).as_ref()
             );
         }
     }