fix(bench): reject incomparable benchmark reports

Check the documented comparison conditions before printing tables, and report missing or conflicting settings without producing a misleading comparison.

Co-Authored-By: OpenAI Codex GPT-6.1 Sol <[email protected]>

docs/benchmark-report.md

@@ -69,8 +69,9 @@ known. Any field can be missing. On mains or battery is `conditions.power`.
 `render_distance`, `simulation_distance`, `window` (`[width, height]` actually rendered), `vsync`,
 `fps_cap` (absent: unlimited), `offscreen` (rendered without presenting: not a player number), `repeats`,
 `warmup_s`, `power` (`ac`, `battery`), `load_avg_1m` (before the run; above about 1 the machine was busy),
-`backend` (when forced). Readers compare reports only with equal `scenario`, distances, `window`, `vsync`,
-`fps_cap` and `offscreen`.
+`backend` (when forced). Readers compare reports only with equal `scenario`, `seed`, distances, `window`, `vsync`,
+`fps_cap` and `offscreen`. `tools/perf_bench.py --table` refuses a comparison when these values differ or a required
+value is unknown; an absent `fps_cap` means unlimited. A single report can still be shown without comparison.
 
 **`metrics`.** Scalar measurements, by name; each is a summary over the run's repeats: `n`, `mean`,
 `stdev` (sample standard deviation, null when `n` is 1), `min`, `max`, `ci95` (half-width of the 95%

tests/test_perf_bench.py

@@ -4,6 +4,78 @@ import unittest
 from tools.perf_bench import HISTOGRAM_EDGES_MS, OPTIONS, frames, rnd, summary
 
 
+class ComparisonConditionsTests(unittest.TestCase):
+    def test_rejects_incompatible_or_unknown_conditions_before_printing(self):
+        import contextlib
+        import io
+        import tempfile
+        from pathlib import Path
+        from unittest import mock
+        import tools.perf_bench as pb
+
+        conditions = {"scenario": "flight", "seed": "1234", "render_distance": 8, "simulation_distance": 12,
+                      "window": [1920, 1080], "vsync": False, "offscreen": False}
+        differences = {"scenario": "other", "seed": "5678", "simulation_distance": 8, "window": [1280, 720],
+                       "vsync": True, "offscreen": True, "fps_cap": 60}
+        with tempfile.TemporaryDirectory() as tmp:
+            root = Path(tmp)
+            reports = root / "benchmarks/perf"
+            reports.mkdir(parents=True)
+            def write(label, values):
+                (reports / f"{label}-rd8.json").write_text(
+                    json.dumps({"conditions": values, "metrics": {}}), encoding="utf-8")
+            write("first", conditions)
+            with mock.patch.object(pb, "ROOT", root):
+                for key, value in differences.items():
+                    with self.subTest(key=key):
+                        write("second", dict(conditions, **{key: value}))
+                        out = io.StringIO()
+                        with contextlib.redirect_stdout(out), self.assertRaisesRegex(ValueError, key):
+                            pb.table(["first", "second"])
+                        self.assertEqual(out.getvalue(), "")
+                incomplete = dict(conditions)
+                del incomplete["window"]
+                write("second", incomplete)
+                with contextlib.redirect_stdout(io.StringIO()), self.assertRaisesRegex(ValueError, "window"):
+                    pb.table(["first", "second"])
+                stderr, stdout = io.StringIO(), io.StringIO()
+                with mock.patch("sys.argv", ["perf_bench.py", "--table", "first", "second"]), \
+                        contextlib.redirect_stdout(stdout), contextlib.redirect_stderr(stderr), \
+                        self.assertRaises(SystemExit) as exit_error:
+                    pb.main()
+                self.assertEqual(exit_error.exception.code, 2)
+                self.assertEqual(stdout.getvalue(), "")
+                self.assertIn("window missing in second", stderr.getvalue())
+                self.assertNotIn("Traceback", stderr.getvalue())
+                write("second", dict(conditions, fps_cap=None))
+                with contextlib.redirect_stdout(io.StringIO()), self.assertRaisesRegex(ValueError, "fps_cap"):
+                    pb.table(["first", "second"])
+
+    def test_matching_reports_and_single_incomplete_report_remain_readable(self):
+        import contextlib
+        import io
+        import tempfile
+        from pathlib import Path
+        from unittest import mock
+        import tools.perf_bench as pb
+
+        conditions = {"scenario": "flight", "seed": "1234", "render_distance": 8, "simulation_distance": 12,
+                      "window": [1920, 1080], "vsync": False, "offscreen": False}
+        with tempfile.TemporaryDirectory() as tmp:
+            root = Path(tmp)
+            reports = root / "benchmarks/perf"
+            reports.mkdir(parents=True)
+            for label, values in (("first", conditions), ("second", conditions),
+                                  ("single", {"render_distance": 8})):
+                (reports / f"{label}-rd8.json").write_text(
+                    json.dumps({"conditions": values, "metrics": {}}), encoding="utf-8")
+            with mock.patch.object(pb, "ROOT", root), contextlib.redirect_stdout(io.StringIO()) as out:
+                pb.table(["first", "second"])
+                pb.table(["single"])
+            self.assertIn("| 8 |", out.getvalue())
+
+
+
 class PerfBenchStatsTests(unittest.TestCase):
     def test_lows_average_the_slowest_frames(self):
         times = [10.0] * 990 + [100.0] * 9 + [1000.0]
@@ -51,6 +123,8 @@ class PerfBenchStatsTests(unittest.TestCase):
         self.assertEqual(OPTIONS["graphicsPreset"], '"custom"')
 
 
+
+
 if __name__ == "__main__":
     unittest.main()
 

tools/perf_bench.py

@@ -891,6 +891,22 @@ def table(labels):
             r = json.loads(path.read_text())
             runs.setdefault(label, {})[r["conditions"]["render_distance"]] = r
     distances = sorted({d for by in runs.values() for d in by})
+    # The report contract requires these conditions to match before comparing runs.
+    for distance in distances:
+        present = [(label, runs[label][distance]["conditions"]) for label in labels
+                   if label in runs and distance in runs[label]]
+        if len(present) < 2:
+            continue
+        for key in ("scenario", "seed", "simulation_distance", "window", "vsync", "fps_cap", "offscreen"):
+            # Only an absent fps_cap has a defined meaning (unlimited); null is unknown.
+            missing = [label for label, c in present
+                       if c.get(key) is None and (key != "fps_cap" or key in c)]
+            if missing:
+                raise ValueError(f"cannot compare RD {distance}: {key} missing in {', '.join(missing)}")
+            baseline = present[0][1].get(key)
+            for label, c in present[1:]:
+                if c.get(key) != baseline:
+                    raise ValueError(f"cannot compare RD {distance}: {key} differs between {present[0][0]} and {label}")
     cell = lambda m, digits=0: "–" if not m else f"{m['mean']:.{digits}f}" + (f" ±{m['ci95']:.{digits}f}" if m["ci95"] else "")
     print("| RD | scene | " + " | ".join(f"{l}: avg / 1% / 0.1% low FPS" for l in labels) + " |")
     print("| --- | --- |" + " --- |" * len(labels))
@@ -946,10 +962,13 @@ def main():
                    help="horizon: measure a later start, after one start to the title screen built the caches")
     p.add_argument("--mod-jars", help="neoforge: the mod jars to load (the user's own; never committed)")
     p.add_argument("--deps", help="neoforge: their required libraries")
-    p.add_argument("--table", nargs="+", metavar="LABEL", help="print markdown tables of saved runs, and exit")
+    p.add_argument("--table", nargs="+", metavar="LABEL", help="compare saved runs with matching recorded conditions, and exit")
     args = p.parse_args()
     if args.table:
-        table(args.table)
+        try:
+            table(args.table)
+        except ValueError as error:
+            p.error(str(error))
         return
     args.label = args.label or f"{args.game}-{platform.node().split('.')[0]}"
     WORK.mkdir(parents=True, exist_ok=True)