mirror of
https://github.com/lemonade-sdk/thenoise.git
synced 2026-11-22 00:55:11 -10:00
214 lines
9.3 KiB
Python
214 lines
9.3 KiB
Python
#!/usr/bin/env python
|
|
"""Say what actually moved between two bench snapshots.
|
|
|
|
``block_bench.py`` and ``vae_bench.py`` photograph; this develops a pair of photographs
|
|
and hands back the rows that changed. Nothing is measured here, so it runs anywhere, in
|
|
seconds, on two JSON files, and it takes either tool's snapshots — they record the same
|
|
fields per case, so one diff reads DiT blocks and one reads VAE codecs. What it will not
|
|
do is compare a snapshot of one tool against the other.
|
|
|
|
The design is all about noise. Every case carries its own scatter (``groups``, or
|
|
``cov_pct`` in a schema-1 snapshot), so a row only counts as slower or faster when the
|
|
delta clears ``--min`` sigmas of the two runs' own spread. Everything below that is
|
|
counted, not printed: the useful answer from comparing two equal runs is the sentence
|
|
"nothing moved", not 75 rows of decimals that each say ±0.3%.
|
|
|
|
The headers are compared first. Two snapshots taken with a different ladder, torch
|
|
build, dtype or ``--refs`` are not the same experiment, and the deltas between them are
|
|
decoration — so they are still printed, under a warning that says as much. The same test
|
|
runs per case, since ``--dtype auto`` lets one codec change dtype between snapshots: that
|
|
row is a different experiment, so it is listed apart and counted nowhere.
|
|
|
|
Usage
|
|
-----
|
|
.venv/bin/python bench-scripts/diff_bench.py before.json after.json
|
|
.venv/bin/python bench-scripts/diff_bench.py a.json b.json --all # every row
|
|
.venv/bin/python bench-scripts/diff_bench.py a.json b.json --min 2 # looser test
|
|
.venv/bin/python bench-scripts/diff_bench.py a.json b.json --gib 5 # memory flag
|
|
|
|
Exit code is 1 when a case got slower or one that used to work now fails. A header
|
|
difference alone does not fail: those numbers often are still comparable.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
import math
|
|
import statistics
|
|
import sys
|
|
|
|
SIGMA_FLOOR = 0.005 # a groups set with zero scatter would flag a 1 ns change
|
|
MIN_COV_FLOOR = 0.1 # ditto for a snapshot that recorded cov_pct == 0.0
|
|
TOOLS = ("block_bench", "vae_bench") # the snapshot layouts this can read
|
|
|
|
|
|
def load(path: str) -> dict:
|
|
try:
|
|
with open(path) as fh:
|
|
snap = json.load(fh)
|
|
except (OSError, ValueError) as exc:
|
|
raise SystemExit(f"{path}: {exc}")
|
|
if snap.get("tool") not in TOOLS or not isinstance(snap.get("cases"), list):
|
|
raise SystemExit(f"{path}: not a {' or '.join(TOOLS)} snapshot")
|
|
return snap
|
|
|
|
|
|
def flat(section: dict, name: str) -> dict:
|
|
"""One level deep, so ``env.vars`` differs per variable rather than per dict."""
|
|
out = {}
|
|
for key, value in section.items():
|
|
if isinstance(value, dict):
|
|
out.update({f"{name}.{key}.{k}": v for k, v in value.items()})
|
|
else:
|
|
out[f"{name}.{key}"] = value
|
|
return out
|
|
|
|
|
|
def shown(value) -> str:
|
|
text = ",".join(map(str, value)) if isinstance(value, (list, tuple)) else str(value)
|
|
return text if len(text) <= 44 else text[:41] + "..."
|
|
|
|
|
|
def header_diff(a: dict, b: dict) -> list[tuple[str, str, str]]:
|
|
"""Everything the two runs recorded about themselves that is not the same."""
|
|
left, right = {}, {}
|
|
for section in ("git", "env", "protocol"):
|
|
left.update(flat(a.get(section) or {}, section))
|
|
right.update(flat(b.get(section) or {}, section))
|
|
if a.get("schema") != b.get("schema"):
|
|
left["schema"], right["schema"] = a.get("schema"), b.get("schema")
|
|
out = []
|
|
for key in sorted(set(left) | set(right)):
|
|
va, vb = left.get(key, "-"), right.get(key, "-")
|
|
if va != vb:
|
|
out.append((key, shown(va), shown(vb)))
|
|
return out
|
|
|
|
|
|
def floor_pct(a: dict, b: dict, k: float) -> float:
|
|
"""Percent change this pair of cases could not have told apart from its own noise."""
|
|
ga, gb = a.get("groups"), b.get("groups")
|
|
if ga and gb and len(ga) > 1 and len(gb) > 1:
|
|
ma, mb = statistics.fmean(ga), statistics.fmean(gb)
|
|
sa = max(statistics.pstdev(ga), SIGMA_FLOOR * ma)
|
|
sb = max(statistics.pstdev(gb), SIGMA_FLOOR * mb)
|
|
return 100 * k * math.sqrt(sa * sa / len(ga) + sb * sb / len(gb)) / ma
|
|
return max(k * max(a.get("cov_pct", 0.0), b.get("cov_pct", 0.0)), MIN_COV_FLOOR)
|
|
|
|
|
|
def main() -> None:
|
|
ap = argparse.ArgumentParser(description="Diff two bench snapshots.")
|
|
ap.add_argument("before")
|
|
ap.add_argument("after")
|
|
ap.add_argument("--min", type=float, default=3.0, dest="sigma", metavar="K",
|
|
help="sigmas of the two runs' spread a delta must clear "
|
|
"(default %(default)s)")
|
|
ap.add_argument("--gib", type=float, default=10.0, metavar="PCT",
|
|
help="report peak-memory changes over this %% (default %(default)s)")
|
|
ap.add_argument("--all", action="store_true",
|
|
help="print every compared case, not only the movers")
|
|
args = ap.parse_args()
|
|
|
|
a, b = load(args.before), load(args.after)
|
|
if a["tool"] != b["tool"]:
|
|
raise SystemExit(f"{args.before} is a {a['tool']} snapshot and {args.after} a "
|
|
f"{b['tool']} one — those are different machines, not a diff")
|
|
diffs = header_diff(a, b)
|
|
if diffs:
|
|
print("these snapshots were taken differently:")
|
|
for key, va, vb in diffs:
|
|
print(f" {key}: {va} -> {vb}")
|
|
print(" ...so read the numbers below with that in mind")
|
|
|
|
A = {c["key"]: c for c in a["cases"]}
|
|
B = {c["key"]: c for c in b["cases"]}
|
|
common = sorted(set(A) & set(B))
|
|
moved, quiet, memory, broke, fixed, dtype_split = [], [], [], [], [], []
|
|
for key in common:
|
|
ra, rb = A[key], B[key]
|
|
if ra["status"] != rb["status"]:
|
|
(broke if rb["status"] != "ok" else fixed).append((key, rb.get("error", "")))
|
|
continue
|
|
if ra["status"] != "ok" or not ra.get("ms"):
|
|
continue
|
|
# Only if both sides name one: an older snapshot records no dtype per case.
|
|
da, db = ra.get("dtype"), rb.get("dtype")
|
|
if da and db and da != db:
|
|
dtype_split.append((key, da, db, ra, rb))
|
|
continue
|
|
delta = 100 * (rb["ms"] - ra["ms"]) / ra["ms"]
|
|
noise = floor_pct(ra, rb, args.sigma)
|
|
dgib = (100 * (rb["peak_gib"] - ra["peak_gib"]) / ra["peak_gib"]
|
|
if ra.get("peak_gib") and rb.get("peak_gib") else 0.0)
|
|
row = (delta, noise, key, ra, rb)
|
|
(moved if abs(delta) > noise else quiet).append(row)
|
|
if abs(dgib) > args.gib:
|
|
memory.append((dgib, key, ra, rb))
|
|
|
|
def line(delta: float, noise: float, key: str, ra: dict, rb: dict, mark: str = "") -> None:
|
|
print(f" {key:44s} {ra.get('dtype') or '':>4s} {ra['ms']:9.2f} {rb['ms']:9.2f} "
|
|
f"{delta:+8.1f} {noise:5.1f} "
|
|
f"{ra['peak_gib']:5.2f} -> {rb['peak_gib']:<5.2f} {mark}".rstrip())
|
|
|
|
def header() -> None:
|
|
print(f" {'case':44s} {'dt':>4s} {'before':>9s} {'after':>9s} {'Δ%':>8s} "
|
|
f"{'floor':>5s} {'GiB':>14s}")
|
|
|
|
if args.all:
|
|
print(f"\nevery compared case ({len(moved) + len(quiet)}), [*] clears "
|
|
f"{args.sigma:g} sigma")
|
|
header()
|
|
for delta, noise, key, ra, rb in sorted(moved + quiet, key=lambda r: -r[0]):
|
|
line(delta, noise, key, ra, rb, "*" if abs(delta) > noise else "")
|
|
else:
|
|
for title, rows in (("slower", sorted([r for r in moved if r[0] > 0],
|
|
key=lambda r: -r[0])),
|
|
("faster", sorted([r for r in moved if r[0] < 0],
|
|
key=lambda r: r[0]))):
|
|
if not rows:
|
|
continue
|
|
print(f"\n{title} ({len(rows)})")
|
|
header()
|
|
for delta, noise, key, ra, rb in rows:
|
|
line(delta, noise, key, ra, rb)
|
|
|
|
if memory:
|
|
print(f"\npeak memory moved more than {args.gib:g}%")
|
|
for dgib, key, ra, rb in sorted(memory, key=lambda r: -abs(r[0])):
|
|
print(f" {key:44s} {ra['peak_gib']:6.2f} -> {rb['peak_gib']:6.2f} GiB "
|
|
f"{dgib:+6.1f}%")
|
|
|
|
if dtype_split:
|
|
print(f"\nnot comparable, different compute dtype ({len(dtype_split)})")
|
|
for key, da, db, ra, rb in sorted(dtype_split):
|
|
print(f" {key:44s} {da:>4s} {ra['ms']:9.2f} -> {db:<4s} {rb['ms']:9.2f} ms")
|
|
|
|
for title, rows in (("now failing", broke), ("no longer failing", fixed)):
|
|
if rows:
|
|
print(f"\n{title} ({len(rows)})")
|
|
for key, err in rows:
|
|
print(f" {key:44s} {err[:80]}")
|
|
for title, keys in (("only in " + args.before, sorted(set(A) - set(B))),
|
|
("only in " + args.after, sorted(set(B) - set(A)))):
|
|
if keys:
|
|
print(f"\n{title} ({len(keys)})")
|
|
for key in keys:
|
|
print(f" {key}")
|
|
|
|
slow = sum(1 for r in moved if r[0] > 0)
|
|
summary = (f"\n{len(common)} cases compared: {slow} slower, "
|
|
f"{len(moved) - slow} faster, {len(quiet)} under the noise floor "
|
|
f"({args.sigma:g} sigma)")
|
|
if memory:
|
|
summary += f", {len(memory)} memory"
|
|
if dtype_split:
|
|
summary += f", {len(dtype_split)} left out for a dtype change"
|
|
if broke:
|
|
summary += f", {len(broke)} broken"
|
|
print(summary)
|
|
sys.exit(1 if slow or broke else 0)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|