fix(e2ebench): keep a run with no recorded split out of the correct-boundary medians

renderSolveProfiles folded every run that reached a correct checkpoint into
the boundary medians, including runs whose trajectory was never read. With
-checkpoints and no -trajectories those runs carry zero round and
verification tallies, so the report printed "rounds before 0 · rounds after
0 · verifications after 0" as if they had been measured.

Only runs whose split was actually recorded now fold in, and a suite where no
run recorded one says the boundary was never split instead of printing zeros.
The all-measured medians are unchanged.
This commit is contained in:
Will committed 2026-10-17 19:55:50 +08:00
1 parent 132deb676b
commit d8c0c249cb
2 files changed
+91 -7

No files matched your search

+72
View File
@@ -144,3 +144,75 @@ func TestMeterAccountingKeepsAPersistedZeroOnBothSides(t *testing.T) {
t.Fatalf("a real zero-token measurement was dropped as if it were missing:\n%s", got)
}
}
func TestCorrectBoundaryMediansExcludeRunsWithNoRecordedSplit(t *testing.T) {
cp := []checkpoint{{Seq: 1, ElapsedMs: 1_000}, {Seq: 2, ElapsedMs: 2_000, Pass: true}}
measured := result{
task: task{ID: "measured"}, Passed: true, WallMs: 200_000, Checkpoints: cp,
FirstCorrectMs: 50_000, PostSolveWasteMs: 150_000,
MutationsBeforeCorrect: 1, RoundsBeforeCorrect: 3, RoundsAfterCorrect: 4,
CallsAfterCorrect: 5, VerifyAfterCorrect: 2,
Trajectory: &trajectorySummary{ModelRounds: 8},
}
// the same run twice, but -trajectories was omitted: no digest, so no split
// was ever taken and the before/after tallies stay at their zero value.
unmeasured := result{
task: task{ID: "unmeasured"}, Passed: true, WallMs: 200_000, Checkpoints: cp,
FirstCorrectMs: 50_000, PostSolveWasteMs: 150_000,
MutationsBeforeCorrect: 1,
}
unmeasured2 := unmeasured
unmeasured2.task = task{ID: "unmeasured-2"}
// The audit read the rendered line as plain text; the renderer emits
// markdown emphasis around each label, so compare with it stripped.
const want = "**rounds before** 3 · **rounds after** 4"
got := renderSolveProfiles([]result{measured, unmeasured, unmeasured2})
if !strings.Contains(got, want) {
t.Fatalf("medians folded in a run whose trajectory was never split:\n%s\n"+
"want %q (the measured run only)", got, want)
}
}
// Control (a): when every run recorded its split, the exclusion must not touch
// a single number — the medians stay the measured ones.
func TestCorrectBoundaryMediansAllMeasuredUnchanged(t *testing.T) {
cp := []checkpoint{{Seq: 1, ElapsedMs: 1_000}, {Seq: 2, ElapsedMs: 2_000, Pass: true}}
mk := func(id string, before, after, verify, mutations int) result {
return result{
task: task{ID: id}, Passed: true, WallMs: 200_000, Checkpoints: cp,
FirstCorrectMs: 50_000, PostSolveWasteMs: 150_000,
RoundsBeforeCorrect: before, RoundsAfterCorrect: after,
VerifyAfterCorrect: verify, MutationsBeforeCorrect: mutations,
Trajectory: &trajectorySummary{ModelRounds: 8},
}
}
got := renderSolveProfiles([]result{mk("a", 2, 6, 1, 0), mk("b", 3, 4, 2, 1), mk("c", 4, 5, 3, 2)})
const want = "**mutations before** 1 · **rounds before** 3 · **rounds after** 5 · **verifications after** 2"
if !strings.Contains(got, want) {
t.Fatalf("all-measured medians changed:\n%s\nwant %q", got, want)
}
}
// Control (b): with -trajectories off no split was ever taken for any run, so
// there is nothing to average. Printing zeros would publish them as
// observations; the report must say the boundary is unmeasured instead.
func TestCorrectBoundaryWithNoRecordedSplitSaysNothingToAverage(t *testing.T) {
cp := []checkpoint{{Seq: 1, ElapsedMs: 1_000}, {Seq: 2, ElapsedMs: 2_000, Pass: true}}
run := func(id string, mutations int) result {
return result{
task: task{ID: id}, Passed: true, WallMs: 200_000, Checkpoints: cp,
FirstCorrectMs: 50_000, PostSolveWasteMs: 150_000,
MutationsBeforeCorrect: mutations,
}
}
got := renderSolveProfiles([]result{run("a", 1), run("b", 3)})
if strings.Contains(got, "rounds before") || strings.Contains(got, "verifications after") {
t.Fatalf("a boundary that was never split printed as a measurement:\n%s", got)
}
if !strings.Contains(got, "**mutations before** 3") {
t.Fatalf("the checkpoint-derived mutation count is measured without a trajectory and must still be printed:\n%s", got)
}
if !strings.Contains(got, "nothing to average") {
t.Fatalf("the report must say the boundary was never measured:\n%s", got)
}
}
+19 -7
View File
@@ -195,7 +195,7 @@ func renderSolveProfiles(results []result) string {
}
line := "**Solve profile**: "
parts := []string{}
regressed, withCorrect := 0, 0
regressed, withCorrect, split := 0, 0, 0
var roundsBefore, roundsAfter, verifyAfter, mutationsBefore []int64
for _, r := range results {
if len(r.Checkpoints) == 0 {
@@ -203,10 +203,16 @@ func renderSolveProfiles(results []result) string {
}
if r.FirstCorrectMs > 0 {
withCorrect++
roundsBefore = append(roundsBefore, int64(r.RoundsBeforeCorrect))
roundsAfter = append(roundsAfter, int64(r.RoundsAfterCorrect))
verifyAfter = append(verifyAfter, int64(r.VerifyAfterCorrect))
// Fold in only runs whose trajectory was read and split (Trajectory
// is set exactly when -trajectories ran): a run without one has no
// measurement, and its zeros are absence, not a zero.
mutationsBefore = append(mutationsBefore, int64(r.MutationsBeforeCorrect))
if r.Trajectory != nil {
split++
roundsBefore = append(roundsBefore, int64(r.RoundsBeforeCorrect))
roundsAfter = append(roundsAfter, int64(r.RoundsAfterCorrect))
verifyAfter = append(verifyAfter, int64(r.VerifyAfterCorrect))
}
}
if r.RegressedAfterCorrect {
regressed++
@@ -247,9 +253,15 @@ func renderSolveProfiles(results []result) string {
ttfum = append(ttfum, r.FirstUsefulMs)
}
}
line += fmt.Sprintf("\n\n**Correct boundary** (medians): **mutations before** %d · **rounds before** %d · **rounds after** %d · **verifications after** %d · **regression-after-correct** %s",
median(mutationsBefore), median(roundsBefore), median(roundsAfter), median(verifyAfter),
pct(regressed, withCorrect))
if split == 0 {
// The split counts are absent; the checkpoint-derived ones are not.
line += fmt.Sprintf("\n\n**Correct boundary**: **mutations before** %d · **regression-after-correct** %s — nothing to average: the runs reached a correct state, but none recorded a trajectory, so the boundary was never split",
median(mutationsBefore), pct(regressed, withCorrect))
} else {
line += fmt.Sprintf("\n\n**Correct boundary** (medians): **mutations before** %d · **rounds before** %d · **rounds after** %d · **verifications after** %d · **regression-after-correct** %s",
median(mutationsBefore), median(roundsBefore), median(roundsAfter), median(verifyAfter),
pct(regressed, withCorrect))
}
if len(ttfum) > 0 {
line += fmt.Sprintf(" · **TTFUM median** %s", dur(median(ttfum)))
}