mirror of
https://github.com/esengine/DeepSeek-Reasonix.git
synced 2026-11-07 10:45:59 -10:00
fix(e2ebench): keep a run with no recorded split out of the correct-boundary medians
renderSolveProfiles folded every run that reached a correct checkpoint into the boundary medians, including runs whose trajectory was never read. With -checkpoints and no -trajectories those runs carry zero round and verification tallies, so the report printed "rounds before 0 · rounds after 0 · verifications after 0" as if they had been measured. Only runs whose split was actually recorded now fold in, and a suite where no run recorded one says the boundary was never split instead of printing zeros. The all-measured medians are unchanged.
This commit is contained in:
1 parent
132deb676b
commit
d8c0c249cb
2 files changed
+91
-7
No files matched your search
@@ -144,3 +144,75 @@ func TestMeterAccountingKeepsAPersistedZeroOnBothSides(t *testing.T) {
|
||||
t.Fatalf("a real zero-token measurement was dropped as if it were missing:\n%s", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCorrectBoundaryMediansExcludeRunsWithNoRecordedSplit(t *testing.T) {
|
||||
cp := []checkpoint{{Seq: 1, ElapsedMs: 1_000}, {Seq: 2, ElapsedMs: 2_000, Pass: true}}
|
||||
measured := result{
|
||||
task: task{ID: "measured"}, Passed: true, WallMs: 200_000, Checkpoints: cp,
|
||||
FirstCorrectMs: 50_000, PostSolveWasteMs: 150_000,
|
||||
MutationsBeforeCorrect: 1, RoundsBeforeCorrect: 3, RoundsAfterCorrect: 4,
|
||||
CallsAfterCorrect: 5, VerifyAfterCorrect: 2,
|
||||
Trajectory: &trajectorySummary{ModelRounds: 8},
|
||||
}
|
||||
// the same run twice, but -trajectories was omitted: no digest, so no split
|
||||
// was ever taken and the before/after tallies stay at their zero value.
|
||||
unmeasured := result{
|
||||
task: task{ID: "unmeasured"}, Passed: true, WallMs: 200_000, Checkpoints: cp,
|
||||
FirstCorrectMs: 50_000, PostSolveWasteMs: 150_000,
|
||||
MutationsBeforeCorrect: 1,
|
||||
}
|
||||
unmeasured2 := unmeasured
|
||||
unmeasured2.task = task{ID: "unmeasured-2"}
|
||||
// The audit read the rendered line as plain text; the renderer emits
|
||||
// markdown emphasis around each label, so compare with it stripped.
|
||||
const want = "**rounds before** 3 · **rounds after** 4"
|
||||
got := renderSolveProfiles([]result{measured, unmeasured, unmeasured2})
|
||||
if !strings.Contains(got, want) {
|
||||
t.Fatalf("medians folded in a run whose trajectory was never split:\n%s\n"+
|
||||
"want %q (the measured run only)", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
// Control (a): when every run recorded its split, the exclusion must not touch
|
||||
// a single number — the medians stay the measured ones.
|
||||
func TestCorrectBoundaryMediansAllMeasuredUnchanged(t *testing.T) {
|
||||
cp := []checkpoint{{Seq: 1, ElapsedMs: 1_000}, {Seq: 2, ElapsedMs: 2_000, Pass: true}}
|
||||
mk := func(id string, before, after, verify, mutations int) result {
|
||||
return result{
|
||||
task: task{ID: id}, Passed: true, WallMs: 200_000, Checkpoints: cp,
|
||||
FirstCorrectMs: 50_000, PostSolveWasteMs: 150_000,
|
||||
RoundsBeforeCorrect: before, RoundsAfterCorrect: after,
|
||||
VerifyAfterCorrect: verify, MutationsBeforeCorrect: mutations,
|
||||
Trajectory: &trajectorySummary{ModelRounds: 8},
|
||||
}
|
||||
}
|
||||
got := renderSolveProfiles([]result{mk("a", 2, 6, 1, 0), mk("b", 3, 4, 2, 1), mk("c", 4, 5, 3, 2)})
|
||||
const want = "**mutations before** 1 · **rounds before** 3 · **rounds after** 5 · **verifications after** 2"
|
||||
if !strings.Contains(got, want) {
|
||||
t.Fatalf("all-measured medians changed:\n%s\nwant %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
// Control (b): with -trajectories off no split was ever taken for any run, so
|
||||
// there is nothing to average. Printing zeros would publish them as
|
||||
// observations; the report must say the boundary is unmeasured instead.
|
||||
func TestCorrectBoundaryWithNoRecordedSplitSaysNothingToAverage(t *testing.T) {
|
||||
cp := []checkpoint{{Seq: 1, ElapsedMs: 1_000}, {Seq: 2, ElapsedMs: 2_000, Pass: true}}
|
||||
run := func(id string, mutations int) result {
|
||||
return result{
|
||||
task: task{ID: id}, Passed: true, WallMs: 200_000, Checkpoints: cp,
|
||||
FirstCorrectMs: 50_000, PostSolveWasteMs: 150_000,
|
||||
MutationsBeforeCorrect: mutations,
|
||||
}
|
||||
}
|
||||
got := renderSolveProfiles([]result{run("a", 1), run("b", 3)})
|
||||
if strings.Contains(got, "rounds before") || strings.Contains(got, "verifications after") {
|
||||
t.Fatalf("a boundary that was never split printed as a measurement:\n%s", got)
|
||||
}
|
||||
if !strings.Contains(got, "**mutations before** 3") {
|
||||
t.Fatalf("the checkpoint-derived mutation count is measured without a trajectory and must still be printed:\n%s", got)
|
||||
}
|
||||
if !strings.Contains(got, "nothing to average") {
|
||||
t.Fatalf("the report must say the boundary was never measured:\n%s", got)
|
||||
}
|
||||
}
|
||||
@@ -195,7 +195,7 @@ func renderSolveProfiles(results []result) string {
|
||||
}
|
||||
line := "**Solve profile**: "
|
||||
parts := []string{}
|
||||
regressed, withCorrect := 0, 0
|
||||
regressed, withCorrect, split := 0, 0, 0
|
||||
var roundsBefore, roundsAfter, verifyAfter, mutationsBefore []int64
|
||||
for _, r := range results {
|
||||
if len(r.Checkpoints) == 0 {
|
||||
@@ -203,10 +203,16 @@ func renderSolveProfiles(results []result) string {
|
||||
}
|
||||
if r.FirstCorrectMs > 0 {
|
||||
withCorrect++
|
||||
roundsBefore = append(roundsBefore, int64(r.RoundsBeforeCorrect))
|
||||
roundsAfter = append(roundsAfter, int64(r.RoundsAfterCorrect))
|
||||
verifyAfter = append(verifyAfter, int64(r.VerifyAfterCorrect))
|
||||
// Fold in only runs whose trajectory was read and split (Trajectory
|
||||
// is set exactly when -trajectories ran): a run without one has no
|
||||
// measurement, and its zeros are absence, not a zero.
|
||||
mutationsBefore = append(mutationsBefore, int64(r.MutationsBeforeCorrect))
|
||||
if r.Trajectory != nil {
|
||||
split++
|
||||
roundsBefore = append(roundsBefore, int64(r.RoundsBeforeCorrect))
|
||||
roundsAfter = append(roundsAfter, int64(r.RoundsAfterCorrect))
|
||||
verifyAfter = append(verifyAfter, int64(r.VerifyAfterCorrect))
|
||||
}
|
||||
}
|
||||
if r.RegressedAfterCorrect {
|
||||
regressed++
|
||||
@@ -247,9 +253,15 @@ func renderSolveProfiles(results []result) string {
|
||||
ttfum = append(ttfum, r.FirstUsefulMs)
|
||||
}
|
||||
}
|
||||
line += fmt.Sprintf("\n\n**Correct boundary** (medians): **mutations before** %d · **rounds before** %d · **rounds after** %d · **verifications after** %d · **regression-after-correct** %s",
|
||||
median(mutationsBefore), median(roundsBefore), median(roundsAfter), median(verifyAfter),
|
||||
pct(regressed, withCorrect))
|
||||
if split == 0 {
|
||||
// The split counts are absent; the checkpoint-derived ones are not.
|
||||
line += fmt.Sprintf("\n\n**Correct boundary**: **mutations before** %d · **regression-after-correct** %s — nothing to average: the runs reached a correct state, but none recorded a trajectory, so the boundary was never split",
|
||||
median(mutationsBefore), pct(regressed, withCorrect))
|
||||
} else {
|
||||
line += fmt.Sprintf("\n\n**Correct boundary** (medians): **mutations before** %d · **rounds before** %d · **rounds after** %d · **verifications after** %d · **regression-after-correct** %s",
|
||||
median(mutationsBefore), median(roundsBefore), median(roundsAfter), median(verifyAfter),
|
||||
pct(regressed, withCorrect))
|
||||
}
|
||||
if len(ttfum) > 0 {
|
||||
line += fmt.Sprintf(" · **TTFUM median** %s", dur(median(ttfum)))
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user