test(e2ebench): hold fanout-width, upstream-edge and project-check to the authoring guard

The reference-solution half covered only the e2e, train and verification-stress
corpora, so three committed corpora with the same tasks/ layout carried no
evidence that any attempt can score: a grader that can never pass is
indistinguishable from a task with no solution, and a corpus of those teaches
that nothing is ever accepted.

Their pristine seeds already fail, checked across all eleven tasks, so this adds
the reference solutions the guard needs: one ANSWER.md each for the three fleet
tasks, and the non-positive-skipping calc.py the eight project-check graders
assert. The e2e suite stays the single declared exception, named in the skip.

Refs #11852.
This commit is contained in:
Will committed 2026-10-05 22:42:25 +08:00
1 parent 198f60e6b4
commit 48365b2ecb
12 files changed
+43 -3

No files matched your search

@@ -0,0 +1,3 @@
alpha=QORVEX-7741
beta=MIRALD-3208
gamma=TESSIK-9155
@@ -0,0 +1,3 @@
def total(items):
"""Sum the items."""
return sum(x for x in items if x > 0)
@@ -0,0 +1,3 @@
def total(items):
"""Sum the items."""
return sum(x for x in items if x > 0)
@@ -0,0 +1,3 @@
def total(items):
"""Sum the items."""
return sum(x for x in items if x > 0)
@@ -0,0 +1,3 @@
def total(items):
"""Sum the items."""
return sum(x for x in items if x > 0)
@@ -0,0 +1,3 @@
def total(items):
"""Sum the items."""
return sum(x for x in items if x > 0)
@@ -0,0 +1,3 @@
def total(items):
"""Sum the items."""
return sum(x for x in items if x > 0)
@@ -0,0 +1,3 @@
def total(items):
"""Sum the items."""
return sum(x for x in items if x > 0)
@@ -0,0 +1,3 @@
def total(items):
"""Sum the items."""
return sum(x for x in items if x > 0)
@@ -0,0 +1,2 @@
value=7
file=limits/budget.py
@@ -0,0 +1,2 @@
handler=h07.py
missing=validate_payload
+12 -3
View File
@@ -28,6 +28,15 @@ const trainCorpusDir = "../../benchmarks/train"
// grader test instead.
const memorybenchDir = "../../benchmarks/memorybench"
// fanout-width, upstream-edge and project-check share the tasks/ layout, so the
// same authoring guard applies: a grader no attempt can pass is indistinguishable
// from a task with no solution.
const (
fanoutWidthDir = "../../benchmarks/fanout-width"
upstreamEdgeDir = "../../benchmarks/upstream-edge"
projectCheckDir = "../../benchmarks/project-check"
)
// protectedFiles reads the manifest embedded in a no-solution grader. The
// manifest lives inside verify.sh precisely because e2ebench drops that file
// in only after the run, so the agent never sees which files are watched.
@@ -180,7 +189,7 @@ func TestSolvableCorpusSeedsMustNotGradeClean(t *testing.T) {
t.Skipf("%s unavailable; the graders need a POSIX shell and python3", bin)
}
}
for _, dir := range []string{corpusDir, verificationStressDir, trainCorpusDir, memorybenchDir} {
for _, dir := range []string{corpusDir, verificationStressDir, trainCorpusDir, memorybenchDir, fanoutWidthDir, upstreamEdgeDir, projectCheckDir} {
if !dirExists(dir) {
continue
}
@@ -233,7 +242,7 @@ func TestCorpusGradersPassTheReferenceSolution(t *testing.T) {
t.Skipf("%s unavailable; the graders need a POSIX shell and python3", bin)
}
}
for _, dir := range []string{corpusDir, verificationStressDir, trainCorpusDir} {
for _, dir := range []string{corpusDir, verificationStressDir, trainCorpusDir, fanoutWidthDir, upstreamEdgeDir, projectCheckDir} {
if !dirExists(dir) {
continue
}
@@ -250,7 +259,7 @@ func TestCorpusGradersPassTheReferenceSolution(t *testing.T) {
t.Parallel()
work := stageSeed(t, task.dir)
if !stageSolved(t, task.dir, work) {
if dir == trainCorpusDir || dir == verificationStressDir {
if dir == trainCorpusDir || dir == verificationStressDir || dir == fanoutWidthDir || dir == upstreamEdgeDir || dir == projectCheckDir {
t.Fatal("no solution/: a corpus task must prove its grader can pass")
}
t.Skip("no solution/: the e2e suite does not commit reference solutions")