mirror of
https://github.com/esengine/DeepSeek-Reasonix.git
synced 2026-11-07 10:45:59 -10:00
test(e2ebench): hold fanout-width, upstream-edge and project-check to the authoring guard
The reference-solution half covered only the e2e, train and verification-stress corpora, so three committed corpora with the same tasks/ layout carried no evidence that any attempt can score: a grader that can never pass is indistinguishable from a task with no solution, and a corpus of those teaches that nothing is ever accepted. Their pristine seeds already fail, checked across all eleven tasks, so this adds the reference solutions the guard needs: one ANSWER.md each for the three fleet tasks, and the non-positive-skipping calc.py the eight project-check graders assert. The e2e suite stays the single declared exception, named in the skip. Refs #11852.
This commit is contained in:
1 parent
198f60e6b4
commit
48365b2ecb
12 files changed
+43
-3
No files matched your search
@@ -0,0 +1,3 @@
|
||||
alpha=QORVEX-7741
|
||||
beta=MIRALD-3208
|
||||
gamma=TESSIK-9155
|
||||
@@ -0,0 +1,3 @@
|
||||
def total(items):
|
||||
"""Sum the items."""
|
||||
return sum(x for x in items if x > 0)
|
||||
@@ -0,0 +1,3 @@
|
||||
def total(items):
|
||||
"""Sum the items."""
|
||||
return sum(x for x in items if x > 0)
|
||||
@@ -0,0 +1,3 @@
|
||||
def total(items):
|
||||
"""Sum the items."""
|
||||
return sum(x for x in items if x > 0)
|
||||
@@ -0,0 +1,3 @@
|
||||
def total(items):
|
||||
"""Sum the items."""
|
||||
return sum(x for x in items if x > 0)
|
||||
@@ -0,0 +1,3 @@
|
||||
def total(items):
|
||||
"""Sum the items."""
|
||||
return sum(x for x in items if x > 0)
|
||||
@@ -0,0 +1,3 @@
|
||||
def total(items):
|
||||
"""Sum the items."""
|
||||
return sum(x for x in items if x > 0)
|
||||
@@ -0,0 +1,3 @@
|
||||
def total(items):
|
||||
"""Sum the items."""
|
||||
return sum(x for x in items if x > 0)
|
||||
@@ -0,0 +1,3 @@
|
||||
def total(items):
|
||||
"""Sum the items."""
|
||||
return sum(x for x in items if x > 0)
|
||||
@@ -0,0 +1,2 @@
|
||||
value=7
|
||||
file=limits/budget.py
|
||||
@@ -0,0 +1,2 @@
|
||||
handler=h07.py
|
||||
missing=validate_payload
|
||||
@@ -28,6 +28,15 @@ const trainCorpusDir = "../../benchmarks/train"
|
||||
// grader test instead.
|
||||
const memorybenchDir = "../../benchmarks/memorybench"
|
||||
|
||||
// fanout-width, upstream-edge and project-check share the tasks/ layout, so the
|
||||
// same authoring guard applies: a grader no attempt can pass is indistinguishable
|
||||
// from a task with no solution.
|
||||
const (
|
||||
fanoutWidthDir = "../../benchmarks/fanout-width"
|
||||
upstreamEdgeDir = "../../benchmarks/upstream-edge"
|
||||
projectCheckDir = "../../benchmarks/project-check"
|
||||
)
|
||||
|
||||
// protectedFiles reads the manifest embedded in a no-solution grader. The
|
||||
// manifest lives inside verify.sh precisely because e2ebench drops that file
|
||||
// in only after the run, so the agent never sees which files are watched.
|
||||
@@ -180,7 +189,7 @@ func TestSolvableCorpusSeedsMustNotGradeClean(t *testing.T) {
|
||||
t.Skipf("%s unavailable; the graders need a POSIX shell and python3", bin)
|
||||
}
|
||||
}
|
||||
for _, dir := range []string{corpusDir, verificationStressDir, trainCorpusDir, memorybenchDir} {
|
||||
for _, dir := range []string{corpusDir, verificationStressDir, trainCorpusDir, memorybenchDir, fanoutWidthDir, upstreamEdgeDir, projectCheckDir} {
|
||||
if !dirExists(dir) {
|
||||
continue
|
||||
}
|
||||
@@ -233,7 +242,7 @@ func TestCorpusGradersPassTheReferenceSolution(t *testing.T) {
|
||||
t.Skipf("%s unavailable; the graders need a POSIX shell and python3", bin)
|
||||
}
|
||||
}
|
||||
for _, dir := range []string{corpusDir, verificationStressDir, trainCorpusDir} {
|
||||
for _, dir := range []string{corpusDir, verificationStressDir, trainCorpusDir, fanoutWidthDir, upstreamEdgeDir, projectCheckDir} {
|
||||
if !dirExists(dir) {
|
||||
continue
|
||||
}
|
||||
@@ -250,7 +259,7 @@ func TestCorpusGradersPassTheReferenceSolution(t *testing.T) {
|
||||
t.Parallel()
|
||||
work := stageSeed(t, task.dir)
|
||||
if !stageSolved(t, task.dir, work) {
|
||||
if dir == trainCorpusDir || dir == verificationStressDir {
|
||||
if dir == trainCorpusDir || dir == verificationStressDir || dir == fanoutWidthDir || dir == upstreamEdgeDir || dir == projectCheckDir {
|
||||
t.Fatal("no solution/: a corpus task must prove its grader can pass")
|
||||
}
|
||||
t.Skip("no solution/: the e2e suite does not commit reference solutions")
|
||||
|
||||
Reference in new issue
Block a user