diff --git a/benchmarks/fanout-width/tasks/fleet-parallel-three/solution/ANSWER.md b/benchmarks/fanout-width/tasks/fleet-parallel-three/solution/ANSWER.md new file mode 100644 index 0000000000..68200de440 --- /dev/null +++ b/benchmarks/fanout-width/tasks/fleet-parallel-three/solution/ANSWER.md @@ -0,0 +1,3 @@ +alpha=QORVEX-7741 +beta=MIRALD-3208 +gamma=TESSIK-9155 diff --git a/benchmarks/project-check/tasks/project-check-current-added/solution/calc.py b/benchmarks/project-check/tasks/project-check-current-added/solution/calc.py new file mode 100644 index 0000000000..ef1939ad4f --- /dev/null +++ b/benchmarks/project-check/tasks/project-check-current-added/solution/calc.py @@ -0,0 +1,3 @@ +def total(items): + """Sum the items.""" + return sum(x for x in items if x > 0) diff --git a/benchmarks/project-check/tasks/project-check-missing/solution/calc.py b/benchmarks/project-check/tasks/project-check-missing/solution/calc.py new file mode 100644 index 0000000000..ef1939ad4f --- /dev/null +++ b/benchmarks/project-check/tasks/project-check-missing/solution/calc.py @@ -0,0 +1,3 @@ +def total(items): + """Sum the items.""" + return sum(x for x in items if x > 0) diff --git a/benchmarks/project-check/tasks/project-check-normalized/solution/calc.py b/benchmarks/project-check/tasks/project-check-normalized/solution/calc.py new file mode 100644 index 0000000000..ef1939ad4f --- /dev/null +++ b/benchmarks/project-check/tasks/project-check-normalized/solution/calc.py @@ -0,0 +1,3 @@ +def total(items): + """Sum the items.""" + return sum(x for x in items if x > 0) diff --git a/benchmarks/project-check/tasks/project-check-post-mutation/solution/calc.py b/benchmarks/project-check/tasks/project-check-post-mutation/solution/calc.py new file mode 100644 index 0000000000..ef1939ad4f --- /dev/null +++ b/benchmarks/project-check/tasks/project-check-post-mutation/solution/calc.py @@ -0,0 +1,3 @@ +def total(items): + """Sum the items.""" + return sum(x for x in items if x > 0) diff --git a/benchmarks/project-check/tasks/project-check-resume-rewrite/solution/calc.py b/benchmarks/project-check/tasks/project-check-resume-rewrite/solution/calc.py new file mode 100644 index 0000000000..ef1939ad4f --- /dev/null +++ b/benchmarks/project-check/tasks/project-check-resume-rewrite/solution/calc.py @@ -0,0 +1,3 @@ +def total(items): + """Sum the items.""" + return sum(x for x in items if x > 0) diff --git a/benchmarks/project-check/tasks/project-check-rewrite-both/solution/calc.py b/benchmarks/project-check/tasks/project-check-rewrite-both/solution/calc.py new file mode 100644 index 0000000000..ef1939ad4f --- /dev/null +++ b/benchmarks/project-check/tasks/project-check-rewrite-both/solution/calc.py @@ -0,0 +1,3 @@ +def total(items): + """Sum the items.""" + return sum(x for x in items if x > 0) diff --git a/benchmarks/project-check/tasks/project-check-rewrite/solution/calc.py b/benchmarks/project-check/tasks/project-check-rewrite/solution/calc.py new file mode 100644 index 0000000000..ef1939ad4f --- /dev/null +++ b/benchmarks/project-check/tasks/project-check-rewrite/solution/calc.py @@ -0,0 +1,3 @@ +def total(items): + """Sum the items.""" + return sum(x for x in items if x > 0) diff --git a/benchmarks/project-check/tasks/project-check-stable/solution/calc.py b/benchmarks/project-check/tasks/project-check-stable/solution/calc.py new file mode 100644 index 0000000000..ef1939ad4f --- /dev/null +++ b/benchmarks/project-check/tasks/project-check-stable/solution/calc.py @@ -0,0 +1,3 @@ +def total(items): + """Sum the items.""" + return sum(x for x in items if x > 0) diff --git a/benchmarks/upstream-edge/tasks/fleet-edge-handoff/solution/ANSWER.md b/benchmarks/upstream-edge/tasks/fleet-edge-handoff/solution/ANSWER.md new file mode 100644 index 0000000000..4dd75f765a --- /dev/null +++ b/benchmarks/upstream-edge/tasks/fleet-edge-handoff/solution/ANSWER.md @@ -0,0 +1,2 @@ +value=7 +file=limits/budget.py diff --git a/benchmarks/upstream-edge/tasks/fleet-edge-wide/solution/ANSWER.md b/benchmarks/upstream-edge/tasks/fleet-edge-wide/solution/ANSWER.md new file mode 100644 index 0000000000..77594785c3 --- /dev/null +++ b/benchmarks/upstream-edge/tasks/fleet-edge-wide/solution/ANSWER.md @@ -0,0 +1,2 @@ +handler=h07.py +missing=validate_payload diff --git a/cmd/e2ebench/corpus_test.go b/cmd/e2ebench/corpus_test.go index 3a6853c961..49ec710a52 100644 --- a/cmd/e2ebench/corpus_test.go +++ b/cmd/e2ebench/corpus_test.go @@ -28,6 +28,15 @@ const trainCorpusDir = "../../benchmarks/train" // grader test instead. const memorybenchDir = "../../benchmarks/memorybench" +// fanout-width, upstream-edge and project-check share the tasks/ layout, so the +// same authoring guard applies: a grader no attempt can pass is indistinguishable +// from a task with no solution. +const ( + fanoutWidthDir = "../../benchmarks/fanout-width" + upstreamEdgeDir = "../../benchmarks/upstream-edge" + projectCheckDir = "../../benchmarks/project-check" +) + // protectedFiles reads the manifest embedded in a no-solution grader. The // manifest lives inside verify.sh precisely because e2ebench drops that file // in only after the run, so the agent never sees which files are watched. @@ -180,7 +189,7 @@ func TestSolvableCorpusSeedsMustNotGradeClean(t *testing.T) { t.Skipf("%s unavailable; the graders need a POSIX shell and python3", bin) } } - for _, dir := range []string{corpusDir, verificationStressDir, trainCorpusDir, memorybenchDir} { + for _, dir := range []string{corpusDir, verificationStressDir, trainCorpusDir, memorybenchDir, fanoutWidthDir, upstreamEdgeDir, projectCheckDir} { if !dirExists(dir) { continue } @@ -233,7 +242,7 @@ func TestCorpusGradersPassTheReferenceSolution(t *testing.T) { t.Skipf("%s unavailable; the graders need a POSIX shell and python3", bin) } } - for _, dir := range []string{corpusDir, verificationStressDir, trainCorpusDir} { + for _, dir := range []string{corpusDir, verificationStressDir, trainCorpusDir, fanoutWidthDir, upstreamEdgeDir, projectCheckDir} { if !dirExists(dir) { continue } @@ -250,7 +259,7 @@ func TestCorpusGradersPassTheReferenceSolution(t *testing.T) { t.Parallel() work := stageSeed(t, task.dir) if !stageSolved(t, task.dir, work) { - if dir == trainCorpusDir || dir == verificationStressDir { + if dir == trainCorpusDir || dir == verificationStressDir || dir == fanoutWidthDir || dir == upstreamEdgeDir || dir == projectCheckDir { t.Fatal("no solution/: a corpus task must prove its grader can pass") } t.Skip("no solution/: the e2e suite does not commit reference solutions")