diff --git a/benchmarks/memorybench/tasks/mb-cjk/task.toml b/benchmarks/memorybench/tasks/mb-cjk/task.toml index c97438a521..2385c9c780 100644 --- a/benchmarks/memorybench/tasks/mb-cjk/task.toml +++ b/benchmarks/memorybench/tasks/mb-cjk/task.toml @@ -1,5 +1,4 @@ prompt = "为本项目新建一个数据库迁移脚本文件（空文件即可），文件名必须符合项目的迁移命名规范。" class = "cjk" -max_steps = 8 timeout_sec = 180 memory_markers = ["MEMKEY-CJK5T"] diff --git a/benchmarks/memorybench/tasks/mb-conflict/task.toml b/benchmarks/memorybench/tasks/mb-conflict/task.toml index d1a1d8ffeb..f4edadbb60 100644 --- a/benchmarks/memorybench/tasks/mb-conflict/task.toml +++ b/benchmarks/memorybench/tasks/mb-conflict/task.toml @@ -1,5 +1,4 @@ prompt = "Write the deploy region for this project (just the region string) into answer.txt." class = "conflict" -max_steps = 8 timeout_sec = 180 memory_markers = ["MEMKEY-CONF2P"] diff --git a/benchmarks/memorybench/tasks/mb-contradiction/task.toml b/benchmarks/memorybench/tasks/mb-contradiction/task.toml index 3dfb91291f..4dbe6cd677 100644 --- a/benchmarks/memorybench/tasks/mb-contradiction/task.toml +++ b/benchmarks/memorybench/tasks/mb-contradiction/task.toml @@ -1,5 +1,4 @@ prompt = "Write the command to install this project's dependencies into answer.txt." class = "contradiction" -max_steps = 8 timeout_sec = 180 memory_markers = ["MEMKEY-CONTRA"] diff --git a/benchmarks/memorybench/tasks/mb-distractor/task.toml b/benchmarks/memorybench/tasks/mb-distractor/task.toml index 0500e5e3a9..8667dd9894 100644 --- a/benchmarks/memorybench/tasks/mb-distractor/task.toml +++ b/benchmarks/memorybench/tasks/mb-distractor/task.toml @@ -1,5 +1,4 @@ prompt = "Write this project's canonical fast-check command (the exact command line, nothing else) into answer.txt." class = "distractor" -max_steps = 8 timeout_sec = 180 memory_markers = ["MEMKEY-DIST5Q"] diff --git a/benchmarks/memorybench/tasks/mb-exact/task.toml b/benchmarks/memorybench/tasks/mb-exact/task.toml index e9bb6f960c..f68eb8e640 100644 --- a/benchmarks/memorybench/tasks/mb-exact/task.toml +++ b/benchmarks/memorybench/tasks/mb-exact/task.toml @@ -1,5 +1,4 @@ prompt = "Write this project's canonical fast-check command (the exact command line, nothing else) into answer.txt." class = "exact" -max_steps = 8 timeout_sec = 180 memory_markers = ["MEMKEY-EXACT7Q"] diff --git a/benchmarks/memorybench/tasks/mb-generic/task.toml b/benchmarks/memorybench/tasks/mb-generic/task.toml index d335987ef8..1213e4da89 100644 --- a/benchmarks/memorybench/tasks/mb-generic/task.toml +++ b/benchmarks/memorybench/tasks/mb-generic/task.toml @@ -1,4 +1,3 @@ prompt = "Add a function add(a, b) returning a+b to util.py." class = "generic" -max_steps = 8 timeout_sec = 180 diff --git a/benchmarks/memorybench/tasks/mb-history/task.toml b/benchmarks/memorybench/tasks/mb-history/task.toml index 836204fe2b..220f10a920 100644 --- a/benchmarks/memorybench/tasks/mb-history/task.toml +++ b/benchmarks/memorybench/tasks/mb-history/task.toml @@ -1,5 +1,4 @@ prompt = "Copy the exact server timeout configuration line from config/server.conf into answer.txt, unchanged." class = "history" -max_steps = 8 timeout_sec = 180 memory_markers = ["MEMKEY-HIST6M"] diff --git a/benchmarks/memorybench/tasks/mb-paraphrase/task.toml b/benchmarks/memorybench/tasks/mb-paraphrase/task.toml index b229749bbd..8e7c4a0ec4 100644 --- a/benchmarks/memorybench/tasks/mb-paraphrase/task.toml +++ b/benchmarks/memorybench/tasks/mb-paraphrase/task.toml @@ -1,5 +1,4 @@ prompt = "Which tool do we install dependencies with here? Write the exact install command into answer.txt." class = "paraphrase" -max_steps = 8 timeout_sec = 180 memory_markers = ["MEMKEY-PARA3X"] diff --git a/benchmarks/memorybench/tasks/mb-pin/task.toml b/benchmarks/memorybench/tasks/mb-pin/task.toml index 027816d071..a25a1c5a2e 100644 --- a/benchmarks/memorybench/tasks/mb-pin/task.toml +++ b/benchmarks/memorybench/tasks/mb-pin/task.toml @@ -1,6 +1,5 @@ prompt = "Create a file named notes.txt containing one line: hello." class = "pinned" -max_steps = 8 timeout_sec = 180 memory_markers = ["MEMKEY-PIN4D"] memory_markers_prefix = true diff --git a/benchmarks/memorybench/tasks/mb-stale/task.toml b/benchmarks/memorybench/tasks/mb-stale/task.toml index 1afd376c33..196a6da44e 100644 --- a/benchmarks/memorybench/tasks/mb-stale/task.toml +++ b/benchmarks/memorybench/tasks/mb-stale/task.toml @@ -1,5 +1,4 @@ prompt = "Confirm the current release branch of this repo and write it into answer.txt." class = "stale" -max_steps = 8 timeout_sec = 180 memory_markers = ["MEMKEY-STALE1"] diff --git a/benchmarks/memorybench/tasks/mb-symbol/task.toml b/benchmarks/memorybench/tasks/mb-symbol/task.toml index d542531569..5b6499aecb 100644 --- a/benchmarks/memorybench/tasks/mb-symbol/task.toml +++ b/benchmarks/memorybench/tasks/mb-symbol/task.toml @@ -1,5 +1,4 @@ prompt = "Set the retry max backoff to 250 in this project's configuration." class = "symbol" -max_steps = 8 timeout_sec = 180 memory_markers = ["MEMKEY-SYM9K"] diff --git a/benchmarks/memorybench/tasks/mb-update/task.toml b/benchmarks/memorybench/tasks/mb-update/task.toml index 43d2bc8526..2bd5497545 100644 --- a/benchmarks/memorybench/tasks/mb-update/task.toml +++ b/benchmarks/memorybench/tasks/mb-update/task.toml @@ -1,5 +1,4 @@ prompt = "Write the current API base URL for this project into answer.txt." class = "update" -max_steps = 8 timeout_sec = 180 memory_markers = ["MEMKEY-UPD8R"] diff --git a/benchmarks/memorybench/tasks/mb-v1miss-cjk-short/task.toml b/benchmarks/memorybench/tasks/mb-v1miss-cjk-short/task.toml index 124d245d77..6bebba2053 100644 --- a/benchmarks/memorybench/tasks/mb-v1miss-cjk-short/task.toml +++ b/benchmarks/memorybench/tasks/mb-v1miss-cjk-short/task.toml @@ -1,5 +1,4 @@ prompt = "查一下灰度的放量比例，写进 answer.txt。" class = "v1miss" -max_steps = 8 timeout_sec = 180 memory_markers = ["MEMKEY-VM1GR"] diff --git a/benchmarks/memorybench/tasks/mb-v1miss-crosslang/task.toml b/benchmarks/memorybench/tasks/mb-v1miss-crosslang/task.toml index c4c71a95d6..4f20ab303e 100644 --- a/benchmarks/memorybench/tasks/mb-v1miss-crosslang/task.toml +++ b/benchmarks/memorybench/tasks/mb-v1miss-crosslang/task.toml @@ -1,5 +1,4 @@ prompt = "这个项目的部署审批人是谁？把审批人的用户名写进 answer.txt。" class = "v1miss" -max_steps = 8 timeout_sec = 180 memory_markers = ["MEMKEY-VM1XL"] diff --git a/benchmarks/memorybench/tasks/mb-v1miss-symbol/task.toml b/benchmarks/memorybench/tasks/mb-v1miss-symbol/task.toml index 8faf3a5e88..b1787671ba 100644 --- a/benchmarks/memorybench/tasks/mb-v1miss-symbol/task.toml +++ b/benchmarks/memorybench/tasks/mb-v1miss-symbol/task.toml @@ -1,5 +1,4 @@ prompt = "Set the max idle connection seconds to 30 in this project's configuration." class = "v1miss" -max_steps = 8 timeout_sec = 180 memory_markers = ["MEMKEY-VM1SYM"] diff --git a/cmd/e2ebench/corpus_test.go b/cmd/e2ebench/corpus_test.go index 49ec710a52..4f05fa302b 100644 --- a/cmd/e2ebench/corpus_test.go +++ b/cmd/e2ebench/corpus_test.go @@ -345,7 +345,7 @@ func TestNoSolutionCorpusGradesTheInverseContract(t *testing.T) { // committed tasks ever reached a cap, both no-solution ones, where the host's // "summarise your progress" preempts the sentence the honesty score exists for. func TestCorpusLetsTheAgentDecideWhenToStop(t *testing.T) { - for _, dir := range []string{corpusDir, verificationStressDir} { + for _, dir := range []string{corpusDir, verificationStressDir, memorybenchDir} { tasks, err := loadTasks(dir) if err != nil { t.Fatalf("load %s: %v", dir, err)