Merge pull request #8252 from esengine/bench/read-only-output

bench(e2e): add the read-only-output shape the corpus never had

benchmarks/e2e/tasks/audit-config-drift/task.toml

@@ -0,0 +1,4 @@
+prompt = "Exactly one setting differs across config/base.toml, config/staging.toml and config/production.toml. Write AUDIT.md naming the setting that drifted, the environment it drifted in, and both values. Do not change any config."
+class = "read-only-output"
+max_steps = 25
+timeout_sec = 480

benchmarks/e2e/tasks/audit-config-drift/verify.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+set -e
+test -f AUDIT.md || { echo "AUDIT.md was never written" >&2; exit 1; }
+doc="$(tr '[:upper:]' '[:lower:]' < AUDIT.md)"
+for want in timeout_sec production 30 5; do
+  case "$doc" in
+    *"$want"*) ;;
+    *) echo "AUDIT.md never mentions $want" >&2; exit 1 ;;
+  esac
+done
+# Naming a setting that did not drift means the three files were not compared.
+for absent in max_body_mb ttl_sec; do
+  case "$doc" in
+    *"$absent"*) echo "AUDIT.md reports $absent, which is identical everywhere" >&2; exit 1 ;;
+  esac
+done
+grep -q "timeout_sec = 5" config/production.toml || {
+  echo "config/production.toml was modified; the task is to report the drift" >&2
+  exit 1
+}

benchmarks/e2e/tasks/audit-config-drift/workdir/README.md

@@ -0,0 +1 @@
+All three environments are kept in lockstep. Any difference is a mistake.

benchmarks/e2e/tasks/audit-config-drift/workdir/config/base.toml

@@ -0,0 +1,7 @@
+[server]
+port = 8080
+timeout_sec = 30
+max_body_mb = 8
+
+[cache]
+ttl_sec = 300

benchmarks/e2e/tasks/audit-config-drift/workdir/config/production.toml

@@ -0,0 +1,7 @@
+[server]
+port = 8080
+timeout_sec = 5
+max_body_mb = 8
+
+[cache]
+ttl_sec = 300

benchmarks/e2e/tasks/audit-config-drift/workdir/config/staging.toml

@@ -0,0 +1,7 @@
+[server]
+port = 8080
+timeout_sec = 30
+max_body_mb = 8
+
+[cache]
+ttl_sec = 300

benchmarks/e2e/tasks/document-dependency-direction/task.toml

@@ -0,0 +1,4 @@
+prompt = "Write ARCHITECTURE.md explaining how the pipeline package is layered. Name all three modules, state which one depends on which, and name the module that depends on nothing else in the package. Do not change any code."
+class = "read-only-output"
+max_steps = 25
+timeout_sec = 480

benchmarks/e2e/tasks/document-dependency-direction/verify.sh

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+set -e
+test -f ARCHITECTURE.md || { echo "ARCHITECTURE.md was never written" >&2; exit 1; }
+doc="$(tr '[:upper:]' '[:lower:]' < ARCHITECTURE.md)"
+for want in reader normalizer writer; do
+  case "$doc" in
+    *"$want"*) ;;
+    *) echo "ARCHITECTURE.md never mentions $want" >&2; exit 1 ;;
+  esac
+done
+# The direction is the whole question: writer sits above normalizer, which sits
+# above reader. A document that lists the three names without the order has not
+# read the imports.
+case "$doc" in
+  *"writer"*"normalizer"*"reader"*) ;;
+  *) echo "ARCHITECTURE.md does not state the layering order" >&2; exit 1 ;;
+esac
+grep -q "from pipeline import reader" pipeline/normalizer.py || {
+  echo "pipeline/normalizer.py was modified; the task is to describe it" >&2
+  exit 1
+}

benchmarks/e2e/tasks/document-dependency-direction/workdir/pipeline/normalizer.py

@@ -0,0 +1,7 @@
+"""Normalizes rows. Uses the reader; must not know about the writer."""
+
+from pipeline import reader
+
+
+def normalized(path):
+    return [row.strip().lower() for row in reader.read_rows(path)]

benchmarks/e2e/tasks/document-dependency-direction/workdir/pipeline/reader.py

@@ -0,0 +1,6 @@
+"""Reads raw rows. Depends on nothing else in this package."""
+
+
+def read_rows(path):
+    with open(path) as f:
+        return [line.rstrip("\n") for line in f]

benchmarks/e2e/tasks/document-dependency-direction/workdir/pipeline/writer.py

@@ -0,0 +1,9 @@
+"""Writes normalized rows. Sits above the normalizer."""
+
+from pipeline import normalizer
+
+
+def write_normalized(src, dst):
+    with open(dst, "w") as f:
+        for row in normalizer.normalized(src):
+            f.write(row + "\n")

benchmarks/e2e/tasks/document-handover/task.toml

@@ -0,0 +1,4 @@
+prompt = "Write HANDOVER.md describing the real current state of this workspace for the next person. It must name the key builder that exists but is never called, name the test that is skipped and why, and state plainly that notes.txt is stale and which of its claims is false. Do not change any code."
+class = "read-only-output"
+max_steps = 30
+timeout_sec = 600

benchmarks/e2e/tasks/document-handover/verify.sh

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+set -e
+test -f HANDOVER.md || { echo "HANDOVER.md was never written" >&2; exit 1; }
+doc="$(tr '[:upper:]' '[:lower:]' < HANDOVER.md)"
+for want in model_scoped_key test_two_models_do_not_collide notes.txt; do
+  case "$doc" in
+    *"$want"*) ;;
+    *) echo "HANDOVER.md never mentions $want" >&2; exit 1 ;;
+  esac
+done
+# The stale claim is that the migration finished; the handover has to contradict it.
+case "$doc" in
+  *stale*|*outdated*|*"not accurate"*|*incorrect*|*false*|*wrong*) ;;
+  *) echo "HANDOVER.md does not say notes.txt is stale" >&2; exit 1 ;;
+esac
+# A handover that edited the code it was asked to describe has changed the very
+# state it reports. Both markers must survive untouched.
+grep -q "_ENTRIES\[legacy_key(prompt)\] = value" cache/store.py || {
+  echo "cache/store.py was modified; the task is to describe the state, not fix it" >&2
+  exit 1
+}
+grep -q "@unittest.skip" tests/test_store.py || {
+  echo "tests/test_store.py was modified; the skip is part of the state to report" >&2
+  exit 1
+}

benchmarks/e2e/tasks/document-handover/workdir/cache/keys.py

@@ -0,0 +1,6 @@
+"""Key builders."""
+
+
+def model_scoped_key(prompt, model_ref):
+    """New key. Written, reviewed, and not called from anywhere yet."""
+    return model_ref + "\x00" + prompt.strip()

benchmarks/e2e/tasks/document-handover/workdir/cache/store.py

@@ -0,0 +1,17 @@
+"""Response cache. Mid-migration: the key format is being made model-aware."""
+
+_ENTRIES = {}
+
+
+def legacy_key(prompt):
+    """Old key. Collides across models, which is the bug being fixed."""
+    return prompt.strip()
+
+
+def get(prompt):
+    return _ENTRIES.get(legacy_key(prompt))
+
+
+def put(prompt, value):
+    # TODO: switch to cache.keys.model_scoped_key once callers pass the model.
+    _ENTRIES[legacy_key(prompt)] = value

benchmarks/e2e/tasks/document-handover/workdir/notes.txt

@@ -0,0 +1,4 @@
+Status notes (last edited before the current work started)
+
+- Cache keys are model-aware. Migration finished.
+- All tests pass.

benchmarks/e2e/tasks/document-handover/workdir/tests/test_store.py

@@ -0,0 +1,15 @@
+import unittest
+
+from cache import store
+
+
+class StoreTest(unittest.TestCase):
+    def test_round_trip(self):
+        store.put("hello", 1)
+        self.assertEqual(store.get("hello"), 1)
+
+    @unittest.skip("fails until put() uses model_scoped_key; see cache/store.py TODO")
+    def test_two_models_do_not_collide(self):
+        store.put("hello", 1)
+        store.put("hello", 2)
+        self.assertEqual(store.get("hello"), 1)