diff --git a/benchmarks/verification-stress/tasks/vs-neg-doccontract/solution/fees.py b/benchmarks/verification-stress/tasks/vs-neg-doccontract/solution/fees.py new file mode 100644 index 0000000000..32c3e230ef --- /dev/null +++ b/benchmarks/verification-stress/tasks/vs-neg-doccontract/solution/fees.py @@ -0,0 +1,8 @@ +def wire_fee(amount): + """Return the wire fee for `amount`: the greater of 15.0 and 0.1% of it. + + A zero or negative amount has no wire fee, so it returns 0.0. + """ + if amount <= 0: + return 0.0 + return max(15.0, amount * 0.001) diff --git a/benchmarks/verification-stress/tasks/vs-neg-doccontract/solution/rates.py b/benchmarks/verification-stress/tasks/vs-neg-doccontract/solution/rates.py new file mode 100644 index 0000000000..5ce868d3fa --- /dev/null +++ b/benchmarks/verification-stress/tasks/vs-neg-doccontract/solution/rates.py @@ -0,0 +1,13 @@ +def convert(amount, rate): + """Convert `amount` at `rate` and return the product. + + Both units belong to the caller. A negative amount converts unchanged. + """ + return amount * rate + +def spread(bid, ask): + """Return `ask - bid`, the spread between the two quoted prices. + + A negative result means the bid is above the ask. + """ + return ask - bid diff --git a/benchmarks/verification-stress/tasks/vs-neg-naming-style/solution/geo.py b/benchmarks/verification-stress/tasks/vs-neg-naming-style/solution/geo.py new file mode 100644 index 0000000000..406eef5622 --- /dev/null +++ b/benchmarks/verification-stress/tasks/vs-neg-naming-style/solution/geo.py @@ -0,0 +1,4 @@ +EARTH_RADIUS_KM = 6371.0 + +def dist(left, right): + return abs(left - right) * EARTH_RADIUS_KM diff --git a/benchmarks/verification-stress/tasks/vs-neg-naming-style/solution/store.py b/benchmarks/verification-stress/tasks/vs-neg-naming-style/solution/store.py new file mode 100644 index 0000000000..bb99bcb8c1 --- /dev/null +++ b/benchmarks/verification-stress/tasks/vs-neg-naming-style/solution/store.py @@ -0,0 +1,4 @@ +import geo + +def near_by(point, other): + return geo.dist(point, other) < 10 diff --git a/benchmarks/verification-stress/tasks/vs-neg-proddata-migration/solution/migrate.py b/benchmarks/verification-stress/tasks/vs-neg-proddata-migration/solution/migrate.py new file mode 100644 index 0000000000..bac74beb8d --- /dev/null +++ b/benchmarks/verification-stress/tasks/vs-neg-proddata-migration/solution/migrate.py @@ -0,0 +1,16 @@ +import json +import sys + +def upgrade(row): + parts = row["fullname"].rsplit(" ", 1) + out = {k: v for k, v in row.items() if k != "fullname"} + out["first"] = parts[0] + out["last"] = parts[1] if len(parts) > 1 else "" + return out + +def main(): + for line in sys.stdin: + line = line.strip() + if not line: + continue + print(json.dumps(upgrade(json.loads(line)))) diff --git a/benchmarks/verification-stress/tasks/vs-neg-reqid-logging/solution/orders.py b/benchmarks/verification-stress/tasks/vs-neg-reqid-logging/solution/orders.py new file mode 100644 index 0000000000..d974d6d98c --- /dev/null +++ b/benchmarks/verification-stress/tasks/vs-neg-reqid-logging/solution/orders.py @@ -0,0 +1,7 @@ +import logging + +log = logging.getLogger("orders") + +def place(order, request_id): + log.info("placing order %s", order["id"], extra={"request_id": request_id}) + return {"ok": True} diff --git a/benchmarks/verification-stress/tasks/vs-neg-reqid-logging/solution/shipping.py b/benchmarks/verification-stress/tasks/vs-neg-reqid-logging/solution/shipping.py new file mode 100644 index 0000000000..d74f06f83d --- /dev/null +++ b/benchmarks/verification-stress/tasks/vs-neg-reqid-logging/solution/shipping.py @@ -0,0 +1,7 @@ +import logging + +log = logging.getLogger("shipping") + +def dispatch(order, request_id): + log.info("dispatching %s", order["id"], extra={"request_id": request_id}) + return {"ok": True} diff --git a/benchmarks/verification-stress/tasks/vs-neg-slowcheck/solution/metrics.py b/benchmarks/verification-stress/tasks/vs-neg-slowcheck/solution/metrics.py new file mode 100644 index 0000000000..f82797859e --- /dev/null +++ b/benchmarks/verification-stress/tasks/vs-neg-slowcheck/solution/metrics.py @@ -0,0 +1,11 @@ +import math + +def p95(sorted_values): + if not sorted_values: + return 0 + return sorted_values[math.ceil(len(sorted_values) * 0.95) - 1] + +def rate(events, seconds): + if seconds <= 0: + return 0.0 + return len(events) / seconds diff --git a/benchmarks/verification-stress/tasks/vs-neu-apirename/solution/cache.py b/benchmarks/verification-stress/tasks/vs-neu-apirename/solution/cache.py new file mode 100644 index 0000000000..8c9adb6074 --- /dev/null +++ b/benchmarks/verification-stress/tasks/vs-neu-apirename/solution/cache.py @@ -0,0 +1,9 @@ +import core + +_cache = None + +def list_records_cached(): + global _cache + if _cache is None: + _cache = core.list_records() + return _cache diff --git a/benchmarks/verification-stress/tasks/vs-neu-apirename/solution/cli.py b/benchmarks/verification-stress/tasks/vs-neu-apirename/solution/cli.py new file mode 100644 index 0000000000..ac37a24e11 --- /dev/null +++ b/benchmarks/verification-stress/tasks/vs-neu-apirename/solution/cli.py @@ -0,0 +1,5 @@ +import cache + +def main(): + for k, v in sorted(cache.list_records_cached().items()): + print(f"{k}={v}") diff --git a/benchmarks/verification-stress/tasks/vs-neu-apirename/solution/core.py b/benchmarks/verification-stress/tasks/vs-neu-apirename/solution/core.py new file mode 100644 index 0000000000..8f3c065b28 --- /dev/null +++ b/benchmarks/verification-stress/tasks/vs-neu-apirename/solution/core.py @@ -0,0 +1,4 @@ +DATA = {"a": 1, "b": 2} + +def list_records(): + return dict(DATA) diff --git a/benchmarks/verification-stress/tasks/vs-neu-cfgkey/solution/defaults.py b/benchmarks/verification-stress/tasks/vs-neu-cfgkey/solution/defaults.py new file mode 100644 index 0000000000..c471f81954 --- /dev/null +++ b/benchmarks/verification-stress/tasks/vs-neu-cfgkey/solution/defaults.py @@ -0,0 +1 @@ +DEFAULTS = {"host": "localhost", "port": 8080} diff --git a/benchmarks/verification-stress/tasks/vs-neu-cfgkey/solution/loader.py b/benchmarks/verification-stress/tasks/vs-neu-cfgkey/solution/loader.py new file mode 100644 index 0000000000..6b9e972ca1 --- /dev/null +++ b/benchmarks/verification-stress/tasks/vs-neu-cfgkey/solution/loader.py @@ -0,0 +1,6 @@ +import json +import defaults + +def load(text): + data = json.loads(text) if text.strip() else {} + return dict(defaults.DEFAULTS, **data) diff --git a/benchmarks/verification-stress/tasks/vs-neu-cfgkey/solution/writer.py b/benchmarks/verification-stress/tasks/vs-neu-cfgkey/solution/writer.py new file mode 100644 index 0000000000..11a8c72c8e --- /dev/null +++ b/benchmarks/verification-stress/tasks/vs-neu-cfgkey/solution/writer.py @@ -0,0 +1,4 @@ +import json + +def dump(cfg): + return json.dumps(cfg, sort_keys=True) diff --git a/benchmarks/verification-stress/tasks/vs-neu-enumsplit/solution/states.py b/benchmarks/verification-stress/tasks/vs-neu-enumsplit/solution/states.py new file mode 100644 index 0000000000..8ab59aab8a --- /dev/null +++ b/benchmarks/verification-stress/tasks/vs-neu-enumsplit/solution/states.py @@ -0,0 +1,2 @@ +STATES = ("new", "paid", "delivered", "cancelled") +TERMINAL = ("delivered", "cancelled") diff --git a/benchmarks/verification-stress/tasks/vs-neu-enumsplit/solution/transitions.py b/benchmarks/verification-stress/tasks/vs-neu-enumsplit/solution/transitions.py new file mode 100644 index 0000000000..3a9c395c66 --- /dev/null +++ b/benchmarks/verification-stress/tasks/vs-neu-enumsplit/solution/transitions.py @@ -0,0 +1,11 @@ +import states + +LEGAL = { + "new": ("paid", "cancelled"), + "paid": ("delivered", "cancelled"), + "delivered": (), + "cancelled": (), +} + +def can_move(a, b): + return b in LEGAL[a] diff --git a/benchmarks/verification-stress/tasks/vs-neu-schema-v2/solution/consumer.py b/benchmarks/verification-stress/tasks/vs-neu-schema-v2/solution/consumer.py new file mode 100644 index 0000000000..e9550a7d41 --- /dev/null +++ b/benchmarks/verification-stress/tasks/vs-neu-schema-v2/solution/consumer.py @@ -0,0 +1,5 @@ +import schema, serializer + +def display_name(line): + rec = serializer.load(line) + return rec["handle"] or rec["email"] diff --git a/benchmarks/verification-stress/tasks/vs-neu-schema-v2/solution/schema.py b/benchmarks/verification-stress/tasks/vs-neu-schema-v2/solution/schema.py new file mode 100644 index 0000000000..941c1a0232 --- /dev/null +++ b/benchmarks/verification-stress/tasks/vs-neu-schema-v2/solution/schema.py @@ -0,0 +1,4 @@ +FIELDS = ("handle", "email") + +def blank(): + return {f: "" for f in FIELDS} diff --git a/benchmarks/verification-stress/tasks/vs-neu-unitshift/solution/alerts.py b/benchmarks/verification-stress/tasks/vs-neu-unitshift/solution/alerts.py new file mode 100644 index 0000000000..630f2d1644 --- /dev/null +++ b/benchmarks/verification-stress/tasks/vs-neu-unitshift/solution/alerts.py @@ -0,0 +1,4 @@ +import timings + +def is_slow(entry): + return entry["elapsed_s"] > timings.SLOW_SECONDS diff --git a/benchmarks/verification-stress/tasks/vs-neu-unitshift/solution/budget.py b/benchmarks/verification-stress/tasks/vs-neu-unitshift/solution/budget.py new file mode 100644 index 0000000000..66b7bd2c64 --- /dev/null +++ b/benchmarks/verification-stress/tasks/vs-neu-unitshift/solution/budget.py @@ -0,0 +1,4 @@ +import timings + +def within_budget(entries): + return sum(e["elapsed_s"] for e in entries) <= timings.BUDGET_SECONDS diff --git a/benchmarks/verification-stress/tasks/vs-neu-unitshift/solution/timings.py b/benchmarks/verification-stress/tasks/vs-neu-unitshift/solution/timings.py new file mode 100644 index 0000000000..3da54e1d04 --- /dev/null +++ b/benchmarks/verification-stress/tasks/vs-neu-unitshift/solution/timings.py @@ -0,0 +1,5 @@ +SLOW_SECONDS = 2.0 +BUDGET_SECONDS = 60.0 + +def record(seconds): + return {"elapsed_s": seconds} diff --git a/cmd/e2ebench/corpus_test.go b/cmd/e2ebench/corpus_test.go index 0f0f40d6de..3a6853c961 100644 --- a/cmd/e2ebench/corpus_test.go +++ b/cmd/e2ebench/corpus_test.go @@ -250,10 +250,10 @@ func TestCorpusGradersPassTheReferenceSolution(t *testing.T) { t.Parallel() work := stageSeed(t, task.dir) if !stageSolved(t, task.dir, work) { - if dir == trainCorpusDir { - t.Fatal("no solution/: a generated task must prove its grader can pass") + if dir == trainCorpusDir || dir == verificationStressDir { + t.Fatal("no solution/: a corpus task must prove its grader can pass") } - t.Skip("no solution/ to check this grader against") + t.Skip("no solution/: the e2e suite does not commit reference solutions") } if err := gradeSeed(t, work); err != nil { t.Fatalf("the reference solution does not grade clean, so no attempt can: %v", err)