V1

.github/workflows/bench.yml

@@ -0,0 +1,161 @@
+name: Benchmark stock vs stack
+
+on:
+  workflow_dispatch:
+    inputs:
+      model:
+        description: "Model for BOTH arms, provider/model[#variant] (falls back to the BENCH_MODEL repo variable)"
+        required: false
+        default: ""
+      waves:
+        description: "Number of waves (each wave runs every task once per arm)"
+        required: true
+        default: "3"
+      tasks:
+        description: "Comma separated task ids, or 'all'"
+        required: true
+        default: "all"
+      opencode_version:
+        description: "npm version of @opencode/cli to install (e.g. latest, 2.1.1)"
+        required: true
+        default: "latest"
+      stack_agent:
+        description: "Optional agent name to run the stack arm with (e.g. bench-builder). Empty = opencode's default agent"
+        required: false
+        default: ""
+      timeout:
+        description: "Per-task timeout in seconds (0 = use each task's task.json)"
+        required: true
+        default: "0"
+      max_parallel:
+        description: "Max jobs running at once (lower this if you hit provider rate limits)"
+        required: true
+        default: "4"
+
+permissions:
+  contents: read
+
+concurrency:
+  group: bench-${{ github.ref }}
+  cancel-in-progress: false
+
+jobs:
+  plan:
+    runs-on: ubuntu-latest
+    outputs:
+      waves: ${{ steps.plan.outputs.waves }}
+      model: ${{ steps.plan.outputs.model }}
+    steps:
+      - id: plan
+        env:
+          WAVES: ${{ inputs.waves }}
+          MODEL: ${{ inputs.model || vars.BENCH_MODEL }}
+        run: |
+          if [ -z "$MODEL" ]; then
+            echo "::error::Set the 'model' input or a BENCH_MODEL repository variable (e.g. anthropic/claude-sonnet-4-5)."
+            exit 1
+          fi
+          case "$WAVES" in (''|*[!0-9]*) echo "::error::waves must be a positive integer"; exit 1;; esac
+          if [ "$WAVES" -lt 1 ] || [ "$WAVES" -gt 20 ]; then echo "::error::waves must be between 1 and 20"; exit 1; fi
+          echo "waves=$(python3 -c "import json; print(json.dumps(list(range(1, $WAVES + 1))))")" >> "$GITHUB_OUTPUT"
+          echo "model=$MODEL" >> "$GITHUB_OUTPUT"
+
+  run:
+    needs: plan
+    runs-on: ubuntu-latest
+    timeout-minutes: 120
+    strategy:
+      fail-fast: false
+      max-parallel: ${{ fromJSON(inputs.max_parallel) }}
+      matrix:
+        wave: ${{ fromJSON(needs.plan.outputs.waves) }}
+        arm: [stock, stack]
+    name: wave ${{ matrix.wave }} / ${{ matrix.arm }}
+    env:
+      # Provider keys: add the ones you use as repository secrets. Both arms see the same keys.
+      ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+      OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
+      GOOGLE_GENERATIVE_AI_API_KEY: ${{ secrets.GOOGLE_GENERATIVE_AI_API_KEY }}
+      OPENCODE_API_KEY: ${{ secrets.OPENCODE_API_KEY }}
+      # PLACEHOLDER: secrets your MCP servers or plugins need. Reference them in
+      # arms/stack/opencode.json as {env:NAME}. Exposed to both arms; the stock arm ignores them.
+      # GITHUB_PERSONAL_ACCESS_TOKEN: ${{ secrets.MCP_GITHUB_TOKEN }}
+      # CONTEXT7_API_KEY: ${{ secrets.CONTEXT7_API_KEY }}
+    steps:
+      - uses: actions/checkout@v5
+
+      - uses: actions/setup-node@v5
+        with:
+          node-version: "22"
+
+      - uses: actions/setup-python@v6
+        with:
+          python-version: "3.12"
+
+      # Most local MCP servers run through npx (Node, above) or uvx (uv, below).
+      - uses: astral-sh/setup-uv@v6
+
+      - name: Install opencode
+        run: |
+          npm install -g "@opencode/cli@${{ inputs.opencode_version }}"
+          opencode --version
+
+      - name: Validate stack config
+        if: matrix.arm == 'stack'
+        run: bash bench/inspect_arm.sh arms/stack
+
+      - name: Run wave
+        env:
+          MODEL: ${{ needs.plan.outputs.model }}
+          ARM: ${{ matrix.arm }}
+          WAVE: ${{ matrix.wave }}
+          TASKS: ${{ inputs.tasks }}
+          TIMEOUT: ${{ inputs.timeout }}
+          STACK_AGENT: ${{ inputs.stack_agent }}
+        run: |
+          agent=""
+          if [ "$ARM" = "stack" ] && [ -n "$STACK_AGENT" ]; then agent="$STACK_AGENT"; fi
+          python bench/run_arm.py --arm "$ARM" --wave "$WAVE" --model "$MODEL" \
+            --agent "$agent" --tasks "$TASKS" --timeout "$TIMEOUT" --out results
+
+      - uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: results-${{ matrix.arm }}-w${{ matrix.wave }}
+          path: results/
+          if-no-files-found: warn
+          retention-days: 30
+
+  report:
+    needs: run
+    if: always()
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v5
+
+      - uses: actions/setup-python@v6
+        with:
+          python-version: "3.12"
+
+      - uses: actions/download-artifact@v5
+        with:
+          pattern: results-*
+          path: results
+          merge-multiple: true
+
+      - name: Compare arms
+        run: |
+          set +e
+          python bench/report.py results --out report
+          status=$?
+          cat report/summary.md >> "$GITHUB_STEP_SUMMARY" 2>/dev/null
+          exit $status
+
+      - uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: report
+          path: |
+            report/
+            results/

.github/workflows/selftest.yml

@@ -0,0 +1,24 @@
+name: Self-test task suite
+
+# Checks the benchmark itself without calling any model: every reference solution must
+# pass its hidden tests, and no untouched fixture may already pass them.
+on:
+  push:
+  pull_request:
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  selftest:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v5
+      - uses: actions/setup-node@v5
+        with:
+          node-version: "22"
+      - uses: actions/setup-python@v6
+        with:
+          python-version: "3.12"
+      - run: bash bench/selftest.sh

.gitignore

@@ -0,0 +1,5 @@
+results/
+report/
+__pycache__/
+*.pyc
+node_modules/

README.md

@@ -0,0 +1,121 @@
+# opencode-stack-bench
+
+A small A/B benchmark that answers one question: **does my opencode setup actually help?**
+
+It runs the same coding tasks, with the same model, through two arms of opencode v2:
+
+| Arm | Config dir | What it gets |
+|---|---|---|
+| `stock` | `arms/stock/` (empty) | opencode's built-in defaults only |
+| `stack` | `arms/stack/` | your `opencode.json`, `AGENTS.md`, agents, skills, plugins and MCP servers |
+
+Each **wave** runs every task once per arm. Several waves give you a repeated, paired measurement,
+and the report tells you whether the difference is real or noise.
+
+## Quick start
+
+1. **Fill in your stack.** Replace the placeholders in [`arms/stack/`](arms/stack/README.md).
+   Everything marked `PLACEHOLDER` is an example.
+2. **Add a provider key** as a repository secret (`ANTHROPIC_API_KEY`, `OPENAI_API_KEY`,
+   `OPENROUTER_API_KEY`, `GOOGLE_GENERATIVE_AI_API_KEY` or `OPENCODE_API_KEY` are wired up already;
+   add others to the `env:` block in `.github/workflows/bench.yml`).
+   Secrets for MCP servers or plugins go in the same block.
+3. **Run it.** Actions → *Benchmark stock vs stack* → *Run workflow*. Set `model`
+   (for example `anthropic/claude-sonnet-4-5`; run `opencode models` locally for exact ids),
+   or set a `BENCH_MODEL` repository variable once.
+4. **Read the result** on the run's summary page. The `report` artifact has `summary.md`,
+   `summary.json`, every raw result row, and each run's full opencode event log.
+
+## What a run does
+
+```
+plan ──► run (wave × arm matrix) ──► report
+```
+
+For each task, `bench/run_arm.py`:
+
+1. copies `tasks/<id>/repo/` into a fresh git workspace;
+2. gives opencode a fresh `HOME`/XDG tree, `OPENCODE_CONFIG_DIR=arms/<arm>`, and
+   `OPENCODE_DISABLE_CLAUDE_CODE=1`, so nothing leaks between tasks or arms;
+3. runs `opencode run --standalone --auto --format json -m <model> "<prompt.md>"` with a timeout;
+4. copies in the hidden tests from `tasks/<id>/hidden/` (the agent never sees them) and scores
+   the fraction that pass;
+5. records tokens, cost, steps, tool calls, wall time and diff size from the JSON event stream.
+
+The stack arm job also prints what opencode really loaded from `arms/stack/` (config sources,
+agents, skills, plugins, MCP servers) in the **Validate stack config** step. opencode drops unknown
+config keys silently, so check that step after you change the stack.
+
+## Reading the report
+
+- **Verdict** uses the paired difference (stack minus stock) over every (task, wave) pair, with a
+  bootstrap 95% confidence interval. "Helps" means the whole interval is above zero.
+- **Sign test** counts how many pairs the stack won, tied and lost.
+- **Wave-to-wave SD** shows how noisy the model is. If it is large compared to the gap
+  between arms, run more waves.
+- **Infra errors** (bad key, unknown model, crash before any model step) are excluded and listed.
+- Cost and time matter too. A stack that ties on score but is cheaper or faster can still be worth it.
+
+Rough guide: 3 waves × 6 tasks gives 18 pairs, enough to spot a large effect. For a small one,
+use 5 to 10 waves or add tasks.
+
+## Workflow inputs
+
+| Input | Default | Notes |
+|---|---|---|
+| `model` | `vars.BENCH_MODEL` | `provider/model[#variant]`, same for both arms |
+| `waves` | `3` | 1 to 20 |
+| `tasks` | `all` | comma separated ids from `tasks/` |
+| `opencode_version` | `latest` | npm version of `@opencode/cli` |
+| `stack_agent` | empty | run the stack arm with `--agent <name>` (e.g. `bench-builder`) |
+| `timeout` | `0` | per-task seconds; 0 uses each `task.json` |
+| `max_parallel` | `4` | lower it if you hit rate limits |
+
+## Tasks
+
+| Id | Kind |
+|---|---|
+| `py-business-days` | bug fix with edge cases |
+| `py-wordfreq-cli` | add CLI flags without breaking behaviour |
+| `py-parse-duration` | implement a parser from a spec |
+| `py-inventory` | multi-module bug hunt plus a new method |
+| `js-lru-cache` | implement a data structure with TTL |
+| `js-semver-sort` | fix and complete a SemVer library (hardest) |
+
+Add your own: create `tasks/<id>/` with `task.json` (`runner` is `python` or `node`, `tests` is the
+hidden test count), `prompt.md`, `repo/` (starting code), `hidden/` (tests: `test*.py` for Python,
+`*.test.js` for Node) and `solution/` (files copied over `repo/` that must pass every hidden test).
+Tasks that mirror your real work tell you the most.
+
+The **Self-test task suite** workflow runs on every push without calling a model. It checks that
+each reference solution passes and no untouched fixture already passes.
+
+## Run locally
+
+```bash
+npm i -g @opencode/cli
+export ANTHROPIC_API_KEY=...
+python3 bench/run_arm.py --arm stock --wave 1 --model anthropic/claude-sonnet-4-5 --out results
+python3 bench/run_arm.py --arm stack --wave 1 --model anthropic/claude-sonnet-4-5 --out results
+python3 bench/report.py results
+bash bench/inspect_arm.sh arms/stack   # what opencode loads from your stack
+bash bench/selftest.sh                 # validate the task suite
+```
+
+`shared/opencode.json` is applied to both arms through `OPENCODE_CONFIG`. Use it for things that
+must be identical in both, such as a custom provider block.
+
+## Notes on opencode v2
+
+Checked against `@opencode/cli` 2.1.1:
+
+- v2 config keys are `agents` (with `system` for the prompt), `plugins`, and `mcp.servers.<name>`
+  (with `disabled`). v1 names (`agent`, `plugin`, `mcp.<name>`, `prompt`, `enabled`) are migrated
+  automatically.
+- From the config dir, opencode loads `opencode.json`, `AGENTS.md`, `agents/*.md`,
+  `skills/<name>/SKILL.md` and `plugins/*.{ts,js}`.
+- Relative `instructions` paths resolve against the task workspace, not the config dir. Use
+  `{env:OPENCODE_CONFIG_DIR}/file.md` for extra rule files that live in `arms/stack/`.
+- `opencode run` without `--standalone` talks to a shared background service, which would mix
+  configs between arms; the harness always passes `--standalone`.
+- opencode picks the project directory from `$PWD`, so the harness sets it for each task.

arms/stack/AGENTS.md

@@ -0,0 +1,7 @@
+<!-- PLACEHOLDER global rules, AGENTS.md in the config dir is loaded automatically as global rules. Replace with your own. -->
+# Working rules
+
+- Read the code and existing tests before changing anything.
+- Run the tests after every change and keep going until they pass.
+- Handle edge cases the task statement mentions explicitly, and test them yourself.
+- Keep diffs minimal: do not reformat or rename things you were not asked to touch.

arms/stack/README.md

@@ -0,0 +1,20 @@
+# Stack arm (fill this in)
+
+Everything in this folder becomes opencode's config directory (`OPENCODE_CONFIG_DIR`) for the
+stack arm. Replace the placeholders with your real setup:
+
+| Path | What it is |
+|---|---|
+| `opencode.json` | Main config: `agents`, `mcp.servers`, `plugins`, `skills`, `instructions`, permissions, etc. |
+| `AGENTS.md` | Global rules, loaded automatically from the config dir. |
+| `agents/*.md` | Agents as markdown (frontmatter + system prompt). |
+| `skills/<name>/SKILL.md` | Skills. The folder name is the skill id. |
+| `plugins/*.ts` | Local plugins, loaded automatically. npm plugins go in `opencode.json` → `plugins`. |
+
+Do not set `model` here if you want a fair test: the workflow passes the same `--model` to both
+arms. If your stack has its own primary agent, set the workflow's `stack_agent` input to its name.
+
+Secrets that MCP servers or plugins need go in repository secrets and get exposed in the
+workflow's `env:` block (see the comment there). Reference them in config as `{env:NAME}`.
+
+This README is not copied into the config dir.

arms/stack/agents/reviewer.md

@@ -0,0 +1,7 @@
+---
+description: PLACEHOLDER subagent. Reviews a diff for bugs and missed edge cases before the main agent finishes.
+mode: subagent
+---
+You are a strict code reviewer. Look at the current git diff and the task statement you are given.
+List concrete bugs, unhandled edge cases, and spec mismatches, each with the file and line.
+Do not edit files. If everything looks right, say so in one line.

arms/stack/opencode.json

@@ -0,0 +1,28 @@
+{
+  "$schema": "https://opencode.ai/config.json",
+
+  "//": "PLACEHOLDER STACK. Replace everything in arms/stack/ with your real setup. Keys below use the v2 names (agents, plugins, mcp.servers); opencode v2 also auto-migrates v1 names (agent, plugin, mcp.<name>). Unknown keys are dropped silently, so check the 'Validate stack config' step in the workflow log.",
+
+  "//instructions": "AGENTS.md next to this file is loaded automatically. Extra rule files can be added with \"instructions\": [\"{env:OPENCODE_CONFIG_DIR}/more-rules.md\"] (relative paths resolve against the task workspace, not this folder).",
+
+  "agents": {
+    "bench-builder": {
+      "description": "PLACEHOLDER primary agent. Replace with your own, or delete and keep using 'build'.",
+      "mode": "primary",
+      "system": "You are a careful software engineer. Before editing, read the relevant files and any existing tests. After editing, run the project's tests (python3 -m unittest, or node --test) and fix failures before you finish. Prefer small, focused diffs."
+    }
+  },
+
+  "mcp": {
+    "servers": {
+      "example-fetch": {
+        "//": "PLACEHOLDER MCP server. Swap for the servers you actually use. Use \"disabled\": true to keep one declared but off.",
+        "type": "local",
+        "command": ["uvx", "mcp-server-fetch"],
+        "disabled": true
+      }
+    }
+  },
+
+  "plugins": []
+}

arms/stack/plugins/bench-marker.ts

@@ -0,0 +1,6 @@
+// PLACEHOLDER plugin. Replace with your own plugins, or list npm plugins under "plugins" in opencode.json.
+// Local plugins in this folder are loaded automatically. This one does nothing, it only proves
+// the plugin pipeline works in CI (it shows up in `opencode plugin list`).
+export const BenchMarker = async () => {
+  return {}
+}

arms/stack/service.json

@@ -0,0 +1,3 @@
+{
+  "password": "DAbtEKdarKm4aVYdwxS8V6HOe0BJAW_iLabc0zlxRXk"
+}

arms/stack/skills/minimal-diff/SKILL.md

@@ -0,0 +1,7 @@
+---
+name: minimal-diff
+description: PLACEHOLDER skill. Use when fixing a bug in existing code, to keep the change small and avoid regressions.
+---
+- Find the root cause first; reproduce it with a failing test or a one-off script.
+- Change the fewest lines that fix the cause. Keep public signatures unchanged.
+- Re-run existing tests to check for regressions before finishing.

arms/stack/skills/test-first/SKILL.md

@@ -0,0 +1,8 @@
+---
+name: test-first
+description: PLACEHOLDER skill. Use when implementing or fixing behaviour described in a spec, to write edge-case tests before changing code.
+---
+1. Turn every rule in the task statement into at least one test case, including error cases.
+2. Run the tests and confirm the new ones fail for the right reason.
+3. Implement the change, then run the full test suite until it passes.
+4. Delete nothing you did not create unless the task asks for it.

arms/stock/README.md

@@ -0,0 +1,4 @@
+# Stock arm
+
+Intentionally empty. The stock arm runs opencode with this directory as its config dir,
+so it gets opencode's built-in defaults and nothing else. Leave it that way.

bench/inspect_arm.sh

@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+# Print what opencode actually loads from an arm's config dir: config documents, agents,
+# skills, plugins and MCP servers. Unknown config keys are dropped silently by opencode,
+# so compare this output against what you meant to declare.
+set -uo pipefail
+arm_dir="$(cd "$1" && pwd)"
+scratch="$(mktemp -d)"
+export HOME="$scratch/home" XDG_CONFIG_HOME="$scratch/home/.config" XDG_DATA_HOME="$scratch/home/.local/share" \
+  XDG_STATE_HOME="$scratch/home/.local/state" XDG_CACHE_HOME="$scratch/home/.cache" \
+  OPENCODE_CONFIG_DIR="$arm_dir" OPENCODE_DISABLE_AUTOUPDATE=1 OPENCODE_DISABLE_CLAUDE_CODE=1
+mkdir -p "$scratch/ws" && cd "$scratch/ws" && git init -q
+export PWD="$scratch/ws"
+
+opencode service start >/dev/null 2>&1 || true
+# Agents and skills load asynchronously after the server starts; wait until they show up.
+for _ in $(seq 1 30); do
+  n=$(opencode api GET /api/agent 2>/dev/null | python3 -c "import json,sys; print(len(json.load(sys.stdin)['data']))" 2>/dev/null || echo 0)
+  [ "${n:-0}" -gt 0 ] && break
+  sleep 2
+done
+
+section() { echo; echo "::group::$1"; shift; "$@" 2>&1 | head -c 20000; echo; echo "::endgroup::"; }
+section "Config sources (opencode debug config)" opencode debug config
+section "Agents" sh -c 'opencode api GET /api/agent | python3 -c "import json,sys; [print(a[\"id\"], a.get(\"mode\"), \"-\", (a.get(\"description\") or \"\")[:80]) for a in json.load(sys.stdin)[\"data\"]]"'
+section "Skills" sh -c 'opencode api GET /api/skill | python3 -c "import json,sys; [print(s[\"id\"], \"-\", (s.get(\"description\") or \"\")[:80]) for s in json.load(sys.stdin)[\"data\"]]"'
+section "Plugins" opencode plugin list
+section "MCP servers" opencode mcp list
+opencode service stop >/dev/null 2>&1 || true

bench/pycheck.py

@@ -0,0 +1,33 @@
+#!/usr/bin/env python3
+"""Run the hidden unittest files in <hidden_dir> against the current directory and print
+{"passed": n, "total": n} as the last line of stdout."""
+import json
+import os
+import sys
+import unittest
+
+
+def flatten(suite):
+    for item in suite:
+        if isinstance(item, unittest.TestSuite):
+            yield from flatten(item)
+        else:
+            yield item
+
+
+def main():
+    hidden = os.path.abspath(sys.argv[1])
+    sys.path.insert(0, os.getcwd())
+    suite = unittest.defaultTestLoader.discover(hidden, pattern="test*.py", top_level_dir=hidden)
+    tests = list(flatten(suite))
+    result = unittest.TextTestRunner(stream=sys.stdout, verbosity=2).run(unittest.TestSuite(tests))
+    # A failing subTest is reported as a _SubTest; count it against its parent test.
+    bad = {id(getattr(t, "test_case", t)) for t, _ in result.failures + result.errors}
+    # Import errors surface as a single failing _FailedTest, which counts as 0 passed.
+    passed = sum(1 for t in tests if id(t) not in bad and not isinstance(t, unittest.loader._FailedTest))
+    sys.stdout.flush()
+    print(json.dumps({"passed": passed, "total": len(tests)}))
+
+
+if __name__ == "__main__":
+    main()

bench/report.py

@@ -0,0 +1,202 @@
+#!/usr/bin/env python3
+"""Aggregate results/*.jsonl into a stock-vs-stack comparison.
+
+Writes summary.md (also suitable for $GITHUB_STEP_SUMMARY) and summary.json.
+The headline number is the paired difference in hidden-test score
+(stack minus stock) over every (task, wave) pair, with a bootstrap 95% CI.
+"""
+import argparse
+import json
+import math
+import random
+import statistics as st
+import sys
+from collections import defaultdict
+from pathlib import Path
+
+BASE, TREAT = "stock", "stack"
+
+
+def load(results_dir):
+    rows = []
+    for p in sorted(Path(results_dir).rglob("*.jsonl")):
+        if "logs" in p.parts:
+            continue
+        for line in p.read_text().splitlines():
+            if line.strip():
+                rows.append(json.loads(line))
+    return rows
+
+
+def mean(xs):
+    return st.fmean(xs) if xs else float("nan")
+
+
+def sd(xs):
+    return st.stdev(xs) if len(xs) > 1 else 0.0
+
+
+def bootstrap_ci(diffs, n=10000, seed=0):
+    if not diffs:
+        return float("nan"), float("nan")
+    rng = random.Random(seed)
+    k = len(diffs)
+    means = sorted(st.fmean(rng.choices(diffs, k=k)) for _ in range(n))
+    return means[int(0.025 * n)], means[int(0.975 * n) - 1]
+
+
+def sign_test_p(wins, losses):
+    """Exact two-sided binomial sign test, ties dropped."""
+    n = wins + losses
+    if n == 0:
+        return 1.0
+    k = min(wins, losses)
+    tail = sum(math.comb(n, i) for i in range(k + 1)) / 2 ** n
+    return min(1.0, 2 * tail)
+
+
+def fmt(x, nd=3):
+    if isinstance(x, float) and math.isnan(x):
+        return "n/a"
+    return f"{x:.{nd}f}" if isinstance(x, float) else str(x)
+
+
+def arm_stats(rows):
+    ok = [r for r in rows if not r.get("infra_error")]
+    waves = defaultdict(list)
+    for r in ok:
+        waves[r["wave"]].append(r["score"])
+    wave_means = [mean(v) for _, v in sorted(waves.items())]
+    return {
+        "runs": len(rows),
+        "infra_errors": len(rows) - len(ok),
+        "timeouts": sum(1 for r in ok if r.get("timed_out")),
+        "mean_score": mean([r["score"] for r in ok]),
+        "solve_rate": mean([1.0 if r["solved"] else 0.0 for r in ok]),
+        "wave_means": wave_means,
+        "wave_sd": sd(wave_means),
+        "mean_duration_s": mean([r.get("duration_s", 0) for r in ok]),
+        "mean_tokens_in": mean([r.get("tokens_in", 0) for r in ok]),
+        "mean_tokens_out": mean([r.get("tokens_out", 0) for r in ok]),
+        "mean_cost_usd": mean([r.get("cost_usd", 0) for r in ok]),
+        "total_cost_usd": sum(r.get("cost_usd", 0) for r in ok),
+        "mean_tool_calls": mean([r.get("tool_calls", 0) for r in ok]),
+        "mean_lines_changed": mean([r.get("lines_added", 0) + r.get("lines_removed", 0) for r in ok]),
+    }
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("results", nargs="?", default="results")
+    ap.add_argument("--out", default=None, help="directory for summary.md/json (default: results dir)")
+    ap.add_argument("--min-effect", type=float, default=0.02,
+                    help="score difference below which a significant result is still called negligible")
+    args = ap.parse_args()
+
+    rows = load(args.results)
+    if not rows:
+        sys.exit(f"no results found under {args.results}")
+    by_arm = defaultdict(list)
+    for r in rows:
+        by_arm[r["arm"]].append(r)
+    order = [a for a in (BASE, TREAT) if a in by_arm] + sorted(a for a in by_arm if a not in (BASE, TREAT))
+    stats = {arm: arm_stats(by_arm[arm]) for arm in order}
+
+    # Paired comparison per (task, wave).
+    idx = {(r["arm"], r["task"], r["wave"]): r for r in rows if not r.get("infra_error")}
+    pairs = []
+    for (arm, task, wave), r in idx.items():
+        if arm == TREAT and (BASE, task, wave) in idx:
+            pairs.append((task, wave, idx[(BASE, task, wave)], r))
+    diffs = [t["score"] - b["score"] for _, _, b, t in pairs]
+    wins = sum(1 for d in diffs if d > 1e-9)
+    losses = sum(1 for d in diffs if d < -1e-9)
+    ties = len(diffs) - wins - losses
+    lo, hi = bootstrap_ci(diffs)
+    mdiff = mean(diffs)
+    p = sign_test_p(wins, losses)
+
+    if not pairs:
+        verdict = "No paired runs: both arms need results for the same task and wave."
+    elif lo > 0 and mdiff >= args.min_effect:
+        verdict = f"**Your stack helps.** It scores {mdiff:+.3f} higher on average (95% CI {lo:+.3f} to {hi:+.3f})."
+    elif hi < 0 and -mdiff >= args.min_effect:
+        verdict = f"**Your stack hurts.** It scores {mdiff:+.3f} lower on average (95% CI {lo:+.3f} to {hi:+.3f})."
+    else:
+        verdict = (f"**No clear quality difference.** Mean difference {mdiff:+.3f} (95% CI {lo:+.3f} to {hi:+.3f}). "
+                   "Run more waves to narrow the interval, or compare cost and time below.")
+
+    per_task = defaultdict(lambda: {BASE: [], TREAT: []})
+    for r in rows:
+        if not r.get("infra_error") and r["arm"] in (BASE, TREAT):
+            per_task[r["task"]][r["arm"]].append(r)
+
+    md = ["# opencode stack benchmark", ""]
+    models = sorted({r.get("model") or "(config default)" for r in rows})
+    waves = sorted({r["wave"] for r in rows})
+    md += [f"Model: `{', '.join(models)}` · Waves: {len(waves)} · Tasks: {len(per_task)} · Paired runs: {len(pairs)}", ""]
+    md += [verdict, ""]
+    md += [f"Paired (task, wave) outcomes: stack better {wins}, tied {ties}, worse {losses}; sign test p = {p:.3f}.", ""]
+
+    md += ["## Arms", "", "| Metric | " + " | ".join(stats) + " |", "|---|" + "---|" * len(stats)]
+    metrics = [
+        ("Mean hidden-test score", "mean_score", 3), ("Solve rate (all tests pass)", "solve_rate", 3),
+        ("Wave-to-wave SD of mean score", "wave_sd", 3), ("Mean wall time (s)", "mean_duration_s", 1),
+        ("Mean input tokens", "mean_tokens_in", 0), ("Mean output tokens", "mean_tokens_out", 0),
+        ("Mean cost per task (USD)", "mean_cost_usd", 4), ("Total cost (USD)", "total_cost_usd", 4),
+        ("Mean tool calls", "mean_tool_calls", 1), ("Mean lines changed", "mean_lines_changed", 1),
+        ("Runs", "runs", None), ("Infra errors (excluded)", "infra_errors", None), ("Timeouts", "timeouts", None),
+    ]
+    for label, key, nd in metrics:
+        cells = []
+        for s in stats.values():
+            v = s[key]
+            cells.append(str(v) if nd is None else (fmt(float(v), nd) if nd else f"{v:,.0f}" if not math.isnan(v) else "n/a"))
+        md.append(f"| {label} | " + " | ".join(cells) + " |")
+    md.append("")
+
+    md += ["## Per task (mean score across waves)", "", f"| Task | {BASE} | {TREAT} | Δ | {BASE} time (s) | {TREAT} time (s) |",
+           "|---|---|---|---|---|---|"]
+    task_rows = {}
+    for task, arms in sorted(per_task.items()):
+        b = mean([r["score"] for r in arms[BASE]])
+        t = mean([r["score"] for r in arms[TREAT]])
+        bt = mean([r.get("duration_s", 0) for r in arms[BASE]])
+        tt = mean([r.get("duration_s", 0) for r in arms[TREAT]])
+        task_rows[task] = {BASE: b, TREAT: t, "delta": t - b}
+        md.append(f"| {task} | {fmt(b)} | {fmt(t)} | {fmt(t - b, 3) if not math.isnan(t - b) else 'n/a'} | {fmt(bt, 1)} | {fmt(tt, 1)} |")
+    md.append("")
+
+    md += ["## Per wave (mean score)", "", "| Wave | " + " | ".join(stats) + " |", "|---|" + "---|" * len(stats)]
+    for w in waves:
+        cells = [fmt(mean([r["score"] for r in by_arm[a] if r["wave"] == w and not r.get("infra_error")])) for a in stats]
+        md.append(f"| {w} | " + " | ".join(cells) + " |")
+    md.append("")
+
+    infra = [r for r in rows if r.get("infra_error")]
+    if infra:
+        md += ["## Infra errors", "", "These runs produced no model steps (bad API key, unknown model, crash) and are excluded:", ""]
+        for r in infra[:20]:
+            md.append(f"- {r['arm']} w{r['wave']} {r['task']}: exit {r.get('exit_code')} {r.get('first_error', '')}".rstrip())
+        md.append("")
+
+    md += ["## How to read this", "",
+           "- Score is the fraction of hidden tests passing after the agent finishes; the agent never sees these tests.",
+           "- Each wave runs every task once per arm, with the same model, so the only difference is your config.",
+           "- The CI comes from resampling the paired (task, wave) differences. If it straddles 0, you need more waves "
+           "or harder tasks before concluding anything.",
+           "- A stack that ties on score but is cheaper or faster can still be worth keeping.", ""]
+
+    out = Path(args.out or args.results)
+    out.mkdir(parents=True, exist_ok=True)
+    (out / "summary.md").write_text("\n".join(md))
+    (out / "summary.json").write_text(json.dumps({
+        "verdict": verdict, "mean_diff": mdiff, "ci95": [lo, hi], "wins": wins, "ties": ties, "losses": losses,
+        "sign_test_p": p, "arms": stats, "per_task": task_rows}, indent=2, default=str))
+    print("\n".join(md))
+    if not pairs:
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

bench/run_arm.py

@@ -0,0 +1,269 @@
+#!/usr/bin/env python3
+"""Run every task once for one arm (stock or stack) in one wave.
+
+Each task gets a fresh git workspace and a fresh HOME/XDG tree, so nothing leaks
+between tasks or arms. The only difference between arms is OPENCODE_CONFIG_DIR:
+`arms/stock` (empty) vs `arms/stack` (your opencode.json, agents, skills, plugins).
+
+Results are appended as JSON lines to <out>/<arm>-w<wave>.jsonl, with raw event
+logs under <out>/logs/.
+"""
+import argparse
+import json
+import os
+import random
+import shutil
+import signal
+import subprocess
+import sys
+import tempfile
+import time
+from collections import Counter
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parent.parent
+TASKS = ROOT / "tasks"
+CHECK_TIMEOUT_S = 180
+
+
+def load_tasks(selector):
+    tasks = sorted(p for p in TASKS.iterdir() if (p / "task.json").exists())
+    if selector and selector != "all":
+        wanted = {s.strip() for s in selector.split(",") if s.strip()}
+        unknown = wanted - {p.name for p in tasks}
+        if unknown:
+            sys.exit(f"unknown task(s): {', '.join(sorted(unknown))}")
+        tasks = [p for p in tasks if p.name in wanted]
+    return tasks
+
+
+def git(ws, *args):
+    subprocess.run(["git", *args], cwd=ws, check=True, capture_output=True)
+
+
+def make_workspace(task_dir, ws):
+    shutil.copytree(task_dir / "repo", ws)
+    git(ws, "init", "-q", "-b", "main")
+    (ws / ".git" / "info" / "exclude").write_text("__pycache__/\n*.pyc\nnode_modules/\n.bench_hidden/\n")
+    git(ws, "-c", "user.name=bench", "-c", "[email protected]", "add", "-A")
+    git(ws, "-c", "user.name=bench", "-c", "[email protected]", "commit", "-qm", "task fixture")
+
+
+def isolated_env(scratch, arm_dir, shared_config, cache_dir):
+    env = {k: v for k, v in os.environ.items() if not k.startswith("OPENCODE_")}
+    home = scratch / "home"
+    home.mkdir(parents=True, exist_ok=True)
+    env.update(
+        HOME=str(home),
+        XDG_CONFIG_HOME=str(home / ".config"),
+        XDG_DATA_HOME=str(home / ".local" / "share"),
+        XDG_STATE_HOME=str(home / ".local" / "state"),
+        XDG_CACHE_HOME=str(cache_dir),
+        OPENCODE_CONFIG_DIR=str(arm_dir),
+        OPENCODE_DISABLE_AUTOUPDATE="1",
+        # Ignore ~/.claude skills and CLAUDE.md so only what the arm declares counts.
+        OPENCODE_DISABLE_CLAUDE_CODE="1",
+        CI="1",
+    )
+    if shared_config and shared_config.exists():
+        env["OPENCODE_CONFIG"] = str(shared_config)
+    return env
+
+
+def run_opencode(args, task_dir, ws, env, log_base, timeout_s):
+    prompt = (task_dir / "prompt.md").read_text()
+    cmd = [args.opencode_bin, "run", "--standalone", "--auto", "--format", "json",
+           "--title", f"{task_dir.name} {args.arm} w{args.wave}"]
+    if args.model:
+        cmd += ["-m", args.model]
+    if args.agent:
+        cmd += ["--agent", args.agent]
+    cmd.append(prompt)
+    # opencode resolves the project directory from $PWD, not the process cwd.
+    env = {**env, "PWD": str(ws)}
+    events_path = log_base.with_suffix(".events.jsonl")
+    stderr_path = log_base.with_suffix(".stderr.log")
+    start = time.monotonic()
+    timed_out = False
+    with open(events_path, "w") as out, open(stderr_path, "w") as err:
+        proc = subprocess.Popen(cmd, cwd=ws, env=env, stdout=out, stderr=err,
+                                stdin=subprocess.DEVNULL, start_new_session=True)
+        try:
+            proc.wait(timeout=timeout_s)
+        except subprocess.TimeoutExpired:
+            timed_out = True
+            os.killpg(proc.pid, signal.SIGTERM)
+            try:
+                proc.wait(timeout=15)
+            except subprocess.TimeoutExpired:
+                os.killpg(proc.pid, signal.SIGKILL)
+                proc.wait()
+    duration = time.monotonic() - start
+    return proc.returncode, timed_out, duration, events_path, stderr_path
+
+
+def summarize_events(events_path):
+    """Sum usage across `step_finish` events and count tool calls from `--format json` output."""
+    s = {"steps": 0, "tokens_in": 0, "tokens_out": 0, "tokens_reasoning": 0,
+         "tokens_cache_read": 0, "tokens_cache_write": 0, "cost_usd": 0.0,
+         "tool_calls": 0, "tool_errors": 0, "error_events": 0, "events": 0}
+    tools = Counter()
+    for line in events_path.read_text(errors="replace").splitlines():
+        try:
+            ev = json.loads(line)
+        except json.JSONDecodeError:
+            continue
+        s["events"] += 1
+        etype = ev.get("type")
+        part = ev.get("part") or {}
+        if etype == "step_finish":
+            s["steps"] += 1
+            tok = part.get("tokens") or {}
+            cache = tok.get("cache") or {}
+            s["tokens_in"] += tok.get("input") or 0
+            s["tokens_out"] += tok.get("output") or 0
+            s["tokens_reasoning"] += tok.get("reasoning") or 0
+            s["tokens_cache_read"] += cache.get("read") or 0
+            s["tokens_cache_write"] += cache.get("write") or 0
+            s["cost_usd"] += part.get("cost") or 0
+        elif etype == "tool_use":
+            s["tool_calls"] += 1
+            tools[part.get("tool", "?")] += 1
+            if (part.get("state") or {}).get("status") == "error":
+                s["tool_errors"] += 1
+        elif etype == "error":
+            s["error_events"] += 1
+            err = ev.get("error") or {}
+            s.setdefault("first_error", f"{err.get('type', '')}: {err.get('message', '')}"[:500])
+    s["tools"] = dict(tools)
+    return s
+
+
+def diff_stats(ws):
+    subprocess.run(["git", "add", "-A"], cwd=ws, capture_output=True)
+    r = subprocess.run(["git", "diff", "--cached", "--numstat", "HEAD"], cwd=ws,
+                       capture_output=True, text=True)
+    files = added = removed = 0
+    for line in r.stdout.splitlines():
+        a, d, _ = line.split("\t", 2)
+        files += 1
+        added += int(a) if a.isdigit() else 0
+        removed += int(d) if d.isdigit() else 0
+    return {"files_changed": files, "lines_added": added, "lines_removed": removed}
+
+
+def check(task_dir, ws, runner):
+    hidden = ws / ".bench_hidden"
+    if hidden.exists():
+        shutil.rmtree(hidden)
+    shutil.copytree(task_dir / "hidden", hidden)
+    try:
+        if runner == "python":
+            r = subprocess.run([sys.executable, str(ROOT / "bench" / "pycheck.py"), str(hidden)],
+                               cwd=ws, capture_output=True, text=True, timeout=CHECK_TIMEOUT_S)
+            last = (r.stdout.strip().splitlines() or ["{}"])[-1]
+            res = json.loads(last)
+            return res.get("passed", 0), res.get("total", 0), r.stdout[-4000:] + r.stderr[-4000:]
+        if runner == "node":
+            files = sorted(str(p) for p in hidden.glob("*.test.js"))
+            r = subprocess.run(["node", "--test", "--test-reporter=tap", *files],
+                               cwd=ws, capture_output=True, text=True, timeout=CHECK_TIMEOUT_S)
+            counts = {}
+            for line in r.stdout.splitlines():
+                parts = line.split()
+                if len(parts) == 3 and parts[0] == "#" and parts[1] in ("pass", "fail", "cancelled") and parts[2].isdigit():
+                    counts[parts[1]] = int(parts[2])
+            passed = counts.get("pass", 0)
+            return passed, passed + counts.get("fail", 0) + counts.get("cancelled", 0), r.stdout[-4000:] + r.stderr[-4000:]
+    except subprocess.TimeoutExpired:
+        return 0, 0, "check timed out"
+    except (json.JSONDecodeError, OSError) as e:
+        return 0, 0, f"check crashed: {e}"
+    finally:
+        shutil.rmtree(hidden, ignore_errors=True)
+    raise ValueError(f"unknown runner {runner}")
+
+
+def main():
+    ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    ap.add_argument("--arm", required=True, help="directory name under arms/ (stock or stack)")
+    ap.add_argument("--wave", type=int, default=1)
+    ap.add_argument("--model", default=os.environ.get("BENCH_MODEL", ""), help="provider/model[#variant]; same for both arms")
+    ap.add_argument("--agent", default="", help="agent to pass with --agent (only use for the stack arm)")
+    ap.add_argument("--tasks", default="all", help="comma separated task ids, or 'all'")
+    ap.add_argument("--timeout", type=int, default=0, help="per-task timeout in seconds (default: task.json)")
+    ap.add_argument("--out", default="results")
+    ap.add_argument("--shared-config", default=str(ROOT / "shared" / "opencode.json"),
+                    help="config file applied to BOTH arms via OPENCODE_CONFIG (providers etc.)")
+    ap.add_argument("--opencode-bin", default=os.environ.get("OPENCODE_BIN", "opencode"))
+    ap.add_argument("--mode", choices=["opencode", "solution", "noop"], default="opencode",
+                    help="solution/noop skip the agent and are used to self-test the task suite")
+    ap.add_argument("--keep-workspaces", action="store_true")
+    args = ap.parse_args()
+
+    arm_src = ROOT / "arms" / args.arm
+    if not arm_src.is_dir():
+        sys.exit(f"no such arm: {arm_src}")
+    out = Path(args.out).resolve()
+    (out / "logs").mkdir(parents=True, exist_ok=True)
+    results_path = out / f"{args.arm}-w{args.wave}.jsonl"
+
+    tasks = load_tasks(args.tasks)
+    # Same shuffled order for both arms within a wave, different order across waves.
+    random.Random(args.wave).shuffle(tasks)
+
+    scratch_root = Path(tempfile.mkdtemp(prefix=f"bench-{args.arm}-w{args.wave}-"))
+    cache_dir = scratch_root / "cache"  # shared within this job so plugin/npm installs are paid once
+    # Copy the arm so plugin installs or caches written next to the config never touch the repo.
+    arm_dir = scratch_root / "arm"
+    shutil.copytree(arm_src, arm_dir, ignore=shutil.ignore_patterns("README.md"))
+
+    print(f"arm={args.arm} wave={args.wave} model={args.model or '(config default)'} tasks={len(tasks)}", flush=True)
+    with open(results_path, "a") as results:
+        for task_dir in tasks:
+            meta = json.loads((task_dir / "task.json").read_text())
+            scratch = scratch_root / task_dir.name
+            ws = scratch / "ws"
+            make_workspace(task_dir, ws)
+            log_base = out / "logs" / f"{args.arm}-w{args.wave}-{task_dir.name}"
+            rec = {"task": task_dir.name, "arm": args.arm, "wave": args.wave, "model": args.model,
+                   "agent": args.agent, "mode": args.mode}
+            if args.mode == "opencode":
+                env = isolated_env(scratch, arm_dir, Path(args.shared_config), cache_dir)
+                timeout_s = args.timeout or meta.get("timeout_s", 600)
+                code, timed_out, duration, events_path, stderr_path = run_opencode(
+                    args, task_dir, ws, env, log_base, timeout_s)
+                usage = summarize_events(events_path)
+                rec.update(exit_code=code, timed_out=timed_out, duration_s=round(duration, 2), **usage)
+                # No model step at all means the run never really started (bad key, bad model id, crash).
+                rec["infra_error"] = usage["steps"] == 0 and usage["tool_calls"] == 0
+                if rec["infra_error"]:
+                    rec["stderr_tail"] = stderr_path.read_text(errors="replace")[-1500:]
+            else:
+                if args.mode == "solution":
+                    shutil.copytree(task_dir / "solution", ws, dirs_exist_ok=True)
+                rec.update(exit_code=0, timed_out=False, duration_s=0.0, infra_error=False)
+            rec.update(diff_stats(ws))
+            passed, total, check_log = check(task_dir, ws, meta["runner"])
+            expected = meta.get("tests") or total
+            total = max(total, expected)
+            rec.update(passed=passed, total=total, score=round(passed / total, 4) if total else 0.0,
+                       solved=bool(total) and passed == total)
+            log_base.with_suffix(".check.log").write_text(check_log)
+            results.write(json.dumps(rec) + "\n")
+            results.flush()
+            flag = " INFRA-ERROR" if rec["infra_error"] else (" TIMEOUT" if rec["timed_out"] else "")
+            print(f"  {task_dir.name:<22} {passed}/{total}  {rec['duration_s']:>7.1f}s"
+                  f"  tok_in={rec.get('tokens_in', 0)} tok_out={rec.get('tokens_out', 0)}{flag}", flush=True)
+            if rec.get("first_error"):
+                print(f"    error: {rec['first_error']}", flush=True)
+            if rec["infra_error"] and rec["stderr_tail"].strip():
+                print("    " + rec["stderr_tail"].strip().replace("\n", "\n    ")[-800:], flush=True)
+
+    if not args.keep_workspaces:
+        shutil.rmtree(scratch_root, ignore_errors=True)
+    print(f"results -> {results_path}")
+
+
+if __name__ == "__main__":
+    main()

bench/selftest.sh

@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+# Validate the task suite without a model: solutions must score 1.0, untouched fixtures must not.
+set -euo pipefail
+cd "$(dirname "$0")/.."
+out="$(mktemp -d)"
+python3 bench/run_arm.py --arm stock --mode solution --out "$out/solution"
+python3 bench/run_arm.py --arm stock --mode noop --out "$out/noop"
+python3 - "$out" <<'PY'
+import json, sys
+from pathlib import Path
+out = Path(sys.argv[1])
+bad = []
+for line in (out / "solution" / "stock-w1.jsonl").read_text().splitlines():
+    r = json.loads(line)
+    tests = json.loads(Path("tasks", r["task"], "task.json").read_text()).get("tests")
+    if not r["solved"]:
+        bad.append(f"{r['task']}: reference solution scores {r['passed']}/{r['total']}")
+    if tests != r["total"]:
+        bad.append(f"{r['task']}: task.json says tests={tests} but the hidden suite has {r['total']}")
+for line in (out / "noop" / "stock-w1.jsonl").read_text().splitlines():
+    r = json.loads(line)
+    if r["solved"]:
+        bad.append(f"{r['task']}: untouched fixture already passes all hidden tests")
+print("\n".join(bad) or "task suite OK")
+sys.exit(1 if bad else 0)
+PY

shared/opencode.json

@@ -0,0 +1,3 @@
+{
+  "$schema": "https://opencode.ai/config.json"
+}

tasks/js-lru-cache/hidden/lru.hidden.test.js

@@ -0,0 +1,71 @@
+'use strict';
+const test = require('node:test');
+const assert = require('node:assert');
+const path = require('node:path');
+const { LRUCache } = require(path.join(process.cwd(), 'lru.js'));
+
+test('basic get/set', () => {
+  const c = new LRUCache({ max: 2 });
+  assert.strictEqual(c.set('a', 1), c);
+  assert.strictEqual(c.get('a'), 1);
+  assert.strictEqual(c.get('zz'), undefined);
+});
+
+test('evicts least recently used', () => {
+  const c = new LRUCache({ max: 2 });
+  c.set('a', 1); c.set('b', 2); c.get('a'); c.set('c', 3);
+  assert.strictEqual(c.has('b'), false);
+  assert.deepStrictEqual(c.keys(), ['c', 'a']);
+});
+
+test('has does not change recency', () => {
+  const c = new LRUCache({ max: 2 });
+  c.set('a', 1); c.set('b', 2); c.has('a'); c.set('c', 3);
+  assert.strictEqual(c.has('a'), false);
+});
+
+test('invalid max throws RangeError', () => {
+  assert.throws(() => new LRUCache({ max: 0 }), RangeError);
+  assert.throws(() => new LRUCache({ max: 1.5 }), RangeError);
+});
+
+test('ttl expiry with injected clock', () => {
+  let t = 1000;
+  const c = new LRUCache({ max: 5, ttlMs: 100, now: () => t });
+  c.set('a', 1); t += 50; c.set('b', 2); t += 60;
+  assert.strictEqual(c.get('a'), undefined);
+  assert.strictEqual(c.get('b'), 2);
+  assert.strictEqual(c.size, 1);
+  assert.deepStrictEqual(c.keys(), ['b']);
+});
+
+test('update resets ttl', () => {
+  let t = 0;
+  const c = new LRUCache({ max: 5, ttlMs: 100, now: () => t });
+  c.set('a', 1); t = 90; c.set('a', 2); t = 150;
+  assert.strictEqual(c.get('a'), 2);
+});
+
+test('delete and clear', () => {
+  const c = new LRUCache({ max: 3 });
+  c.set('a', 1); c.set('b', 2);
+  assert.strictEqual(c.delete('a'), true);
+  assert.strictEqual(c.delete('a'), false);
+  c.clear();
+  assert.strictEqual(c.size, 0);
+});
+
+test('onEvict only on LRU eviction', () => {
+  const evicted = [];
+  const c = new LRUCache({ max: 1, onEvict: (k, v) => evicted.push([k, v]) });
+  c.set('a', 1); c.set('b', 2); c.delete('b'); c.set('c', 3); c.clear();
+  assert.deepStrictEqual(evicted, [['a', 1]]);
+});
+
+test('non-string keys', () => {
+  const c = new LRUCache({ max: 2 });
+  const k = {};
+  c.set(k, 'obj'); c.set(1, 'num');
+  assert.strictEqual(c.get(k), 'obj');
+  assert.strictEqual(c.get('1'), undefined);
+});

tasks/js-lru-cache/prompt.md

@@ -0,0 +1,14 @@
+Implement the `LRUCache` class exported from `lru.js` (CommonJS). Requirements:
+
+- `new LRUCache({ max, ttlMs, now })`: `max` is a positive integer (throw `RangeError` otherwise).
+  `ttlMs` is optional (no expiry when omitted). `now` is an optional clock function returning
+  milliseconds, defaulting to `Date.now`.
+- `get(key)` returns the value or `undefined`, and marks the key as most recently used.
+  Expired entries are treated as missing and removed.
+- `set(key, value)` inserts or updates and marks as most recently used, evicting the least recently
+  used entry when over `max`. Returns `this`. Updating an entry resets its TTL.
+- `has(key)` (does NOT change recency), `delete(key)` (returns boolean), `clear()`.
+- `size` getter (count of non-expired entries).
+- `keys()` returns an array of keys from most to least recently used, excluding expired entries.
+- Optional `onEvict(key, value)` callback in the constructor options, called on LRU eviction
+  (not on delete, clear, or expiry).

tasks/js-lru-cache/repo/lru.js

@@ -0,0 +1,9 @@
+'use strict';
+
+class LRUCache {
+  constructor(options) {
+    throw new Error('not implemented');
+  }
+}
+
+module.exports = { LRUCache };

tasks/js-lru-cache/repo/package.json

@@ -0,0 +1 @@
+{ "name": "lru-task", "version": "1.0.0", "private": true, "scripts": { "test": "node --test" } }

tasks/js-lru-cache/solution/lru.js

@@ -0,0 +1,58 @@
+'use strict';
+
+class LRUCache {
+  constructor({ max, ttlMs, now = Date.now, onEvict } = {}) {
+    if (!Number.isInteger(max) || max < 1) throw new RangeError('max must be a positive integer');
+    this.max = max;
+    this.ttlMs = ttlMs;
+    this.now = now;
+    this.onEvict = onEvict;
+    this.map = new Map();
+  }
+
+  _expired(entry) {
+    return this.ttlMs !== undefined && this.now() - entry.at >= this.ttlMs;
+  }
+
+  _prune() {
+    for (const [k, e] of this.map) if (this._expired(e)) this.map.delete(k);
+  }
+
+  get(key) {
+    const e = this.map.get(key);
+    if (!e) return undefined;
+    if (this._expired(e)) { this.map.delete(key); return undefined; }
+    this.map.delete(key);
+    this.map.set(key, e);
+    return e.value;
+  }
+
+  set(key, value) {
+    this.map.delete(key);
+    this.map.set(key, { value, at: this.now() });
+    if (this.map.size > this.max) this._prune();
+    while (this.map.size > this.max) {
+      const [k, e] = this.map.entries().next().value;
+      this.map.delete(k);
+      if (this.onEvict) this.onEvict(k, e.value);
+    }
+    return this;
+  }
+
+  has(key) {
+    const e = this.map.get(key);
+    if (!e) return false;
+    if (this._expired(e)) { this.map.delete(key); return false; }
+    return true;
+  }
+
+  delete(key) { return this.map.delete(key); }
+
+  clear() { this.map.clear(); }
+
+  get size() { this._prune(); return this.map.size; }
+
+  keys() { this._prune(); return [...this.map.keys()].reverse(); }
+}
+
+module.exports = { LRUCache };

tasks/js-lru-cache/task.json

@@ -0,0 +1,6 @@
+{
+  "title": "Implement an LRU cache with TTL",
+  "runner": "node",
+  "timeout_s": 600,
+  "tests": 9
+}

tasks/js-semver-sort/hidden/semver.hidden.test.js

@@ -0,0 +1,62 @@
+'use strict';
+const test = require('node:test');
+const assert = require('node:assert');
+const path = require('node:path');
+const sv = require(path.join(process.cwd(), 'semver.js'));
+
+test('parse basic and v prefix', () => {
+  assert.deepStrictEqual(sv.parse('v1.2.3-alpha.1+build.5'), {
+    major: 1, minor: 2, patch: 3, prerelease: ['alpha', 1], build: ['build', '5'],
+  });
+});
+
+test('parse rejects invalid', () => {
+  for (const bad of ['1.2', '01.2.3', '1.2.3-01', 'a.b.c', '1.2.3-', '1.2.3+']) {
+    assert.strictEqual(sv.parse(bad), null, bad);
+  }
+});
+
+test('compare precedence chain from spec', () => {
+  const chain = ['1.0.0-alpha', '1.0.0-alpha.1', '1.0.0-alpha.beta', '1.0.0-beta',
+    '1.0.0-beta.2', '1.0.0-beta.11', '1.0.0-rc.1', '1.0.0'];
+  for (let i = 0; i < chain.length - 1; i++) {
+    assert.strictEqual(sv.compare(chain[i], chain[i + 1]), -1, `${chain[i]} < ${chain[i + 1]}`);
+    assert.strictEqual(sv.compare(chain[i + 1], chain[i]), 1);
+  }
+});
+
+test('compare ignores build and throws on invalid', () => {
+  assert.strictEqual(sv.compare('1.0.0+a', '1.0.0+b'), 0);
+  assert.throws(() => sv.compare('x', '1.0.0'), TypeError);
+});
+
+test('sort is non-mutating and drops invalid', () => {
+  const input = ['1.10.0', 'bad', '1.2.0', '1.2.0-rc.1'];
+  const out = sv.sort(input);
+  assert.deepStrictEqual(out, ['1.2.0-rc.1', '1.2.0', '1.10.0']);
+  assert.strictEqual(input[0], '1.10.0');
+});
+
+test('satisfies comparators and ||', () => {
+  assert.ok(sv.satisfies('1.5.0', '>=1.2.0 <2.0.0'));
+  assert.ok(!sv.satisfies('2.0.0', '>=1.2.0 <2.0.0'));
+  assert.ok(sv.satisfies('3.0.0', '^1.0.0 || >=3.0.0'));
+  assert.ok(sv.satisfies('1.2.3', '1.2.3'));
+  assert.ok(sv.satisfies('9.9.9', '*'));
+});
+
+test('caret and tilde', () => {
+  assert.ok(sv.satisfies('1.9.9', '^1.2.3'));
+  assert.ok(!sv.satisfies('2.0.0', '^1.2.3'));
+  assert.ok(sv.satisfies('0.2.9', '^0.2.3'));
+  assert.ok(!sv.satisfies('0.3.0', '^0.2.3'));
+  assert.ok(!sv.satisfies('0.0.4', '^0.0.3'));
+  assert.ok(sv.satisfies('1.2.9', '~1.2.3'));
+  assert.ok(!sv.satisfies('1.3.0', '~1.2.3'));
+});
+
+test('prerelease rules', () => {
+  assert.ok(!sv.satisfies('1.3.0-beta', '^1.2.0'));
+  assert.ok(sv.satisfies('1.2.3-beta.2', '>=1.2.3-beta.1 <2.0.0'));
+  assert.ok(!sv.satisfies('1.2.4-beta.2', '>=1.2.3-beta.1 <2.0.0'));
+});

tasks/js-semver-sort/prompt.md

@@ -0,0 +1,14 @@
+Implement `semver.js` (CommonJS, no dependencies) exporting:
+
+- `parse(v)`: parses a SemVer 2.0.0 string (an optional leading `v` is allowed) into
+  `{ major, minor, patch, prerelease: [...], build: [...] }` where numeric prerelease identifiers are
+  numbers. Returns `null` for invalid input (including leading zeros in numeric parts).
+- `compare(a, b)`: returns -1, 0 or 1 following SemVer 2.0.0 precedence (build metadata ignored).
+  Throws `TypeError` if either input is invalid.
+- `sort(list)`: returns a new array sorted ascending by precedence; invalid entries are dropped.
+- `satisfies(v, range)`: supports ranges made of space-separated comparators that must all match
+  (`>=1.2.0 <2.0.0`), `||` alternatives, exact versions, `^` and `~` with the usual npm meaning
+  (including `^0.x` behaviour), and `*`. Prerelease versions only satisfy a range if a comparator in
+  the same set has a prerelease on the same `major.minor.patch`, as in npm.
+
+There is an existing partial implementation; fix and complete it.

tasks/js-semver-sort/repo/package.json

@@ -0,0 +1 @@
+{ "name": "semver-task", "version": "1.0.0", "private": true, "scripts": { "test": "node --test" } }

tasks/js-semver-sort/repo/semver.js

@@ -0,0 +1,29 @@
+'use strict';
+
+function parse(v) {
+  const m = /^v?(\d+)\.(\d+)\.(\d+)(?:-([0-9A-Za-z.-]+))?$/.exec(v);
+  if (!m) return null;
+  return {
+    major: +m[1], minor: +m[2], patch: +m[3],
+    prerelease: m[4] ? m[4].split('.') : [],
+    build: [],
+  };
+}
+
+function compare(a, b) {
+  const x = parse(a), y = parse(b);
+  for (const k of ['major', 'minor', 'patch']) {
+    if (x[k] !== y[k]) return x[k] < y[k] ? -1 : 1;
+  }
+  return 0;
+}
+
+function sort(list) {
+  return list.sort(compare);
+}
+
+function satisfies(v, range) {
+  throw new Error('not implemented');
+}
+
+module.exports = { parse, compare, sort, satisfies };

tasks/js-semver-sort/solution/semver.js

@@ -0,0 +1,89 @@
+'use strict';
+
+const NUM = '(0|[1-9]\\d*)';
+const PRE_ID = '(?:0|[1-9]\\d*|\\d*[A-Za-z-][0-9A-Za-z-]*)';
+const RE = new RegExp(`^v?${NUM}\\.${NUM}\\.${NUM}(?:-(${PRE_ID}(?:\\.${PRE_ID})*))?(?:\\+([0-9A-Za-z-]+(?:\\.[0-9A-Za-z-]+)*))?$`);
+
+function parse(v) {
+  if (typeof v !== 'string') return null;
+  const m = RE.exec(v.trim());
+  if (!m) return null;
+  return {
+    major: +m[1], minor: +m[2], patch: +m[3],
+    prerelease: m[4] ? m[4].split('.').map((p) => (/^\d+$/.test(p) ? +p : p)) : [],
+    build: m[5] ? m[5].split('.') : [],
+  };
+}
+
+function cmpParsed(x, y) {
+  for (const k of ['major', 'minor', 'patch']) if (x[k] !== y[k]) return x[k] < y[k] ? -1 : 1;
+  const a = x.prerelease, b = y.prerelease;
+  if (!a.length && !b.length) return 0;
+  if (!a.length) return 1;
+  if (!b.length) return -1;
+  for (let i = 0; i < Math.max(a.length, b.length); i++) {
+    if (a[i] === undefined) return -1;
+    if (b[i] === undefined) return 1;
+    if (a[i] === b[i]) continue;
+    const an = typeof a[i] === 'number', bn = typeof b[i] === 'number';
+    if (an && bn) return a[i] < b[i] ? -1 : 1;
+    if (an) return -1;
+    if (bn) return 1;
+    return a[i] < b[i] ? -1 : 1;
+  }
+  return 0;
+}
+
+function compare(a, b) {
+  const x = parse(a), y = parse(b);
+  if (!x || !y) throw new TypeError('invalid version');
+  return cmpParsed(x, y);
+}
+
+function sort(list) {
+  return list.filter((v) => parse(v)).sort(compare);
+}
+
+function expand(token) {
+  if (token === '*' || token === '') return [];
+  const m = /^(\^|~|>=|<=|>|<|=)?(.*)$/.exec(token);
+  const op = m[1] || '=';
+  const v = parse(m[2]);
+  if (!v) throw new TypeError(`bad comparator ${token}`);
+  const s = (a, b, c) => ({ major: a, minor: b, patch: c, prerelease: [], build: [] });
+  if (op === '^') {
+    let upper;
+    if (v.major > 0) upper = s(v.major + 1, 0, 0);
+    else if (v.minor > 0) upper = s(0, v.minor + 1, 0);
+    else upper = s(0, 0, v.patch + 1);
+    return [{ op: '>=', v }, { op: '<', v: { ...upper, prerelease: [0] }, synthetic: true }];
+  }
+  if (op === '~') return [{ op: '>=', v }, { op: '<', v: { ...s(v.major, v.minor + 1, 0), prerelease: [0] }, synthetic: true }];
+  return [{ op, v }];
+}
+
+function test(c, v) {
+  const r = cmpParsed(v, c.v);
+  switch (c.op) {
+    case '=': return r === 0;
+    case '>': return r > 0;
+    case '>=': return r >= 0;
+    case '<': return r < 0;
+    case '<=': return r <= 0;
+  }
+  return false;
+}
+
+function satisfies(version, range) {
+  const v = parse(version);
+  if (!v) return false;
+  return range.split('||').some((set) => {
+    const comps = set.trim().split(/\s+/).flatMap(expand);
+    if (!comps.every((c) => test(c, v))) return false;
+    if (!v.prerelease.length) return true;
+    return comps.some((c) => !c.synthetic && c.v.prerelease.length &&
+      c.v.major === v.major && c.v.minor === v.minor && c.v.patch === v.patch);
+  });
+}
+
+module.exports = { parse, compare, sort, satisfies };

tasks/js-semver-sort/task.json

@@ -0,0 +1,6 @@
+{
+  "title": "Implement SemVer comparison and range matching",
+  "runner": "node",
+  "timeout_s": 600,
+  "tests": 8
+}

tasks/py-business-days/hidden/test_hidden_business_days.py

@@ -0,0 +1,38 @@
+import unittest
+from datetime import date
+
+from calendar_utils import business_days
+
+
+class HiddenBusinessDays(unittest.TestCase):
+    def test_full_week(self):
+        self.assertEqual(business_days(date(2024, 1, 1), date(2024, 1, 7)), 5)
+
+    def test_single_day_weekday(self):
+        self.assertEqual(business_days(date(2024, 1, 3), date(2024, 1, 3)), 1)
+
+    def test_single_day_weekend(self):
+        self.assertEqual(business_days(date(2024, 1, 6), date(2024, 1, 6)), 0)
+
+    def test_reversed_range(self):
+        self.assertEqual(business_days(date(2024, 1, 10), date(2024, 1, 1)), 0)
+
+    def test_holiday_on_weekday(self):
+        self.assertEqual(business_days(date(2024, 1, 1), date(2024, 1, 5), [date(2024, 1, 1)]), 4)
+
+    def test_holiday_on_weekend_ignored(self):
+        self.assertEqual(business_days(date(2024, 1, 1), date(2024, 1, 7), [date(2024, 1, 6)]), 5)
+
+    def test_holiday_outside_range_ignored(self):
+        self.assertEqual(business_days(date(2024, 1, 1), date(2024, 1, 5), [date(2024, 2, 1)]), 5)
+
+    def test_duplicate_holidays(self):
+        h = [date(2024, 1, 2), date(2024, 1, 2)]
+        self.assertEqual(business_days(date(2024, 1, 1), date(2024, 1, 5), h), 4)
+
+    def test_month(self):
+        self.assertEqual(business_days(date(2024, 2, 1), date(2024, 2, 29)), 21)
+
+
+if __name__ == "__main__":
+    unittest.main()

tasks/py-business-days/prompt.md

@@ -0,0 +1,4 @@
+The `business_days(start, end, holidays=None)` function in `calendar_utils.py` gives wrong answers.
+It should return the number of weekdays (Mon-Fri) in the inclusive range [start, end] that are
+not in `holidays`. If `start > end` it should return 0. Holidays that fall on weekends must not be
+subtracted twice. Fix the function. Keep the public signature unchanged.

tasks/py-business-days/repo/calendar_utils.py

@@ -0,0 +1,13 @@
+from datetime import date, timedelta
+
+
+def business_days(start: date, end: date, holidays=None) -> int:
+    """Count weekdays in [start, end] that are not holidays."""
+    holidays = holidays or []
+    days = (end - start).days
+    count = 0
+    for i in range(days):
+        d = start + timedelta(days=i)
+        if d.weekday() < 6:
+            count += 1
+    return count - len(holidays)

tasks/py-business-days/repo/test_calendar_utils.py

@@ -0,0 +1,13 @@
+import unittest
+from datetime import date
+
+from calendar_utils import business_days
+
+
+class TestBusinessDays(unittest.TestCase):
+    def test_single_week(self):
+        self.assertEqual(business_days(date(2024, 1, 1), date(2024, 1, 5)), 5)
+
+
+if __name__ == "__main__":
+    unittest.main()

tasks/py-business-days/solution/calendar_utils.py

@@ -0,0 +1,15 @@
+from datetime import date, timedelta
+
+
+def business_days(start: date, end: date, holidays=None) -> int:
+    """Count weekdays in [start, end] that are not holidays."""
+    if start > end:
+        return 0
+    skip = set(holidays or [])
+    count = 0
+    d = start
+    while d <= end:
+        if d.weekday() < 5 and d not in skip:
+            count += 1
+        d += timedelta(days=1)
+    return count

tasks/py-business-days/task.json

@@ -0,0 +1,6 @@
+{
+  "title": "Fix business-day counting bugs",
+  "runner": "python",
+  "timeout_s": 600,
+  "tests": 9
+}

tasks/py-inventory/hidden/test_hidden_inventory.py

@@ -0,0 +1,59 @@
+import unittest
+from decimal import Decimal
+
+from inventory import Inventory, OutOfStock, order_total
+
+
+def make():
+    inv = Inventory()
+    inv.add_item("A", Decimal("10.00"), 5)
+    inv.add_item("B", Decimal("2.50"), 10)
+    return inv
+
+
+class HiddenInventory(unittest.TestCase):
+    def test_discount_applied_once(self):
+        self.assertEqual(order_total([(Decimal("10.00"), 2)], "SAVE10"), Decimal("19.44"))
+
+    def test_half_discount_multi_line(self):
+        self.assertEqual(order_total([(Decimal("10.00"), 1), (Decimal("2.50"), 4)], "HALF"), Decimal("10.80"))
+
+    def test_unknown_code(self):
+        with self.assertRaises(ValueError):
+            order_total([(Decimal("1"), 1)], "NOPE")
+
+    def test_order_exact_stock(self):
+        inv = make()
+        inv.order({"A": 5})
+        self.assertEqual(inv.stock("A"), 0)
+
+    def test_out_of_stock_all_or_nothing(self):
+        inv = make()
+        with self.assertRaises(OutOfStock):
+            inv.order({"B": 3, "A": 6})
+        self.assertEqual(inv.stock("B"), 10)
+        self.assertEqual(inv.history, [])
+
+    def test_order_total_with_code(self):
+        inv = make()
+        self.assertEqual(inv.order({"A": 2}, "SAVE10"), Decimal("19.44"))
+        self.assertEqual(inv.history, [("order", {"A": 2}, "SAVE10")])
+
+    def test_restock(self):
+        inv = make()
+        inv.restock("A", 3)
+        self.assertEqual(inv.stock("A"), 8)
+        self.assertEqual(inv.history[-1], ("restock", "A", 3))
+
+    def test_restock_errors(self):
+        inv = make()
+        with self.assertRaises(KeyError):
+            inv.restock("Z", 1)
+        for bad in (0, -1, 1.5, "2"):
+            with self.subTest(bad=bad):
+                with self.assertRaises(ValueError):
+                    inv.restock("A", bad)
+
+
+if __name__ == "__main__":
+    unittest.main()

tasks/py-inventory/prompt.md

@@ -0,0 +1,8 @@
+Customers report that order totals in the `inventory` package are too low when a discount code is used,
+and that ordering exactly the remaining stock fails. Find and fix the bugs.
+
+Then add a `restock(sku, qty)` method to `Inventory` that increases stock for an existing SKU
+(raise `KeyError` for an unknown SKU, `ValueError` if `qty` is not a positive integer) and records a
+`("restock", sku, qty)` entry in `Inventory.history`, the same way orders are recorded.
+
+Keep the existing public API working. There are some existing tests in `tests/`.

tasks/py-inventory/repo/inventory/__init__.py

@@ -0,0 +1,4 @@
+from .core import Inventory, OutOfStock
+from .pricing import order_total
+
+__all__ = ["Inventory", "OutOfStock", "order_total"]

tasks/py-inventory/repo/inventory/core.py

@@ -0,0 +1,33 @@
+from decimal import Decimal
+
+from .pricing import order_total
+
+
+class OutOfStock(Exception):
+    pass
+
+
+class Inventory:
+    def __init__(self):
+        self._items = {}
+        self.history = []
+
+    def add_item(self, sku: str, price: Decimal, stock: int):
+        if sku in self._items:
+            raise ValueError(f"duplicate sku: {sku}")
+        self._items[sku] = {"price": Decimal(price), "stock": int(stock)}
+
+    def stock(self, sku: str) -> int:
+        return self._items[sku]["stock"]
+
+    def order(self, items, code=None) -> Decimal:
+        """items: dict sku -> qty. All-or-nothing: no stock changes if any line fails."""
+        for sku, qty in items.items():
+            if self._items[sku]["stock"] <= qty:
+                raise OutOfStock(sku)
+        lines = []
+        for sku, qty in items.items():
+            self._items[sku]["stock"] -= qty
+            lines.append((self._items[sku]["price"], qty))
+        self.history.append(("order", dict(items), code))
+        return order_total(lines, code)

tasks/py-inventory/repo/inventory/pricing.py

@@ -0,0 +1,21 @@
+from decimal import Decimal, ROUND_HALF_UP
+
+DISCOUNTS = {"SAVE10": Decimal("0.10"), "HALF": Decimal("0.50")}
+TAX_RATE = Decimal("0.08")
+
+
+def apply_discount(amount: Decimal, code=None) -> Decimal:
+    if not code:
+        return amount
+    rate = DISCOUNTS.get(code)
+    if rate is None:
+        raise ValueError(f"unknown discount code: {code}")
+    return amount * (1 - rate)
+
+
+def order_total(lines, code=None) -> Decimal:
+    """lines: iterable of (unit_price: Decimal, qty: int). Discount applies before tax."""
+    subtotal = sum((apply_discount(price, code) * qty for price, qty in lines), Decimal("0"))
+    discounted = apply_discount(subtotal, code)
+    total = discounted * (1 + TAX_RATE)
+    return total.quantize(Decimal("0.01"), rounding=ROUND_HALF_UP)

tasks/py-inventory/repo/tests/test_basic.py

@@ -0,0 +1,16 @@
+import unittest
+from decimal import Decimal
+
+from inventory import Inventory
+
+
+class TestBasic(unittest.TestCase):
+    def test_order_no_discount(self):
+        inv = Inventory()
+        inv.add_item("A", Decimal("10.00"), 5)
+        self.assertEqual(inv.order({"A": 2}), Decimal("21.60"))
+        self.assertEqual(inv.stock("A"), 3)
+
+
+if __name__ == "__main__":
+    unittest.main()

tasks/py-inventory/solution/inventory/__init__.py

@@ -0,0 +1,4 @@
+from .core import Inventory, OutOfStock
+from .pricing import order_total
+
+__all__ = ["Inventory", "OutOfStock", "order_total"]

tasks/py-inventory/solution/inventory/core.py

@@ -0,0 +1,42 @@
+from decimal import Decimal
+
+from .pricing import order_total
+
+
+class OutOfStock(Exception):
+    pass
+
+
+class Inventory:
+    def __init__(self):
+        self._items = {}
+        self.history = []
+
+    def add_item(self, sku: str, price: Decimal, stock: int):
+        if sku in self._items:
+            raise ValueError(f"duplicate sku: {sku}")
+        self._items[sku] = {"price": Decimal(price), "stock": int(stock)}
+
+    def stock(self, sku: str) -> int:
+        return self._items[sku]["stock"]
+
+    def order(self, items, code=None) -> Decimal:
+        """items: dict sku -> qty. All-or-nothing: no stock changes if any line fails."""
+        for sku, qty in items.items():
+            if self._items[sku]["stock"] < qty:
+                raise OutOfStock(sku)
+        lines = []
+        for sku, qty in items.items():
+            self._items[sku]["stock"] -= qty
+            lines.append((self._items[sku]["price"], qty))
+        total = order_total(lines, code)
+        self.history.append(("order", dict(items), code))
+        return total
+
+    def restock(self, sku: str, qty: int):
+        if sku not in self._items:
+            raise KeyError(sku)
+        if isinstance(qty, bool) or not isinstance(qty, int) or qty < 1:
+            raise ValueError(f"qty must be a positive integer: {qty!r}")
+        self._items[sku]["stock"] += qty
+        self.history.append(("restock", sku, qty))

tasks/py-inventory/solution/inventory/pricing.py

@@ -0,0 +1,21 @@
+from decimal import Decimal, ROUND_HALF_UP
+
+DISCOUNTS = {"SAVE10": Decimal("0.10"), "HALF": Decimal("0.50")}
+TAX_RATE = Decimal("0.08")
+
+
+def apply_discount(amount: Decimal, code=None) -> Decimal:
+    if not code:
+        return amount
+    rate = DISCOUNTS.get(code)
+    if rate is None:
+        raise ValueError(f"unknown discount code: {code}")
+    return amount * (1 - rate)
+
+
+def order_total(lines, code=None) -> Decimal:
+    """lines: iterable of (unit_price: Decimal, qty: int). Discount applies before tax."""
+    subtotal = sum((price * qty for price, qty in lines), Decimal("0"))
+    discounted = apply_discount(subtotal, code)
+    total = discounted * (1 + TAX_RATE)
+    return total.quantize(Decimal("0.01"), rounding=ROUND_HALF_UP)

tasks/py-inventory/task.json

@@ -0,0 +1,6 @@
+{
+  "title": "Fix pricing bug and add restock in a multi-module package",
+  "runner": "python",
+  "timeout_s": 600,
+  "tests": 8
+}

tasks/py-parse-duration/hidden/test_hidden_durations.py

@@ -0,0 +1,47 @@
+import unittest
+
+from durations import format_duration, parse_duration
+
+
+class HiddenParse(unittest.TestCase):
+    def test_simple(self):
+        self.assertEqual(parse_duration("1h30m"), 5400)
+        self.assertEqual(parse_duration("2d4h5s"), 2 * 86400 + 4 * 3600 + 5)
+        self.assertEqual(parse_duration("1w"), 604800)
+
+    def test_whitespace(self):
+        self.assertEqual(parse_duration("1h 30m"), 5400)
+
+    def test_decimal(self):
+        self.assertEqual(parse_duration("1.5h"), 5400)
+        self.assertEqual(parse_duration("1m500ms"), 60)
+        self.assertEqual(parse_duration("2500ms"), 2)
+
+    def test_bare_integer(self):
+        self.assertEqual(parse_duration("90"), 90)
+
+    def test_negative(self):
+        self.assertEqual(parse_duration("-1h"), -3600)
+
+    def test_errors(self):
+        for bad in ["", "abc", "1x", "30m1h", "1h1h", "1h30", "h", "1h-30m", "1..5h"]:
+            with self.subTest(bad=bad):
+                with self.assertRaises(ValueError):
+                    parse_duration(bad)
+
+
+class HiddenFormat(unittest.TestCase):
+    def test_format(self):
+        self.assertEqual(format_duration(5400), "1h30m")
+        self.assertEqual(format_duration(0), "0s")
+        self.assertEqual(format_duration(-61), "-1m1s")
+        self.assertEqual(format_duration(90061), "1d1h1m1s")
+
+    def test_roundtrip(self):
+        for n in [1, 59, 3600, 86399, 172800, 123456]:
+            with self.subTest(n=n):
+                self.assertEqual(parse_duration(format_duration(n)), n)
+
+
+if __name__ == "__main__":
+    unittest.main()

tasks/py-parse-duration/prompt.md

@@ -0,0 +1,13 @@
+Implement `parse_duration(s: str) -> int` in `durations.py`. It converts a human duration string into
+whole seconds. Rules:
+
+- Units: `w` (weeks), `d` (days), `h` (hours), `m` (minutes), `s` (seconds), `ms` (milliseconds).
+- Components may be combined in strictly descending unit order, e.g. `1h30m`, `2d4h5s`, `1m500ms`.
+  Whitespace between components is allowed (`1h 30m`). Each unit may appear at most once.
+- Decimal values are allowed (`1.5h` = 5400). The total is rounded down to whole seconds.
+- A bare integer with no unit means seconds (`90` = 90).
+- An optional leading `-` negates the whole duration.
+- Anything else (empty string, unknown unit, wrong order, repeated unit, trailing garbage) raises `ValueError`.
+
+Also implement `format_duration(seconds: int) -> str`, the inverse for whole seconds using `d`, `h`, `m`, `s`
+with zero components omitted (`5400` -> `1h30m`, `0` -> `0s`, `-61` -> `-1m1s`).

tasks/py-parse-duration/repo/durations.py

@@ -0,0 +1,6 @@
+def parse_duration(s: str) -> int:
+    raise NotImplementedError
+
+
+def format_duration(seconds: int) -> str:
+    raise NotImplementedError

tasks/py-parse-duration/solution/durations.py

@@ -0,0 +1,44 @@
+import re
+
+UNITS = [("w", 604800), ("d", 86400), ("h", 3600), ("m", 60), ("s", 1), ("ms", 0.001)]
+COMPONENT = re.compile(r"\s*(\d+(?:\.\d+)?)(ms|w|d|h|m|s)\s*")
+
+
+def parse_duration(s: str) -> int:
+    if not isinstance(s, str):
+        raise ValueError("not a string")
+    body = s.strip()
+    sign = 1
+    if body.startswith("-"):
+        sign, body = -1, body[1:]
+    if re.fullmatch(r"\d+", body):
+        return sign * int(body)
+    if not body:
+        raise ValueError("empty duration")
+    order = [u for u, _ in UNITS]
+    mult = dict(UNITS)
+    pos, last, total = 0, -1, 0.0
+    while pos < len(body):
+        m = COMPONENT.match(body, pos)
+        if not m:
+            raise ValueError(f"bad duration: {s!r}")
+        idx = order.index(m.group(2))
+        if idx <= last:
+            raise ValueError(f"bad unit order: {s!r}")
+        last = idx
+        total += float(m.group(1)) * mult[m.group(2)]
+        pos = m.end()
+    return sign * int(total)
+
+
+def format_duration(seconds: int) -> str:
+    if seconds == 0:
+        return "0s"
+    sign = "-" if seconds < 0 else ""
+    n = abs(seconds)
+    out = []
+    for unit, size in (("d", 86400), ("h", 3600), ("m", 60), ("s", 1)):
+        q, n = divmod(n, size)
+        if q:
+            out.append(f"{q}{unit}")
+    return sign + "".join(out)

tasks/py-parse-duration/task.json

@@ -0,0 +1,6 @@
+{
+  "title": "Implement a duration parser",
+  "runner": "python",
+  "timeout_s": 600,
+  "tests": 8
+}

tasks/py-wordfreq-cli/hidden/test_hidden_wordfreq.py

@@ -0,0 +1,73 @@
+import os
+import subprocess
+import sys
+import tempfile
+import unittest
+
+import wordfreq
+
+HERE = os.getcwd()
+TEXT = "The cat and the hat. The CAT sat; it's 2 cats, don't you see? a a\n"
+
+
+def run_cli(*args, stdin=None):
+    return subprocess.run(
+        [sys.executable, os.path.join(HERE, "wordfreq.py"), *args],
+        input=stdin, capture_output=True, text=True, timeout=30,
+    )
+
+
+class HiddenWordfreq(unittest.TestCase):
+    def setUp(self):
+        fd, self.path = tempfile.mkstemp(suffix=".txt")
+        with os.fdopen(fd, "w") as f:
+            f.write(TEXT)
+
+    def tearDown(self):
+        os.unlink(self.path)
+
+    def test_count_words_default(self):
+        got = wordfreq.count_words(TEXT)
+        self.assertEqual(got[0], ("The", 2))
+        self.assertIn(("2", 1), got)
+        self.assertIn(("it's", 1), got)
+        self.assertIn(("don't", 1), got)
+
+    def test_order_ties_alphabetical(self):
+        got = wordfreq.count_words("b a c a b")
+        self.assertEqual(got, [("a", 2), ("b", 2), ("c", 1)])
+
+    def test_ignore_case(self):
+        got = dict(wordfreq.count_words(TEXT, ignore_case=True))
+        self.assertEqual(got["the"], 3)
+        self.assertEqual(got["cat"], 2)
+        self.assertNotIn("The", got)
+
+    def test_min_length(self):
+        got = dict(wordfreq.count_words(TEXT, min_length=3))
+        self.assertNotIn("a", got)
+        self.assertNotIn("it", got)
+        self.assertIn("cat", got)
+
+    def test_cli_top(self):
+        r = run_cli(self.path, "--top", "2", "--ignore-case")
+        self.assertEqual(r.returncode, 0, r.stderr)
+        self.assertEqual(r.stdout.strip().splitlines(), ["the\t3", "a\t2"])
+
+    def test_cli_bad_top(self):
+        self.assertEqual(run_cli(self.path, "--top", "0").returncode, 2)
+        self.assertEqual(run_cli(self.path, "--top", "x").returncode, 2)
+
+    def test_cli_stdin(self):
+        r = run_cli("-", "--top", "1", stdin="z z y")
+        self.assertEqual(r.returncode, 0, r.stderr)
+        self.assertEqual(r.stdout.strip(), "z\t2")
+
+    def test_cli_default_still_works(self):
+        r = run_cli(self.path)
+        self.assertEqual(r.returncode, 0, r.stderr)
+        self.assertTrue(r.stdout.startswith("The\t2\n"))
+
+
+if __name__ == "__main__":
+    unittest.main()

tasks/py-wordfreq-cli/prompt.md

@@ -0,0 +1,11 @@
+`wordfreq.py` is a small CLI that prints word counts for a text file, one `word<TAB>count` per line,
+sorted by count descending and then alphabetically. Add these features without breaking existing behaviour:
+
+1. `--top N` prints only the first N lines (N must be a positive integer; otherwise exit with status 2).
+2. `--ignore-case` counts words case-insensitively and prints them lowercased.
+3. `--min-length L` ignores words shorter than L characters (default 1).
+4. Reading from stdin when the path argument is `-`.
+
+Also expose a function `count_words(text, ignore_case=False, min_length=1)` that returns a list of
+`(word, count)` tuples in the same order the CLI prints them. A "word" is a maximal run of letters,
+digits, or apostrophes (`[A-Za-z0-9']+`).

tasks/py-wordfreq-cli/repo/wordfreq.py

@@ -0,0 +1,17 @@
+import re
+import sys
+from collections import Counter
+
+
+def main(argv):
+    path = argv[1]
+    with open(path) as f:
+        text = f.read()
+    counts = Counter(re.findall(r"[A-Za-z]+", text))
+    for word, n in sorted(counts.items(), key=lambda kv: -kv[1]):
+        print(f"{word}\t{n}")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv))

tasks/py-wordfreq-cli/solution/wordfreq.py

@@ -0,0 +1,41 @@
+import argparse
+import re
+import sys
+from collections import Counter
+
+WORD = re.compile(r"[A-Za-z0-9']+")
+
+
+def count_words(text, ignore_case=False, min_length=1):
+    words = WORD.findall(text)
+    if ignore_case:
+        words = [w.lower() for w in words]
+    counts = Counter(w for w in words if len(w) >= min_length)
+    return sorted(counts.items(), key=lambda kv: (-kv[1], kv[0]))
+
+
+def positive_int(s):
+    n = int(s)
+    if n < 1:
+        raise argparse.ArgumentTypeError("must be positive")
+    return n
+
+
+def main(argv):
+    p = argparse.ArgumentParser()
+    p.add_argument("path")
+    p.add_argument("--top", type=positive_int)
+    p.add_argument("--ignore-case", action="store_true")
+    p.add_argument("--min-length", type=int, default=1)
+    a = p.parse_args(argv[1:])
+    text = sys.stdin.read() if a.path == "-" else open(a.path).read()
+    rows = count_words(text, a.ignore_case, a.min_length)
+    if a.top:
+        rows = rows[: a.top]
+    for word, n in rows:
+        print(f"{word}\t{n}")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv))

tasks/py-wordfreq-cli/task.json

@@ -0,0 +1,6 @@
+{
+  "title": "Add flags to a word-frequency CLI",
+  "runner": "python",
+  "timeout_s": 600,
+  "tests": 8
+}