Phase 5c: the grader smoke and the real-agent smoke scripts (v0-T262, v0-T263) (#59)

* ygg-smoke-grade.sh: the grader smoke on a real engine, with the public sample suite of smoke-notes (v0-T262) Co-Authored-By: Claude Sonnet 5.5 <[email protected]> * ygg-smoke-real.sh: the real-agent smoke with real caps, a dry run, a wall clock and the cost from the ledger (v0-T263) Co-Authored-By: Claude Sonnet 5.5 <[email protected]> * ygg-smoke-grade.test.sh: a dry run that loads the credentials in the main shell now fails the test Co-Authored-By: Claude Sonnet 5.5 <[email protected]> * ygg-smoke-real.sh: no xtrace in the credential subshells, and a strict --wall Co-Authored-By: Claude Sonnet 5.5 <[email protected]> * ygg-smoke-grade.sh: stop the engine poller on any exit, and a null reward does not pass as below 1 Co-Authored-By: Claude Sonnet 5.5 <[email protected]> * ygg-cost.sh and ygg-smoke-real.sh: odd ledger lines, a missing cap and a failing cost no longer break the run's result Co-Authored-By: Claude Sonnet 5.5 <[email protected]> --------- Co-authored-by: Bruno Guilhermo de Barros Valério <[email protected]> Co-authored-by: Claude Sonnet 5.5 <[email protected]>
2026-11-07 06:15:59 -10:00 · 2026-10-19 12:06:55 +02:00
1 parent d9d9018a15
commit 618c8c553b
@@ -38,6 +38,8 @@ folder.
 | Push, open a pull request or read GitHub as `ouroboric-pr-bot` | `scripts/as-bot.sh <repo> <cmd>`; `--read all` to read. The only way an agent reaches GitHub |
 | Any real-engine run (Podman, devcontainer, the sandbox): the engine, registry config and checks | `scripts/ygg-engine.sh check\|env\|run --folder DIR`. Never set `CONTAINER_HOST` by hand: the root socket is the wrong one |
 | The realms smoke runs (v0-H13, v0-H15) and their evidence | `scripts/ygg-smoke-realms.sh`, `scripts/ygg-smoke-loop.sh`, `scripts/ygg-evidence.sh` |
+| The grader smoke on a real engine (v0-H16), with the public sample suite | `scripts/ygg-smoke-grade.sh [--build]`. Never a hidden suite |
+| The real-agent smoke (v0-H17): costs money; `--dry-run` first; the cost from the ledger | `scripts/ygg-smoke-real.sh --dry-run`, then without it; `scripts/ygg-cost.sh LEDGER`. It never prints the credentials file |
 | Build a plan from reference commits, or execute one task by task | `scripts/plan-*` (`plan-generate.py`, `plan-replay.py`, `plan-states.sh`, `plan-red.sh`, `plan-defences.py`; `plan-blocks.py`, `plan-task-start.sh`, `plan-task-done.sh`, `plan-task-land.sh`) |
 | Check that the repos have the required setup, and that the realms' shared files match | `scripts/as-bot.sh --read all scripts/check-repos.sh`; `--check shared` runs the realms check alone |
 | Scan git history for secrets | `scripts/scan-history.sh` |
@@ -0,0 +1,294 @@
+#!/usr/bin/env bash
+# Tests for ygg-smoke-grade.sh, ygg-smoke-real.sh and ygg-cost.sh.   scripts/tests/ygg-smoke-grade.test.sh
+# A fake `ygg`, a fake `podman` and a fake `devcontainer` stand in for the real tools: no model, no engine, no
+# network, no money. The task folder is a small made-up mimir with a made-up public suite.
+# shellcheck disable=SC2329,SC2016 # check calls has and hasnt by name; the $ in the patterns is literal
+set -uo pipefail
+HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+S="$HERE/.."
+T="$(mktemp -d)"
+trap 'rm -rf -- "$T"' EXIT
+fail=0
+check() {
+  local d="$1"; shift
+  if "$@"; then echo "ok   $d"; else echo "FAIL $d"; fail=1; [ -z "${VERBOSE:-}" ] || sed 's/^/     | /' "$T/out"; fi
+}
+run() { "$@" >"$T/out" 2>&1; rc=$?; }
+has() { grep -q -F -- "$1" "$T/out"; }
+hasnt() { ! grep -q -F -- "$1" "$T/out"; }
+export HOME="$T/home"; mkdir -p "$HOME"
+export YGG_ENGINE_CHECK=0 # the engine environment has its own test (ygg-engine.test.sh)
+export GIT_CONFIG_GLOBAL=/dev/null GIT_CONFIG_NOSYSTEM=1
+
+# A made-up mimir: the task, its public suite and its reference solution.
+M="$T/mimir"; mkdir -p "$M/smoke-notes/sample-suite/tests" "$M/smoke-notes/sample-suite/reference"
+printf '[metadata]\nsuite = "v1"\n' >"$M/smoke-notes/task.toml"
+printf '# brief\n' >"$M/smoke-notes/instruction.md"
+printf '{"schema":"ygg.suite/v1"}\n' >"$M/smoke-notes/sample-suite/tests/manifest.json"
+printf '#!/bin/bash\n' >"$M/smoke-notes/sample-suite/tests/test.sh"
+printf '#!/bin/bash\nexec node server.js\n' >"$M/smoke-notes/sample-suite/reference/start.sh"
+printf 'console.log("it has a \\"quote\\" and a tab\\there");\n' >"$M/smoke-notes/sample-suite/reference/server.js"
+printf '# reference\n' >"$M/smoke-notes/sample-suite/reference/README.md"
+# A made-up odin: the realm and the example config of the real run.
+O="$T/odin/testdata/smoke-grade"; mkdir -p "$O"
+printf 'name: smoke-grade\n' >"$O/pack.yaml"
+cat >"$O/factory.real.example.yaml" <<'EOF'
+mode: unattended
+roles:
+  implementer: { agent: claude-code, model: sonnet }
+  reviewer: { agent: opencode, model: openrouter/deepseek/deepseek-v4-flash }
+budget:
+  per_run: { tokens: 400000, turns: 100, usd: 3 }
+  per_day: { tokens: 1000000, turns: 200, usd: 6 }
+sandbox:
+  engine: @ENGINE@
+  mounts:
+    - { host: @FOLDER@, engine: @BIND@ }
+grader:
+  tasks_dir: @TASKS@
+  suite_dir: @SUITES@
+  timeout: 10m
+  memory: none
+  cpus: 0
+EOF
+
+# The fake ygg. `run` makes a run folder with a ledger and a score.json (reward 1 when the scripted answers hold
+# the reference solution, else 0); FAKE_LEAK puts a second score.json in the project; FAKE_RUN_SLEEP makes it hang.
+mkdir -p "$T/bin" "$T/it"
+cat >"$T/bin/ygg" <<'EOF'
+#!/usr/bin/env bash
+echo "$*" >>"${FAKE_ARGS_LOG:-/dev/null}"
+proj='' cfg=''
+[ -z "${CLAUDE_CODE_OAUTH_TOKEN:-}" ] || echo "loaded:$FAKE_VERB" >>"${FAKE_ARGS_LOG:-/dev/null}"
+while [ $# -gt 0 ]; do case "$1" in --project) proj="$2" ;; --config) cfg="$2" ;; esac; shift; done
+case "$FAKE_VERB" in
+  doctor)
+    [ -z "${FAKE_DOCTOR_RC:-}" ] || { echo "ERROR graders: 2 grader container(s) of ygg runs"; exit "$FAKE_DOCTOR_RC"; }
+    echo "OK    graders: 0 grader container(s) of ygg runs" ;;
+  status) echo "status:   done" ;;
+  build) echo "built" ;;
+  run)
+    [ -z "${FAKE_RUN_SLEEP:-}" ] || exec sleep "$FAKE_RUN_SLEEP"
+    id=20261018-000000-000001
+    mkdir -p "$proj/.factory/runs/$id"
+    script="$(grep -o 'script: [^ ,}]*' "$cfg" | head -n 1 | sed 's/^script: //')"
+    reward=0
+    [ -z "$script" ] || ! grep -q 'server.js' "$script" || reward=1
+    [ -z "${FAKE_REWARD:-}" ] || reward="$FAKE_REWARD"
+    printf '{"schema":"ygg.score/v1","reward":%s,"task":"smoke-notes","task_version":"1.0.0","fail_to_pass":{"passed":6,"total":6},"pass_to_pass":{"passed":1,"total":1}}\n' "$reward" >"$proj/.factory/runs/$id/score.json"
+    [ -z "${FAKE_LEAK:-}" ] || cp "$proj/.factory/runs/$id/score.json" "$proj/score.json"
+    {
+      echo '{"seq":1,"event":"node_finished","ygg.node.id":"build","gen_ai.usage.input_tokens":1000,"gen_ai.usage.output_tokens":200,"gen_ai.usage.cache_read.input_tokens":5000,"ygg.usage.turns":3,"ygg.usage.cost_usd":0.25}'
+      echo '{"seq":2,"event":"node_failed","ygg.node.id":"build","gen_ai.usage.input_tokens":500,"gen_ai.usage.output_tokens":100,"ygg.usage.turns":2,"ygg.usage.cost_usd":0.125,"ygg.usage.cost_estimated":true}'
+      echo '{"seq":3,"event":"grade_finished","ygg.grade.reward":'"$reward"'}'
+      echo '{"seq":4,"event":"run_finished","gen_ai.usage.input_tokens":1500,"gen_ai.usage.output_tokens":300,"ygg.usage.turns":5,"ygg.usage.cost_usd":0.375}'
+    } >"$proj/.factory/runs/$id/ledger.jsonl"
+    [ -z "${FAKE_LEDGER_UNREADABLE:-}" ] || chmod 000 "$proj/.factory/runs/$id/ledger.jsonl"
+    echo "Run $id is done."
+    exit "${FAKE_RUN_RC:-0}" ;;
+  grade)
+    id="$(ls -1 "$proj/.factory/runs" | tail -n 1)"
+    echo '{"seq":9,"event":"grade_finished"}' >>"$proj/.factory/runs/$id/ledger.jsonl"
+    echo "Graded run $id: smoke-notes 1.0.0 against suite v1." ;;
+esac
+EOF
+# The fake podman: no container, or the containers a test asks for. A container line is
+# "<name> <grader label or -> <mount source>": the agents of each case, and the grader that holds the suite.
+cat >"$T/bin/podman" <<'EOF'
+#!/usr/bin/env bash
+w="$(ls -d "$FAKE_IT"/smoke-*/ 2>/dev/null | tail -n 1)"; w="${w%/}"
+case "$1" in
+  ps)
+    case "$*" in *label=ygg.grader*) [ -z "${FAKE_LEFTOVER:-}" ] || echo abc123 ;; *) [ -z "$w" ] || echo agent1 ;; esac ;;
+  inspect)
+    [ -z "$w" ] || {
+      echo "ygg_reference_work_1 - $w/reference/project/repos/notes"
+      echo "ygg_empty_work_1 - $w/empty/project/repos/notes"
+      echo "ygg-grader-run-smoke-notes $FAKE_RUN $w/suites/smoke-notes/v1/tests"
+      [ -z "${FAKE_SUITE_LEAK:-}" ] || echo "ygg_leaky_work_1 - $w/suites/smoke-notes"
+    } ;;
+esac
+EOF
+printf '#!/usr/bin/env bash\necho 0.89.0\n' >"$T/bin/devcontainer"
+chmod +x "$T/bin/ygg" "$T/bin/podman" "$T/bin/devcontainer"
+export PATH="$T/bin:$PATH" FAKE_IT="$T/it" FAKE_RUN=run1
+# YGG_CMD is how ygg runs; this wrapper gives the fake its verb.
+cat >"$T/bin/ygg-cmd" <<'EOF'
+#!/usr/bin/env bash
+verb="$1"; [ "$verb" != sandbox ] || verb=build
+FAKE_VERB="$verb" exec "$(dirname "$0")/ygg" "$@"
+EOF
+chmod +x "$T/bin/ygg-cmd"
+export YGG_CMD="$T/bin/ygg-cmd" YGG_BIND="$T/bind" YGG_ODIN="$T/odin"
+mkdir -p "$T/bind"
+D="$T/it"
+newest() { find "$D" -maxdepth 1 -type d -name "$1" | sort | tail -n 1; } # the newest folder the script made
+gargs=(--folder "$D" --tasks "$M" --odin "$T/odin")
+
+# --- ygg-cost.sh.
+L="$T/ledger.jsonl"
+{
+  echo '{"seq":1,"event":"node_started"}'
+  echo '{"seq":2,"event":"node_finished","gen_ai.usage.input_tokens":1000,"gen_ai.usage.output_tokens":200,"gen_ai.usage.cache_read.input_tokens":5000,"ygg.usage.turns":3,"ygg.usage.cost_usd":0.25}'
+  echo '{"seq":3,"event":"node_failed","gen_ai.usage.input_tokens":500,"gen_ai.usage.output_tokens":100,"ygg.usage.turns":2,"ygg.usage.cost_usd":0.125,"ygg.usage.cost_estimated":true}'
+  echo '{"seq":4,"event":"run_finished","gen_ai.usage.input_tokens":1500,"gen_ai.usage.output_tokens":300,"ygg.usage.turns":5,"ygg.usage.cost_usd":0.375}'
+  echo 'not json'
+} >"$L"
+printf 'budget:\n  per_run: { tokens: 400000, turns: 100, usd: 3 }\n' >"$T/cap.yaml"
+printf 'mode: unattended\n' >"$T/nocap.yaml"
+run "$S/ygg-cost.sh" "$L" "$T/cap.yaml"
+check "the cost counts the two sessions and not the run's own total" has "2 session line(s): 1500 tokens in, 300 out, 5000 cache read, 5 turns"
+check "the cost is the sum of the sessions" has 'cost $0.3750'
+check "an estimated cost is marked" has "an estimate"
+check "the per_run cap is printed beside it" has 'per_run cap is $3'
+run "$S/ygg-cost.sh" "$L" "$T/nocap.yaml"
+check "a factory.yaml with no per_run line does not abort the cost" test "$rc" -eq 0
+check "and says the cap is unknown" has "cap unknown"
+{
+  echo '[1,2]'
+  echo '5'
+  echo 'null'
+  echo '"text"'
+  echo '{"seq":1,"event":"node_finished","gen_ai.usage.input_tokens":null,"gen_ai.usage.output_tokens":7,"ygg.usage.turns":null,"ygg.usage.cost_usd":null}'
+} >"$T/odd.jsonl"
+run "$S/ygg-cost.sh" "$T/odd.jsonl"
+check "non-object lines and null usage fields are tolerated" test "$rc" -eq 0
+check "and the good fields still count" has "1 session line(s): 0 tokens in, 7 out"
+run "$S/ygg-cost.sh" "$T/nope.jsonl"
+check "a missing ledger is refused" test "$rc" -eq 2
+
+# --- ygg-smoke-grade.sh.
+run "$S/ygg-smoke-grade.sh" --nope
+check "an unknown flag is refused" test "$rc" -eq 2
+run "$S/ygg-smoke-grade.sh" "${gargs[@]}"
+check "a missing factory.yaml is refused" test "$rc" -eq 2
+printf 'sandbox:\n  engine: unix:///run/ygg-test.sock\n' >"$D/factory.yaml"
+run "$S/ygg-smoke-grade.sh" --folder "$D" --tasks "$T/nowhere" --odin "$T/odin"
+check "a missing task folder is refused" test "$rc" -eq 2
+run "$S/ygg-smoke-grade.sh" "${gargs[@]}"
+check "a good run passes every check and exits 0" test "$rc" -eq 0
+check "the reference case grades 1" has "smoke-grade: reference reward 1"
+check "the empty case grades 0" has "smoke-grade: empty reward 0"
+check "no check failed" hasnt "FAIL"
+check "the watch sees the grader hold the suite folder" has "the watch saw the grader container hold the suite folder"
+w="$(newest "smoke-grade-*" | head -n 1)"
+check "the suite folder is a copy of the PUBLIC suite and nothing else" diff -r "$M/smoke-notes/sample-suite/tests" "$w/suites/smoke-notes/v1/tests"
+check "the suite is named by the task's metadata.suite" test -f "$w/suites/smoke-notes/v1/tests/manifest.json"
+check "the reference answers write the reference files" grep -q 'path: "server.js"' "$w/reference/script.yaml"
+check "and make start.sh executable" grep -q '"chmod", "+x", "start.sh"' "$w/reference/script.yaml"
+check "the empty answers write no reference file" test -z "$(grep 'server.js' "$w/empty/script.yaml")"
+check "a content with quotes and tabs is written as one YAML string" grep -q 'a \\\\\\"quote' "$w/reference/script.yaml"
+check "the project holds the start.sh that runs the leaf's" grep -q 'repos/notes' "$w/reference/project/start.sh"
+check "the grader runs with no limit on an engine that has none" grep -q 'memory: none' "$w/reference/factory.yaml"
+check "the folder of the suite is outside both projects" test -z "$(find "$w/reference/project" "$w/empty/project" -path '*suites*' -not -path '*/.git/*' | head -n 1)"
+sleep 1
+run "$S/ygg-smoke-grade.sh" "${gargs[@]}" --limits
+check "--limits asks for 4g and 4 CPUs" grep -q 'memory: 4g' "$(newest "smoke-grade-*")/reference/factory.yaml"
+sleep 1
+FAKE_LEAK=1 run "$S/ygg-smoke-grade.sh" "${gargs[@]}"
+check "a score.json outside the run folder fails the smoke" test "$rc" -eq 1
+check "and the failing line says which check" has "FAIL  reference: score.json is in the run folder and nowhere else"
+sleep 1
+FAKE_REWARD=0.5 run "$S/ygg-smoke-grade.sh" "${gargs[@]}"
+check "a reference that does not grade 1 fails the smoke" has "FAIL  reference: reward is 1"
+sleep 1
+FAKE_REWARD=1 run "$S/ygg-smoke-grade.sh" "${gargs[@]}"
+check "an empty product that grades 1 fails the smoke" has "FAIL  empty: reward is below 1"
+sleep 1
+FAKE_LEFTOVER=1 run "$S/ygg-smoke-grade.sh" "${gargs[@]}"
+check "a grader container left behind fails the smoke" has "FAIL  reference: no container with the label ygg.grader is left"
+sleep 1
+FAKE_SUITE_LEAK=1 run "$S/ygg-smoke-grade.sh" "${gargs[@]}"
+check "an agent container that holds the suite folder fails the smoke" has "FAIL  reference: the watch saw"
+check "and the smoke exits 1" test "$rc" -eq 1
+sleep 1
+FAKE_DOCTOR_RC=1 run "$S/ygg-smoke-grade.sh" "${gargs[@]}"
+check "a doctor that does not find 0 grader containers fails the smoke" has "FAIL  reference: ygg doctor finds 0 grader containers"
+sleep 1
+FAKE_REWARD=null run "$S/ygg-smoke-grade.sh" "${gargs[@]}"
+check "an empty product with a null reward fails the smoke" has "FAIL  empty: reward is below 1"
+sleep 1
+FAKE_RUN_SLEEP=31337 "$S/ygg-smoke-grade.sh" "${gargs[@]}" >"$T/out" 2>&1 &
+gpid=$!
+sleep 4
+kill -TERM "$gpid" 2>/dev/null
+pkill -f 'sleep 31337' 2>/dev/null
+wait "$gpid" 2>/dev/null
+sleep 3
+check "a smoke that is killed leaves no poller of the engine behind" test -z "$(pgrep -f "ygg-smoke-grade.sh --folder $D" || true)"
+
+# --- ygg-smoke-real.sh. The credentials are made-up values; none may appear anywhere in the output.
+MARK_A='sekret-claude-0123456789'
+MARK_B='sekret-openrouter-9876543210'
+rargs=(--folder "$D" --tasks "$M" --odin "$T/odin" --env "$T/factory.env")
+run "$S/ygg-smoke-real.sh" "${rargs[@]}" --dry-run
+check "a missing credentials file is refused" test "$rc" -eq 2
+printf 'CLAUDE_CODE_OAUTH_TOKEN=%s\n' "$MARK_A" >"$T/factory.env"
+run "$S/ygg-smoke-real.sh" "${rargs[@]}" --dry-run
+check "a credentials file with one name missing is refused" test "$rc" -eq 2
+check "and the refusal names the missing variable" has "OPENROUTER_API_KEY"
+check "and prints no value" hasnt "$MARK_A"
+printf 'CLAUDE_CODE_OAUTH_TOKEN=%s\nOPENROUTER_API_KEY=\n' "$MARK_A" >"$T/factory.env"
+run "$S/ygg-smoke-real.sh" "${rargs[@]}" --dry-run
+check "a name with no value is refused" test "$rc" -eq 2
+printf 'export CLAUDE_CODE_OAUTH_TOKEN=%s\nOPENROUTER_API_KEY=%s\n' "$MARK_A" "$MARK_B" >"$T/factory.env"
+rm -f "$T/args.log"
+sleep 1
+FAKE_ARGS_LOG="$T/args.log" run "$S/ygg-smoke-real.sh" "${rargs[@]}" --dry-run
+check "a dry run exits 0" test "$rc" -eq 0
+check "a dry run says what it would run, under a wall clock" has "would run: timeout --signal=INT --kill-after=2m 90m"
+check "a dry run runs the doctor" grep -q '^doctor ' "$T/args.log"
+check "a dry run does not start the run" test -z "$(grep '^run ' "$T/args.log")"
+check "a dry run does not load the credentials, not even for the doctor" test -z "$(grep '^loaded:' "$T/args.log")"
+check "a dry run prints the caps" has "per_run: { tokens: 400000, turns: 100, usd: 3 }"
+rc_yaml="$(newest "smoke-real-*")/factory.yaml"
+check "the config has no place left to fill" test -z "$(grep -h '@[A-Z]*@' "$rc_yaml")"
+check "the config has the real agents" grep -q 'agent: claude-code' "$rc_yaml"
+check "and the reviewer on OpenRouter" grep -q 'agent: opencode' "$rc_yaml"
+check "and dollar caps for the day" grep -q 'per_day:.*usd: 6' "$rc_yaml"
+sleep 1
+FAKE_DOCTOR_RC=1 FAKE_ARGS_LOG="$T/args2.log" run "$S/ygg-smoke-real.sh" "${rargs[@]}"
+check "a config that doctor refuses stops the script before the run" test "$rc" -eq 2
+check "and the run did not start" test -z "$(grep '^run ' "$T/args2.log")"
+sleep 1
+FAKE_ARGS_LOG="$T/args3.log" run "$S/ygg-smoke-real.sh" "${rargs[@]}" --build
+check "a run with the fake ygg exits with the run's code" test "$rc" -eq 0
+check "the images are built on request" has "built"
+check "the credentials were loaded for the run" grep -q '^loaded:run$' "$T/args3.log"
+check "and not for the first doctor, which comes before the run" test "$(head -n 1 "$T/args3.log" | cut -d' ' -f1)" = doctor
+check "no credential value is printed" hasnt "$MARK_A"
+check "nor the second" hasnt "$MARK_B"
+check "the score is printed" has "reward 0, fail-to-pass 6/6"
+check "the cost is printed from the ledger" has 'cost $0.3750'
+check "and the cap beside it" has 'per_run cap is $3'
+w="$(newest "smoke-real-*")"
+check "no credential value is in the saved report or run output" test -z "$(grep -rl -e "$MARK_A" -e "$MARK_B" "$w" 2>/dev/null | head -n 1)"
+sleep 1
+FAKE_RUN_SLEEP=30 run "$S/ygg-smoke-real.sh" "${rargs[@]}" --wall 1s
+check "the wall clock ends a run that hangs, and the code is 124" test "$rc" -eq 124
+run "$S/ygg-smoke-real.sh" "${rargs[@]}" --wall soon
+check "a wall clock that is not a duration is refused" test "$rc" -eq 2
+run "$S/ygg-smoke-real.sh" "${rargs[@]}" --wall "1s; touch $T/pwned; echo h"
+check "a hostile wall clock is refused with exit 2" test "$rc" -eq 2
+check "and nothing ran" test ! -e "$T/pwned"
+run "$S/ygg-smoke-real.sh" "${rargs[@]}" --wall "1x;m"
+check "a wall clock like 1x;m is refused" test "$rc" -eq 2
+sleep 1
+FAKE_RUN_RC=3 FAKE_LEDGER_UNREADABLE=1 run "$S/ygg-smoke-real.sh" "${rargs[@]}"
+check "a cost that cannot be read does not replace the run's exit code" test "$rc" -eq 3
+mkdir -p "$T/odin2/testdata/smoke-grade"; cp "$O/pack.yaml" "$T/odin2/testdata/smoke-grade/"
+grep -v -E '^  per_(run|day):' "$O/factory.real.example.yaml" >"$T/odin2/testdata/smoke-grade/factory.real.example.yaml"
+sleep 1
+run "$S/ygg-smoke-real.sh" --folder "$D" --tasks "$M" --odin "$T/odin2" --env "$T/factory.env" --dry-run
+check "a config with no cap lines still dry-runs to exit 0" test "$rc" -eq 0
+sleep 1
+bash -x "$S/ygg-smoke-real.sh" "${rargs[@]}" >"$T/out" 2>&1
+check "a run under bash -x leaves no credential value in the output" test -z "$(grep -F -e "$MARK_A" -e "$MARK_B" "$T/out")"
+check "nor in any saved file" test -z "$(grep -rl -e "$MARK_A" -e "$MARK_B" "$(newest "smoke-real-*")" 2>/dev/null | head -n 1)"
+# The script does not read the credentials file: every line that names it is on this list.
+bad="$(grep -n 'envfile' "$S/ygg-smoke-real.sh" | grep -v -E 'envfile=|--env\)|\[ -f "\$envfile" \]|grep -c -E "[^"]*" "\$envfile"|\. "\$envfile"|\$envfile (has|is missing)|source=|FILE' || true)"
+check "the only uses of the credentials file are the existence test, grep -c on a name, and the subshell load" test -z "$bad"
+check "exactly two lines source the credentials file, both in subshells" test "$(grep -c '^[[:space:]]*\. "\$envfile"' "$S/ygg-smoke-real.sh")" -eq 2
+check "the credentials file is not read by cat, head, tail, sed or awk" test -z "$(grep -E '(cat|head|tail|sed|awk|cut) [^|]*envfile' "$S/ygg-smoke-real.sh")"
+
+exit "$fail"
@@ -0,0 +1,41 @@
+#!/usr/bin/env bash
+# Print what one ygg run spent, read from its ledger.
+#   ygg-cost.sh LEDGER [FACTORY_YAML]
+# Every ledger line that carries a session's usage is added up: the lines of the finished and the failed sessions.
+# The run's own totals (run_finished, run_paused, run_failed) are left out, because they repeat the sessions. The
+# cost is the sum of ygg.usage.cost_usd; a cost the adapter estimated is marked as such. With FACTORY_YAML, the
+# per_run cap is printed beside the spend. The ledger holds no credential; this script reads nothing else.
+set -euo pipefail
+[ $# -ge 1 ] && [ $# -le 2 ] || { echo "usage: ygg-cost.sh LEDGER [FACTORY_YAML]" >&2; exit 2; }
+[ -f "$1" ] || { echo "ygg-cost: no ledger at $1" >&2; exit 2; }
+python3 -I - "$1" <<'PY'
+import json, sys
+totals = {"run_finished", "run_paused", "run_failed"}
+n = tin = tout = cache = turns = 0
+usd = 0.0
+estimated = False
+with open(sys.argv[1], encoding="utf-8") as f:
+    for line in f:
+        try:
+            e = json.loads(line)
+        except ValueError:
+            continue
+        if not isinstance(e, dict) or e.get("event") in totals:
+            continue
+        keys = ("gen_ai.usage.input_tokens", "gen_ai.usage.output_tokens", "ygg.usage.turns", "ygg.usage.cost_usd")
+        if not any(e.get(k) for k in keys):
+            continue
+        n += 1
+        tin += e.get("gen_ai.usage.input_tokens") or 0
+        tout += e.get("gen_ai.usage.output_tokens") or 0
+        cache += e.get("gen_ai.usage.cache_read.input_tokens") or 0
+        turns += e.get("ygg.usage.turns") or 0
+        usd += e.get("ygg.usage.cost_usd") or 0.0
+        estimated = estimated or bool(e.get("ygg.usage.cost_estimated"))
+print("ygg-cost: %d session line(s): %d tokens in, %d out, %d cache read, %d turns" % (n, tin, tout, cache, turns))
+print("ygg-cost: cost $%.4f%s" % (usd, " (at least one session's cost is an estimate)" if estimated else ""))
+PY
+if [ $# -eq 2 ] && [ -f "$2" ]; then
+  cap="$( (grep -E '^  per_run:' "$2" || true) | head -n 1 | sed -n 's/.*usd:[[:space:]]*\([0-9.]*\).*/\1/p')"
+  if [ -n "$cap" ]; then echo "ygg-cost: the per_run cap is \$$cap"; else echo "ygg-cost: the per_run cap unknown (no per_run usd in $2)"; fi
+fi
@@ -0,0 +1,239 @@
+#!/usr/bin/env bash
+# Run the grader smoke (v0-H16): `ygg run` with scripted agents on a real engine, ending in the grade node, then
+# `ygg grade` again, on the reference solution and on an empty product of the task smoke-notes.
+#   ygg-smoke-grade.sh [--build] [--folder DIR] [--tasks DIR] [--odin DIR] [--limits] [--clip]
+# The suite is the PUBLIC sample suite of the task (smoke-notes/sample-suite/tests): this script never reads or
+# writes a hidden suite. It copies the public suite into DIR/smoke-grade-<time>/suites/smoke-notes/v1/tests, and
+# that folder is the grader's suite_dir. Needs DIR/factory.yaml (it names the engine; this script never writes
+# it), the devcontainer CLI 0.89.0, jq and python3, the engine up, the bind mount present, and a checkout of
+# `mimir` (--tasks, default: the mimir folder next to this repo in the umbrella). It
+#   1. checks the engine (ygg-engine.sh), the tools and the task;
+#   2. makes DIR/smoke-grade-<time> with one project per case (reference, empty), each with a committed
+#      `start.sh` that starts the leaf's own, because the grader's /app is the project, and the leaf is in repos/notes;
+#   3. writes the scripted answers from the reference solution of the task (the files are not copied by hand);
+#   4. runs `ygg run` on each project (the brief is the task's instruction.md; the graph ends in the grade node),
+#      and watches the engine while it runs;
+#      (--build first builds the sandbox image of the realm, which a first run needs);
+#   5. runs `ygg grade` on each finished run, and prints one line per check: PASS or FAIL.
+# The checks: reward 1 for the reference; reward below 1 for the empty product; no container with the label
+# ygg.grader is left; ygg doctor finds none; no container that is not a grader ever held the suite folder
+# (the engine is polled every second while the run goes); score.json is in the run folder and in no other place.
+# Exit 0 only when every check passes. The grader runs without a memory or CPU limit (grader.memory: none,
+# cpus: 0), because rootless Podman without cgroup delegation cannot apply them; --limits asks for 4g and 4 CPUs.
+# Defaults: DIR=$HOME/ygg-it. For tests and other machines: YGG_CMD (how ygg runs), YGG_ODIN (the odin folder),
+# YGG_BIND (the bind mount, default /mnt/wsl/ygg-bind), YGG_CLI_VERSION (default 0.89.0). The engine environment
+# comes from ygg-engine.sh (YGG_ENGINE_CHECK=0 skips it).
+set -euo pipefail
+HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+dir="$HOME/ygg-it" tasks="$HERE/../../mimir" limits=0 clip=0 build=0
+odin="${YGG_ODIN:-$HERE/../../yggdrasil/odin}"
+usage() { echo "usage: ygg-smoke-grade.sh [--folder DIR] [--tasks DIR] [--odin DIR] [--build] [--limits] [--clip]" >&2; exit 2; }
+while [ $# -gt 0 ]; do
+  case "$1" in
+    --folder) [ $# -ge 2 ] || usage; dir="$2"; shift 2 ;;
+    --tasks) [ $# -ge 2 ] || usage; tasks="$2"; shift 2 ;;
+    --odin) [ $# -ge 2 ] || usage; odin="$2"; shift 2 ;;
+    --build) build=1; shift ;;
+    --limits) limits=1; shift ;;
+    --clip) clip=1; shift ;;
+    *) usage ;;
+  esac
+done
+fail() { echo "smoke-grade: $*" >&2; exit 2; }
+task=smoke-notes
+bind="${YGG_BIND:-/mnt/wsl/ygg-bind}"
+
+export PATH="$dir/dc/node_modules/.bin:$PATH"
+if [ "${YGG_ENGINE_CHECK:-1}" = 1 ]; then
+  engine_env="$("$HERE/ygg-engine.sh" env --folder "$dir")" || exit 2
+  eval "$engine_env"
+fi
+[ -f "$dir/factory.yaml" ] || fail "$dir/factory.yaml is missing; it names the engine (sandbox.engine)"
+engine="$(sed -n 's/^[[:space:]]*engine:[[:space:]]*\([^[:space:]#]*\).*/\1/p' "$dir/factory.yaml" | head -n 1)"
+case "$engine" in unix://*) ;; *) fail "sandbox.engine in $dir/factory.yaml is '${engine:-missing}'; it must be a unix:// URL" ;; esac
+for tool in jq python3 podman git; do command -v "$tool" >/dev/null 2>&1 || fail "$tool is not installed"; done
+tasks="$(cd "$tasks" 2>/dev/null && pwd)" || fail "no mimir folder at $tasks (--tasks)"
+odin="$(cd "$odin" 2>/dev/null && pwd)" || fail "no odin folder at $odin (--odin)"
+realm="$odin/testdata/smoke-grade"
+public="$tasks/$task/sample-suite"
+for f in "$tasks/$task/task.toml" "$tasks/$task/instruction.md" "$public/tests/manifest.json" "$public/tests/test.sh" "$public/reference/start.sh" "$realm/pack.yaml"; do
+  [ -f "$f" ] || fail "$f is missing"
+done
+suite="$(sed -n 's/^suite[[:space:]]*=[[:space:]]*"\([^"]*\)".*/\1/p' "$tasks/$task/task.toml" | head -n 1)"
+[ -n "$suite" ] || fail "$tasks/$task/task.toml has no metadata.suite"
+
+work="$dir/smoke-grade-$(date +%Y%m%d-%H%M%S)"
+mkdir -p "$work/suites/$task/$suite"
+cp -r "$public/tests" "$work/suites/$task/$suite/tests"
+echo "smoke-grade: made $work"
+echo "smoke-grade: the suite is the PUBLIC sample suite $public/tests (suite $suite)"
+
+ygg="${YGG_CMD:-}"
+if [ -z "$ygg" ]; then
+  if command -v ygg >/dev/null 2>&1; then ygg=ygg
+  else
+    mkdir -p "$work/bin"
+    (cd "$odin" && go build -o "$work/bin/ygg" ./cmd/ygg) || fail "go build of ygg failed in $odin"
+    ygg="'$work/bin/ygg'"
+  fi
+fi
+
+# The scripted answers: the implementer writes the files of the case; the reviewer finds nothing.
+script() { # script CASE OUT: reference writes every file of the task's reference solution; empty writes a README
+  python3 -I - "$1" "$public/reference" >"$2" <<'PY'
+import json, os, sys
+case, ref = sys.argv[1], sys.argv[2]
+usage = "usage: { in: 2000, out: 400, turns: 5 }"
+print("steps:\n  build:\n    - actions:")
+if case == "reference":
+    for name in sorted(os.listdir(ref)):
+        with open(os.path.join(ref, name), encoding="utf-8") as f:
+            print("        - write: { path: %s, content: %s }" % (json.dumps(name), json.dumps(f.read())))
+    print('        - run: { argv: ["chmod", "+x", "start.sh"] }')
+else:
+    print("        - write: { path: README.md, content: %s }" % json.dumps("Nothing is built here.\n"))
+print("      result: { status: ok, summary: built it, %s }" % usage)
+print("  build-review:\n    - result: { status: ok, summary: reviewed, report: [], usage: { in: 800, out: 100, turns: 2 } }")
+PY
+}
+
+# config CASE: the factory.yaml of a case, on stdout.
+config() {
+  local memory=none cpus=0
+  [ "$limits" -eq 0 ] || { memory=4g cpus=4; }
+  cat <<EOF
+mode: unattended
+privacy: strict
+roles:
+  implementer: { agent: scripted, model: none, script: $work/$1/script.yaml }
+  reviewer: { agent: scripted, model: none, script: $work/$1/script.yaml }
+budget:
+  per_run: { tokens: 100000, turns: 100 }
+  per_day: { tokens: 1000000, turns: 1000 }
+sandbox:
+  engine: $engine
+  mounts:
+    - { host: $dir, engine: $bind }
+grader:
+  tasks_dir: $tasks
+  suite_dir: $work/suites
+  timeout: 10m
+  memory: $memory
+  cpus: $cpus
+EOF
+}
+
+results=() bad=0
+check() { # check DESCRIPTION COMMAND...: PASS when the command succeeds
+  local d="$1"
+  shift
+  if "$@" >/dev/null 2>&1; then echo "PASS  $d"; results+=("PASS  $d"); else echo "FAIL  $d"; results+=("FAIL  $d"); bad=1; fi
+}
+# Conditions for check, one per kind.
+is_done() { [ "$1" -eq 0 ] && [ -n "$2" ]; }
+is_zero() { [ "$1" -eq 0 ]; }
+is_empty() { [ -z "$1" ]; }
+is_same() { [ "$1" = "$2" ]; }
+is_count() { [ "$(grep -Ec "$2" "$1")" -eq "$3" ]; }
+seen_agents() { # seen_agents LOG MINE: how many containers of this run (they mount a path under MINE) the watch saw, grader excluded
+  awk -v m="$2" '$2 == "-" && index($0, m) {print $1}' "$1" | sort -u | wc -l
+}
+not_held() { # not_held LOG NEEDLE MINE: the watch saw containers of this run, and none that is not a grader held NEEDLE
+  [ "$(seen_agents "$1" "$3")" -gt 0 ] && ! awk -v n="$2" '$2 == "-" && index($0, n)' "$1" | grep -q .
+}
+held() { # held LOG NEEDLE: the grader container (it has the label) held NEEDLE
+  awk -v n="$2" '$2 != "-" && index($0, n)' "$1" | grep -q .
+}
+
+# watch FILE: polls the engine until the file FILE.stop exists; writes one line per container and poll:
+# "<name> <grader label or -> <mount sources...>".
+watch() {
+  while [ ! -e "$1.stop" ]; do
+    ids="$(podman ps -a -q 2>/dev/null || true)"
+    if [ -n "$ids" ]; then
+      # shellcheck disable=SC2086
+      podman inspect --format '{{.Name}} {{or (index .Config.Labels "ygg.grader") "-"}} {{range .Mounts}}{{.Source}} {{end}}' $ids >>"$1" 2>/dev/null || true
+    fi
+    sleep 1
+  done
+}
+
+oneCase() { # oneCase CASE: build the project, run it to its end, grade it again
+  local c="$1" v="$work/$1" project id rc
+  project="$v/project"
+  mkdir -p "$project"
+  git -C "$project" init -q -b main
+  printf 'root: true\nowner: ouroboric-dev\nvisibility: private\n' >"$project/umbrella.yaml"
+  # The grader's /app is the project. The product is the leaf's; this file starts it.
+  # shellcheck disable=SC2016
+  printf '#!/bin/sh\ncd "$(dirname "$0")/repos/notes" && exec ./start.sh\n' >"$project/start.sh"
+  chmod +x "$project/start.sh"
+  git -C "$project" add umbrella.yaml start.sh
+  git -C "$project" -c user.name=ygg -c user.email=[email protected] commit -q -m "the grader smoke project"
+  script "$c" "$v/script.yaml"
+  config "$c" >"$v/factory.yaml"
+  : >"$v/mounts.log"
+  watch "$v/mounts.log" &
+  watcher=$! watchlog="$v/mounts.log"
+  # An interrupt or an early exit must not leave the poller running.
+  trap 'touch "$watchlog.stop"; kill "$watcher" 2>/dev/null || true' EXIT
+  set +e
+  eval "$ygg run --project '$project' --config '$v/factory.yaml' --realm '$realm' --brief '$tasks/$task/instruction.md'" 2>&1 | tee "$v/run.txt"
+  rc="${PIPESTATUS[0]}"
+  set -e
+  id="$(find "$project/.factory/runs" -mindepth 1 -maxdepth 1 -type d -printf '%f\n' 2>/dev/null | sort | tail -n 1 || true)"
+  check "$c: ygg run ended done (exit $rc, run ${id:-none})" is_done "$rc" "$id"
+  if [ -n "$id" ]; then
+    set +e
+    eval "$ygg grade '$id' --project '$project'" 2>&1 | tee "$v/grade.txt"
+    local grc="${PIPESTATUS[0]}"
+    eval "$ygg doctor --project '$project' --config '$v/factory.yaml'" >"$v/doctor.txt" 2>&1
+    set -e
+    check "$c: ygg grade of the finished run exited 0" is_zero "$grc"
+  fi
+  touch "$v/mounts.log.stop"
+  wait "$watcher" || true
+  trap - EXIT
+  local score="$project/.factory/runs/${id:-none}/score.json" reward=none
+  [ -f "$score" ] && reward="$(jq -r '.reward' "$score")"
+  echo "smoke-grade: $c reward $reward"
+  if [ "$c" = reference ]; then
+    check "reference: reward is 1" jq -e '.reward == 1' "$score"
+  else
+    check "empty: reward is below 1 (it is $reward)" jq -e '(.reward | type) == "number" and .reward < 1' "$score"
+  fi
+  check "$c: the ledger has grade_finished twice (the node, then the command)" is_count "$project/.factory/runs/${id:-none}/ledger.jsonl" '"event": ?"grade_finished"' 2
+  check "$c: no container with the label ygg.grader is left" is_empty "$(podman ps -a -q --filter label=ygg.grader 2>/dev/null || true)"
+  check "$c: ygg doctor finds 0 grader containers" grep -q ' 0 grader container' "$v/doctor.txt"
+  local needle
+  needle="$(basename "$work")/suites"
+  check "$c: the watch saw $(seen_agents "$v/mounts.log" "$(basename "$work")/$c/") agent container(s) of this run, and none held the suite folder" \
+    not_held "$v/mounts.log" "$needle" "$(basename "$work")/$c/"
+  if held "$v/mounts.log" "$needle"; then
+    echo "PASS  $c: the watch saw the grader container hold the suite folder (the watch works)"
+  else
+    echo "NOTE  $c: the watch did not catch the grader container while it ran (a 1 second poll can miss it)"
+  fi
+  local found
+  found="$(find "$v" -name 'score.json' | sort)"
+  check "$c: score.json is in the run folder and nowhere else" is_same "$found" "$score"
+}
+
+if [ "$build" -eq 1 ]; then
+  mkdir -p "$work/reference"
+  config reference >"$work/reference/factory.yaml"
+  eval "$ygg sandbox build --config '$work/reference/factory.yaml' --realm '$realm'"
+fi
+oneCase reference
+oneCase empty
+
+echo
+echo "smoke-grade: the folder is $work"
+report="$work/report.txt"
+printf '%s\n' "${results[@]}" >"$report"
+if [ "$clip" -eq 1 ]; then
+  if command -v clip.exe >/dev/null 2>&1; then clip.exe <"$report"; echo "(copied to the clipboard)" >&2
+  else echo "smoke-grade: --clip needs clip.exe (WSL); the report is in $report" >&2; fi
+fi
+if [ "$bad" -ne 0 ]; then echo "smoke-grade: a check failed"; exit 1; fi
+echo "smoke-grade: every check passed"
@@ -0,0 +1,160 @@
+#!/usr/bin/env bash
+# Run the real-agent smoke (v0-H17): the task smoke-notes built by the real Claude Code and the real OpenCode on
+# OpenRouter, then graded. IT COSTS MONEY: the caps are in odin/testdata/smoke-grade/factory.real.example.yaml.
+#   ygg-smoke-real.sh [--dry-run] [--build] [--folder DIR] [--env FILE] [--tasks DIR] [--odin DIR] [--wall TIME] [--clip]
+# The suite is the PUBLIC sample suite of the task (no hidden suite of smoke-notes exists); the agents never mount
+# it. Needs DIR/factory.yaml (it names the engine; this script never writes it), the credentials file, the
+# devcontainer CLI 0.89.0, jq and python3, the engine up, the bind mount present, and a checkout of `mimir`. It
+#   1. checks the engine, the tools and the task;
+#   2. checks the credentials file WITHOUT reading it: the file exists, and each of the two names
+#      CLAUDE_CODE_OAUTH_TOKEN and OPENROUTER_API_KEY is on one line with a value after it (grep -c counts the
+#      lines; no value is printed or kept);
+#   3. makes DIR/smoke-real-<time> with the project, the public suite and the factory.yaml filled from the
+#      example (its caps are the real ones);
+#   4. runs `ygg doctor` on that config. A config that doctor refuses stops the script before any money is spent;
+#   5. with --dry-run, stops here and prints what it would run;
+#   6. otherwise runs `ygg run` under `timeout` (--wall, default 90m) with the credentials loaded in a subshell
+#      only, and keeps the raw output in DIR/smoke-real-<time>/run.txt (shown live: do not paste that file);
+#   7. prints the evidence report (credentials removed), the score, and the COST read from the ledger.
+# The credentials are loaded by `. FILE` in the subshell that runs ygg, as the other smoke scripts do, and nothing
+# in this script echoes, greps for a value, or copies them. The exit code is the run's (0 done, 1 a node failed,
+# 3 stopped, 4 paused, 5 done with tasks given up, 124 the wall clock ended it) or 2 for a refusal before it.
+# Defaults: DIR=$HOME/ygg-it, FILE=$HOME/.config/ouroboric/factory.env. For tests and other machines: YGG_CMD (how
+# ygg runs), YGG_ODIN (the odin folder), YGG_BIND (the bind mount, default /mnt/wsl/ygg-bind), YGG_CLI_VERSION
+# (default 0.89.0). The engine environment comes from ygg-engine.sh (YGG_ENGINE_CHECK=0 skips it).
+set -euo pipefail
+HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+dir="$HOME/ygg-it" envfile="$HOME/.config/ouroboric/factory.env" tasks="$HERE/../../mimir" wall=90m dry=0 build=0 clip=()
+odin="${YGG_ODIN:-$HERE/../../yggdrasil/odin}"
+usage() { echo "usage: ygg-smoke-real.sh [--dry-run] [--build] [--folder DIR] [--env FILE] [--tasks DIR] [--odin DIR] [--wall TIME] [--clip]" >&2; exit 2; }
+while [ $# -gt 0 ]; do
+  case "$1" in
+    --dry-run) dry=1; shift ;;
+    --build) build=1; shift ;;
+    --folder) [ $# -ge 2 ] || usage; dir="$2"; shift 2 ;;
+    --env) [ $# -ge 2 ] || usage; envfile="$2"; shift 2 ;;
+    --tasks) [ $# -ge 2 ] || usage; tasks="$2"; shift 2 ;;
+    --odin) [ $# -ge 2 ] || usage; odin="$2"; shift 2 ;;
+    --wall) [ $# -ge 2 ] || usage; wall="$2"; shift 2 ;;
+    --clip) clip=(--clip); shift ;;
+    *) usage ;;
+  esac
+done
+fail() { echo "smoke-real: $*" >&2; exit 2; }
+task=smoke-notes
+bind="${YGG_BIND:-/mnt/wsl/ygg-bind}"
+[[ $wall =~ ^[1-9][0-9]*[smh]$ ]] || fail "--wall is a number and s, m or h, such as 90m"
+
+export PATH="$dir/dc/node_modules/.bin:$PATH"
+if [ "${YGG_ENGINE_CHECK:-1}" = 1 ]; then
+  engine_env="$("$HERE/ygg-engine.sh" env --folder "$dir")" || exit 2
+  eval "$engine_env"
+fi
+[ -f "$dir/factory.yaml" ] || fail "$dir/factory.yaml is missing; it names the engine (sandbox.engine)"
+engine="$(sed -n 's/^[[:space:]]*engine:[[:space:]]*\([^[:space:]#]*\).*/\1/p' "$dir/factory.yaml" | head -n 1)"
+case "$engine" in unix://*) ;; *) fail "sandbox.engine in $dir/factory.yaml is '${engine:-missing}'; it must be a unix:// URL" ;; esac
+for tool in jq python3 git timeout; do command -v "$tool" >/dev/null 2>&1 || fail "$tool is not installed"; done
+tasks="$(cd "$tasks" 2>/dev/null && pwd)" || fail "no mimir folder at $tasks (--tasks)"
+odin="$(cd "$odin" 2>/dev/null && pwd)" || fail "no odin folder at $odin (--odin)"
+realm="$odin/testdata/smoke-grade"
+public="$tasks/$task/sample-suite"
+example="$realm/factory.real.example.yaml"
+for f in "$tasks/$task/task.toml" "$tasks/$task/instruction.md" "$public/tests/manifest.json" "$public/tests/test.sh" "$realm/pack.yaml" "$example"; do
+  [ -f "$f" ] || fail "$f is missing"
+done
+suite="$(sed -n 's/^suite[[:space:]]*=[[:space:]]*"\([^"]*\)".*/\1/p' "$tasks/$task/task.toml" | head -n 1)"
+[ -n "$suite" ] || fail "$tasks/$task/task.toml has no metadata.suite"
+
+# The credentials file: its existence, and the two names. grep -c prints a count and nothing else; the file's
+# lines are never shown, stored or compared. A name needs one line, with something after the equals sign.
+[ -f "$envfile" ] || fail "the credentials file $envfile is missing"
+for name in CLAUDE_CODE_OAUTH_TOKEN OPENROUTER_API_KEY; do
+  n="$(grep -c -E "^(export[[:space:]]+)?${name}=[^[:space:]]" "$envfile" || true)"
+  [ "$n" = 1 ] || fail "$envfile has $n line(s) that set $name with a value; it needs exactly 1"
+done
+
+work="$dir/smoke-real-$(date +%Y%m%d-%H%M%S)"
+mkdir -p "$work/suites/$task/$suite" "$work/project"
+cp -r "$public/tests" "$work/suites/$task/$suite/tests"
+project="$work/project"
+git -C "$project" init -q -b main
+printf 'root: true\nowner: ouroboric-dev\nvisibility: private\n' >"$project/umbrella.yaml"
+# The grader's /app is the project. The product is built in the leaf; this file starts it.
+# shellcheck disable=SC2016
+printf '#!/bin/sh\ncd "$(dirname "$0")/repos/notes" && exec ./start.sh\n' >"$project/start.sh"
+chmod +x "$project/start.sh"
+git -C "$project" add umbrella.yaml start.sh
+git -C "$project" -c user.name=ygg -c user.email=[email protected] commit -q -m "the real-agent smoke project"
+echo "smoke-real: made $work"
+echo "smoke-real: the suite is the PUBLIC sample suite $public/tests (suite $suite)"
+
+# The config: the example with its places filled. Its caps are the example's.
+text="$(<"$example")"
+text="${text//@ENGINE@/$engine}"
+text="${text//@FOLDER@/$dir}"
+text="${text//@BIND@/$bind}"
+text="${text//@TASKS@/$tasks}"
+text="${text//@SUITES@/$work/suites}"
+printf '%s\n' "$text" >"$work/factory.yaml"
+if grep -q '@[A-Z]*@' "$work/factory.yaml"; then fail "$work/factory.yaml still has a place to fill: $(grep -o '@[A-Z]*@' "$work/factory.yaml" | sort -u | tr '\n' ' ')"; fi
+
+ygg="${YGG_CMD:-}"
+if [ -z "$ygg" ]; then
+  if command -v ygg >/dev/null 2>&1; then ygg=ygg
+  else
+    mkdir -p "$work/bin"
+    (cd "$odin" && go build -o "$work/bin/ygg" ./cmd/ygg) || fail "go build of ygg failed in $odin"
+    ygg="'$work/bin/ygg'"
+  fi
+fi
+
+echo "smoke-real: the caps in $work/factory.yaml:"
+{ grep -E '^  per_(run|day):' "$work/factory.yaml" || true; } | sed 's/^/  /'
+echo "smoke-real: ygg doctor on that config"
+if ! eval "$ygg doctor --project '$project' --config '$work/factory.yaml'"; then fail "ygg doctor refused the config; nothing was spent"; fi
+
+run="$ygg run --project '$project' --config '$work/factory.yaml' --realm '$realm' --brief '$tasks/$task/instruction.md'"
+if [ "$dry" -eq 1 ]; then
+  [ "$build" -eq 0 ] || echo "smoke-real: would run: $ygg sandbox build --config '$work/factory.yaml' --realm '$realm'"
+  echo "smoke-real: would run: timeout --signal=INT --kill-after=2m $wall $run"
+  echo "smoke-real: dry run: nothing was started and the credentials were not loaded"
+  exit 0
+fi
+
+out="$work/run.txt"
+# The credentials live only in this subshell; nothing here prints them.
+set +e
+(
+  { set +x; } 2>/dev/null # no trace of the sourced assignments, even under bash -x or SHELLOPTS=xtrace
+  set -a
+  # shellcheck source=/dev/null
+  . "$envfile"
+  set +a
+  if [ "$build" -eq 1 ]; then eval "$ygg sandbox build --config '$work/factory.yaml' --realm '$realm'"; fi
+  eval "timeout --signal=INT --kill-after=2m '$wall' $run"
+) 2>&1 | tee "$out"
+rc="${PIPESTATUS[0]}"
+set -e
+echo "smoke-real: the run ended with exit code $rc (0 done, 1 a node failed, 3 stopped, 4 paused, 5 done with tasks given up, 124 wall clock)" >&2
+
+report="$work/report.txt"
+( { set +x; } 2>/dev/null # as above
+  set -a
+  # shellcheck source=/dev/null
+  . "$envfile"
+  set +a
+  YGG_CMD="$ygg" "$HERE/ygg-evidence.sh" --config "$work/factory.yaml" --output "$out" ${clip[@]+"${clip[@]}"} "$project"
+) | tee "$report" || echo "smoke-real: no evidence report: the run left no ledger" >&2
+
+# The score and the cost come from the run folder and its ledger; neither holds a credential.
+id="$(find "$project/.factory/runs" -mindepth 1 -maxdepth 1 -type d -printf '%f\n' 2>/dev/null | sort | tail -n 1 || true)"
+if [ -n "$id" ] && [ -f "$project/.factory/runs/$id/score.json" ]; then
+  jq -r '"smoke-real: score \(.task) \(.task_version): reward \(.reward), fail-to-pass \(.fail_to_pass.passed)/\(.fail_to_pass.total), pass-to-pass \(.pass_to_pass.passed)/\(.pass_to_pass.total)"' "$project/.factory/runs/$id/score.json"
+else
+  echo "smoke-real: no score.json: the run did not reach the grade node"
+fi
+if [ -n "$id" ] && [ -f "$project/.factory/runs/$id/ledger.jsonl" ]; then
+  "$HERE/ygg-cost.sh" "$project/.factory/runs/$id/ledger.jsonl" "$work/factory.yaml" || echo "smoke-real: the cost could not be read from the ledger" >&2
+fi
+echo "smoke-real: the folder is $work" >&2
+exit "$rc"