From e982c3a26a28b9d250a32d2f22f6faede1911c9c Mon Sep 17 00:00:00 2001 From: "ouroboric-pr-bot[bot]" <341648780+ouroboric-pr-bot[bot]@users.noreply.github.com> Date: Fri, 23 Oct 2026 10:43:31 +0200 Subject: [PATCH] ygg-proof-run.sh: the reviewer's reasoning effort and --reject (gallery-runtime T28, T29) (#78) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * The proof run sets the reviewer's reasoning effort, medium by default (F2) --reviewer-reasoning low|medium|high|off goes into the reviewer role of the proof config, the plan line and proof.meta; --continue refuses it as a setup flag. (gallery-runtime-T28) Co-Authored-By: Claude Opus 5.5 * A proof folder from before --reviewer-reasoning can still be continued (gallery-runtime-T28) Without a reviewer_reasoning line in proof.meta, --continue shows "reasoning unset" in the plan; a bad value still fails. Co-Authored-By: Claude Opus 5.5 * ygg-proof-run.sh --continue --reject GATE --reason TEXT, run as --approve is; re-wrap the header (gallery-runtime-T29) Co-Authored-By: Claude Opus 5.5 * ygg-proof-run.sh: neutral step 9 wording, report header records reasoning, wrap fixes (gallery-runtime-T29) Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Bruno Guilhermo de Barros Valério Co-authored-by: Claude Opus 5.5 --- AGENTS.md | 2 +- .../testdata/proof/factory.proof.example.yaml | 5 +- scripts/tests/ygg-proof-run.test.sh | 75 ++++++++++++++-- scripts/ygg-proof-run.sh | 90 +++++++++++++------ 4 files changed, 136 insertions(+), 36 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index fd51f74..00c2432 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -41,7 +41,7 @@ folder. | The grader smoke on a real engine (v0-H16), with the public sample suite | `scripts/ygg-smoke-grade.sh [--build]`. Never a hidden suite by default; `--task ID --suite-dir DIR --reference DIR [--suite-engine DIR]` points the grader at a suite store as it is (never copied or read) and grades the task's reference solution; its output holds numbers only | | The real-agent smoke (v0-H17): costs money; `--dry-run` first; the cost from the ledger; a start from a planted-fault leaf; another implementer; a note before the brief; the repair path from the ledger | `scripts/ygg-smoke-real.sh --dry-run`, then without it; `--from-leaf LEAF`, `--implementer AGENT:MODEL`, `--brief-note FILE` (narrows the implementer's task; the run is graded against a derived task version in its folder; the ygg line says if the build used go.work); `scripts/ygg-cost.sh LEDGER`, `scripts/ygg-repair-path.sh LEDGER`. It never prints the credentials file | | Compare reviewer models: costs money unless `--dry-run`; on the planted-fault leaves in `scripts/testdata/reviewer-bench`; runs the OpenCode version gleipnir pins (installed once with npm; `--opencode BIN`, `--any-opencode`) | `scripts/ygg-reviewer-bench.sh --model MODEL --leaf DIR --sessions N --dry-run`, then without it | -| A Phase 6 proof run (v0-T290, v0-T291): costs money without `--dry-run`; `--dry-run` first; the `software` graph on the real task with three roles and caps you set, then the grade and the evidence; never reads the suites folder or prints the credentials | `scripts/ygg-proof-run.sh --name NAME --dry-run [--planner AGENT:MODEL --implementer AGENT:MODEL --reviewer AGENT:MODEL --run-usd N ...]`, then without `--dry-run`. `--build` builds the sandbox images first; `--baseline RUN_ID` compares with a run of an earlier proof run; `--continue RUN_ID --approve GATE` or `--resume` continues a stopped or paused run; `--clip` copies the report. The run's output is in `run.txt` and the evidence in `report.txt`, in `DIR/proof-NAME-TIME`. The example config is `scripts/testdata/proof/factory.proof.example.yaml` | +| A Phase 6 proof run (v0-T290, v0-T291): costs money without `--dry-run`; `--dry-run` first; the `software` graph on the real task with three roles and caps you set, then the grade and the evidence; never reads the suites folder or prints the credentials | `scripts/ygg-proof-run.sh --name NAME --dry-run [--planner AGENT:MODEL --implementer AGENT:MODEL --reviewer AGENT:MODEL --run-usd N ...]`, then without `--dry-run`. `--reviewer-reasoning EFFORT` (low, medium, high or off; default `medium`) sets the reviewer's reasoning effort; `--build` builds the sandbox images first; `--baseline RUN_ID` compares with a run of an earlier proof run; `--continue RUN_ID --approve GATE`, `--reject GATE --reason TEXT` or `--resume` continues a stopped or paused run (a rejection starts sessions, so it needs the credentials as an approval does); `--clip` copies the report. The run's output is in `run.txt` and the evidence in `report.txt`, in `DIR/proof-NAME-TIME`. The example config is `scripts/testdata/proof/factory.proof.example.yaml` | | Build a plan from reference commits, or execute one task by task | `scripts/plan-*` (`plan-generate.py`, `plan-replay.py`, `plan-states.sh`, `plan-red.sh`, `plan-defences.py`; `plan-blocks.py`, `plan-task-start.sh`, `plan-task-done.sh`, `plan-task-land.sh`) | | Check that the repos have the required setup, and that the realms' shared files match | `scripts/as-bot.sh --read all scripts/check-repos.sh`; `--check shared` runs the realms check alone | | Scan git history for secrets | `scripts/scan-history.sh` | diff --git a/scripts/testdata/proof/factory.proof.example.yaml b/scripts/testdata/proof/factory.proof.example.yaml index 7d4e61c..181745e 100644 --- a/scripts/testdata/proof/factory.proof.example.yaml +++ b/scripts/testdata/proof/factory.proof.example.yaml @@ -1,13 +1,14 @@ # The factory.yaml of a proof run (v0-H19 to v0-H23), with the places between two at signs that # scripts/ygg-proof-run.sh fills from its flags. It never writes your factory.yaml. The credentials come from the # environment, never from this file. A "usd" line is written only when a role pays per token (an opencode role): -# the caps then count what the API billed, not the cost figure of a subscription's sessions. +# the caps then count what the API billed, not the cost figure of a subscription's sessions. The reviewer's +# "reasoning" is the effort it asks for (a claude-code agent ignores it). mode: @MODE@ privacy: strict roles: planner: { agent: @PLANNER_AGENT@, model: @PLANNER_MODEL@ } implementer: { agent: @IMPLEMENTER_AGENT@, model: @IMPLEMENTER_MODEL@ } - reviewer: { agent: @REVIEWER_AGENT@, model: @REVIEWER_MODEL@ } + reviewer: { agent: @REVIEWER_AGENT@, model: @REVIEWER_MODEL@, reasoning: @REVIEWER_REASONING@ } budget: per_run: { tokens: @RUN_TOKENS@, turns: @RUN_TURNS@@RUN_USD@ } per_day: { tokens: @DAY_TOKENS@, turns: @DAY_TURNS@@DAY_USD@ } diff --git a/scripts/tests/ygg-proof-run.test.sh b/scripts/tests/ygg-proof-run.test.sh index f1db174..8ea85ab 100755 --- a/scripts/tests/ygg-proof-run.test.sh +++ b/scripts/tests/ygg-proof-run.test.sh @@ -57,7 +57,7 @@ cat >"$FB/ygg" <<'EOF3' #!/bin/sh # Fake ygg (v0-T291): records its arguments and the credential NAMES in its environment, answers each subcommand. # FAKE_RUN_RC is the exit code of `run`; FAKE_PLANT is a value it prints from run, metrics, report, grade and status. -# FAKE_CONT_RC is the exit code of `approve` and `resume`. +# FAKE_CONT_RC is the exit code of `approve`, `resume` and `reject`. echo "$*" >>"$FAKE_DIR/ygg.calls" printf '%s: %s\n' "$1" "$(env | grep -E '^(CLAUDE_CODE_OAUTH_TOKEN|OPENROUTER_API_KEY)=' | cut -d= -f1 | sort | tr '\n' ' ')" >>"$FAKE_DIR/ygg.env" proj="" prev="" @@ -84,7 +84,7 @@ case "$1" in while :; do sleep 0.2; done fi exit "${FAKE_RUN_RC:-0}" ;; - approve | resume) + approve | resume | reject) echo "$1 runs" [ -z "${FAKE_PLANT:-}" ] || echo "$1 saw $FAKE_PLANT" exit "${FAKE_CONT_RC:-0}" ;; @@ -206,7 +206,7 @@ check "the umbrella.yaml sets root: true" grep -q '^root: true$' "$P/project/umb check "the project folder is DIR/proof--