mirror of
https://github.com/esengine/DeepSeek-Reasonix.git
synced 2026-11-10 01:51:40 -10:00
Compare commits
2
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
8fc6882c22 | ||
|
|
f53edb599a |
No files matched your search
@@ -2,7 +2,6 @@
|
||||
.gitignore text eol=lf
|
||||
.npmrc text eol=lf
|
||||
*.astro text eol=lf
|
||||
*.bat text eol=crlf
|
||||
*.css text eol=lf
|
||||
*.go text eol=lf
|
||||
*.html text eol=lf
|
||||
@@ -27,4 +26,3 @@
|
||||
*.yml text eol=lf
|
||||
.githooks/** text eol=lf
|
||||
benchmarks/**/*.sh text eol=lf
|
||||
internal/runtime/agent/testdata/perseveration/** -text
|
||||
+13
-42
@@ -1,43 +1,14 @@
|
||||
# Ownership for the studio line. Standard: docs/DOCS_STANDARD.md.
|
||||
#
|
||||
# Each rule lists exactly two owners: primary first, backup second.
|
||||
# GitHub requests review from both. The last matching rule wins.
|
||||
# repolint (codeowners, doc-owner) fails a rule for a missing path, a rule
|
||||
# without a backup, and a document header that names different owners.
|
||||
# Every pull request requests review from the maintainers below.
|
||||
# Either maintainer can review and merge any change.
|
||||
* @SivanCola @esengine
|
||||
|
||||
# Default: kernel, Studio shell, tooling, standing instructions.
|
||||
* @esengine @SivanCola
|
||||
|
||||
# User-facing guides and references.
|
||||
/docs/ @SivanCola @esengine
|
||||
/sdk/ @SivanCola @esengine
|
||||
/README.md @SivanCola @esengine
|
||||
/README.zh-CN.md @SivanCola @esengine
|
||||
/SECURITY.md @SivanCola @esengine
|
||||
|
||||
# Contracts and processes the Studio line runs on.
|
||||
/docs/DOCS_STANDARD.md @esengine @SivanCola
|
||||
/docs/STUDIO_*.md @esengine @SivanCola
|
||||
/docs/SESSION_MEMORY_RETRIEVAL.md @esengine @SivanCola
|
||||
/docs/*_CONTRACT.md @esengine @SivanCola
|
||||
/docs/TOOL_APPROVAL_MODES.md @esengine @SivanCola
|
||||
/docs/CAPABILITY_DIAGNOSTICS.md @esengine @SivanCola
|
||||
/docs/CODE_SIGNING_POLICY.md @esengine @SivanCola
|
||||
/docs/CHECKPOINTS.md @esengine @SivanCola
|
||||
/docs/EXTENSION_*.md @esengine @SivanCola
|
||||
/docs/SUBAGENT_*.md @esengine @SivanCola
|
||||
/docs/ROADMAP.md @esengine @SivanCola
|
||||
/docs/MIGRATING.md @esengine @SivanCola
|
||||
/docs/design/ @esengine @SivanCola
|
||||
/release-notes/ @esengine @SivanCola
|
||||
|
||||
# Release signing: provider policy, its contract and the workflow that uses it.
|
||||
/.signpath/ @esengine @SivanCola
|
||||
/.github/workflows/release-studio.yml @esengine @SivanCola
|
||||
/.github/workflows/studio-certum-signing-smoke.yml @esengine @SivanCola
|
||||
/.github/actions/setup-certum/ @esengine @SivanCola
|
||||
/cmd/signpath-contract/ @esengine @SivanCola
|
||||
/scripts/sign-certum.ps1 @esengine @SivanCola
|
||||
/scripts/verify-windows-authenticode.ps1 @esengine @SivanCola
|
||||
/scripts/verify-windows-package.ps1 @esengine @SivanCola
|
||||
/scripts/windows-signing-lib.ps1 @esengine @SivanCola
|
||||
# Keep the provider-side signing policy and its GitHub control plane under the
|
||||
# same explicit review boundary as the machine-readable contract.
|
||||
/.signpath/ @SivanCola @esengine
|
||||
/.github/workflows/release-stable.yml @SivanCola @esengine
|
||||
/.github/workflows/release-stable-trigger.yml @SivanCola @esengine
|
||||
/.github/workflows/release-desktop.yml @SivanCola @esengine
|
||||
/scripts/complete-signpath-request.ps1 @SivanCola @esengine
|
||||
/scripts/package-windows-desktop.sh @SivanCola @esengine
|
||||
/scripts/verify-windows-authenticode.ps1 @SivanCola @esengine
|
||||
/cmd/signpath-contract/ @SivanCola @esengine
|
||||
@@ -7,25 +7,23 @@ body:
|
||||
attributes:
|
||||
value: |
|
||||
Thanks for filing a bug. Pick which version line you're on below — it gets
|
||||
labeled automatically, no maintainer action needed. Not sure which line
|
||||
you run? See the [version roadmap](https://github.com/esengine/DeepSeek-Reasonix/discussions/10748).
|
||||
labeled (`v1`/`v2`) automatically, no maintainer action needed.
|
||||
- type: dropdown
|
||||
id: version-line
|
||||
attributes:
|
||||
label: Version line
|
||||
description: Which line are you running?
|
||||
options:
|
||||
- "2.x — Reasonix Studio, studio branch (active development)"
|
||||
- "1.x — main-v2 branch (maintenance / stable)"
|
||||
- "0.x — legacy TypeScript (maintenance only)"
|
||||
- "v2 — Go rewrite (1.x), main-v2 (active development)"
|
||||
- "v1 — Legacy TypeScript (0.x), maintenance only"
|
||||
validations:
|
||||
required: true
|
||||
- type: input
|
||||
id: version
|
||||
attributes:
|
||||
label: Exact version
|
||||
description: Output of `reasonix --version`, or the version shown in Studio's settings.
|
||||
placeholder: e.g. 2.20.0 or 1.39.0
|
||||
description: Output of `reasonix --version`.
|
||||
placeholder: e.g. 1.0.0
|
||||
validations:
|
||||
required: true
|
||||
- type: textarea
|
||||
|
||||
@@ -1,8 +1,5 @@
|
||||
blank_issues_enabled: false
|
||||
contact_links:
|
||||
- name: Version roadmap (1.x / 2.x)
|
||||
url: https://github.com/esengine/DeepSeek-Reasonix/discussions/10748
|
||||
about: Which line to use, which branch takes which changes, and where Reasonix is heading.
|
||||
- name: Questions & discussions
|
||||
url: https://github.com/esengine/DeepSeek-Reasonix/discussions
|
||||
about: For usage questions, ideas, and general discussion — please use Discussions instead of opening an issue.
|
||||
@@ -9,9 +9,8 @@ body:
|
||||
label: Version line
|
||||
description: Which line is this for?
|
||||
options:
|
||||
- "2.x — Reasonix Studio, studio branch (active development)"
|
||||
- "1.x — main-v2 branch (maintenance / stable)"
|
||||
- "0.x — legacy TypeScript (maintenance only)"
|
||||
- "v2 — Go rewrite (1.x), main-v2 (active development)"
|
||||
- "v1 — Legacy TypeScript (0.x), maintenance only"
|
||||
validations:
|
||||
required: true
|
||||
- type: textarea
|
||||
|
||||
@@ -1,35 +0,0 @@
|
||||
name: Connect to Certum cloud signing
|
||||
description: Verify the pinned SimplySign installer and connect to the configured certificate.
|
||||
inputs:
|
||||
username:
|
||||
required: true
|
||||
description: SimplySign account
|
||||
otp-uri:
|
||||
required: true
|
||||
description: TOTP provisioning URI
|
||||
thumbprint:
|
||||
required: true
|
||||
description: Expected SHA-1 certificate thumbprint
|
||||
runs:
|
||||
using: composite
|
||||
steps:
|
||||
- name: Verify SimplySign installer
|
||||
shell: pwsh
|
||||
run: |
|
||||
$ErrorActionPreference = 'Stop'
|
||||
Invoke-WebRequest 'https://files.certum.eu/software/SimplySignDesktop/Windows/9.4.3.90/SimplySignDesktop-9.4.3.90-64-bit-en.msi' -OutFile SimplySignDesktop.msi
|
||||
$hash = (Get-FileHash -Algorithm SHA256 SimplySignDesktop.msi).Hash
|
||||
if ($hash -ne 'CE2B38EF0124A574BC08E5558726099CAAC79A7C58B8D3D712F629371FE1BCB7') {
|
||||
throw "SimplySign installer SHA-256 $hash does not match the pinned 9.4.3.90 build"
|
||||
}
|
||||
$signature = Get-AuthenticodeSignature SimplySignDesktop.msi
|
||||
if ($signature.Status -ne 'Valid' -or $signature.SignerCertificate.Subject -notmatch 'Asseco Data Systems') {
|
||||
throw 'SimplySign installer signature or publisher verification failed'
|
||||
}
|
||||
- name: Authenticate cloud certificate
|
||||
uses: dismine/windows-app-signing-setup-action@89ae3b032d4bc7a5b98d1a42a34e61ecb6faad64
|
||||
with:
|
||||
certum-username: ${{ inputs.username }}
|
||||
certum-otp-uri: ${{ inputs.otp-uri }}
|
||||
certum-key-id: ${{ inputs.thumbprint }}
|
||||
capture-diagnostics: 'false'
|
||||
+2
-61
@@ -1,6 +1,5 @@
|
||||
version: 2
|
||||
updates:
|
||||
# 2.x (studio): the development line.
|
||||
- package-ecosystem: "gomod"
|
||||
directory: "/"
|
||||
schedule:
|
||||
@@ -22,7 +21,7 @@ updates:
|
||||
update-types: ["minor", "patch"]
|
||||
|
||||
- package-ecosystem: "npm"
|
||||
directory: "/desktop/frontend-next"
|
||||
directory: "/desktop/frontend"
|
||||
schedule:
|
||||
interval: "weekly"
|
||||
open-pull-requests-limit: 5
|
||||
@@ -32,7 +31,7 @@ updates:
|
||||
update-types: ["minor", "patch"]
|
||||
|
||||
- package-ecosystem: "npm"
|
||||
directory: "/desktop/electron"
|
||||
directory: "/site"
|
||||
schedule:
|
||||
interval: "weekly"
|
||||
open-pull-requests-limit: 5
|
||||
@@ -48,61 +47,3 @@ updates:
|
||||
groups:
|
||||
actions:
|
||||
patterns: ["*"]
|
||||
|
||||
# 1.x (main-v2): maintenance. Dependabot reads this file from the default
|
||||
# branch only, so the 1.x line is named here; it takes patch updates alone.
|
||||
- package-ecosystem: "gomod"
|
||||
directory: "/"
|
||||
target-branch: "main-v2"
|
||||
schedule:
|
||||
interval: "weekly"
|
||||
open-pull-requests-limit: 3
|
||||
ignore:
|
||||
- dependency-name: "*"
|
||||
update-types: ["version-update:semver-major", "version-update:semver-minor"]
|
||||
groups:
|
||||
go-1x:
|
||||
patterns: ["*"]
|
||||
update-types: ["patch"]
|
||||
|
||||
- package-ecosystem: "gomod"
|
||||
directory: "/desktop"
|
||||
target-branch: "main-v2"
|
||||
schedule:
|
||||
interval: "weekly"
|
||||
open-pull-requests-limit: 3
|
||||
ignore:
|
||||
- dependency-name: "*"
|
||||
update-types: ["version-update:semver-major", "version-update:semver-minor"]
|
||||
groups:
|
||||
go-1x:
|
||||
patterns: ["*"]
|
||||
update-types: ["patch"]
|
||||
|
||||
- package-ecosystem: "npm"
|
||||
directory: "/desktop/frontend"
|
||||
target-branch: "main-v2"
|
||||
schedule:
|
||||
interval: "weekly"
|
||||
open-pull-requests-limit: 3
|
||||
ignore:
|
||||
- dependency-name: "*"
|
||||
update-types: ["version-update:semver-major", "version-update:semver-minor"]
|
||||
groups:
|
||||
npm-1x:
|
||||
patterns: ["*"]
|
||||
update-types: ["patch"]
|
||||
|
||||
- package-ecosystem: "npm"
|
||||
directory: "/site"
|
||||
target-branch: "main-v2"
|
||||
schedule:
|
||||
interval: "weekly"
|
||||
open-pull-requests-limit: 3
|
||||
ignore:
|
||||
- dependency-name: "*"
|
||||
update-types: ["version-update:semver-major", "version-update:semver-minor"]
|
||||
groups:
|
||||
npm-1x:
|
||||
patterns: ["*"]
|
||||
update-types: ["patch"]
|
||||
+9
-9
@@ -6,36 +6,36 @@
|
||||
agent:
|
||||
- changed-files:
|
||||
- any-glob-to-any-file:
|
||||
- 'internal/runtime/agent/**'
|
||||
- 'internal/session/control/**'
|
||||
- 'internal/agent/**'
|
||||
- 'internal/control/**'
|
||||
|
||||
mcp:
|
||||
- changed-files:
|
||||
- any-glob-to-any-file:
|
||||
- 'internal/ext/plugin/**'
|
||||
- 'internal/plugin/**'
|
||||
- 'internal/codegraph/**'
|
||||
|
||||
config:
|
||||
- changed-files:
|
||||
- any-glob-to-any-file:
|
||||
- 'internal/contract/config/**'
|
||||
- 'internal/assembly/boot/**'
|
||||
- 'internal/config/**'
|
||||
- 'internal/boot/**'
|
||||
|
||||
provider:
|
||||
- changed-files:
|
||||
- any-glob-to-any-file:
|
||||
- 'internal/contract/provider/**'
|
||||
- 'internal/provider/**'
|
||||
|
||||
skills:
|
||||
- changed-files:
|
||||
- any-glob-to-any-file:
|
||||
- 'internal/ext/skill/**'
|
||||
- 'internal/contract/tool/**'
|
||||
- 'internal/skill/**'
|
||||
- 'internal/tool/**'
|
||||
|
||||
tui:
|
||||
- changed-files:
|
||||
- any-glob-to-any-file:
|
||||
- 'internal/frontend/cli/**'
|
||||
- 'internal/cli/**'
|
||||
|
||||
desktop:
|
||||
- changed-files:
|
||||
|
||||
@@ -1,9 +1,3 @@
|
||||
<!--
|
||||
Target branch: 2.x features and fixes go to `studio`. 1.x takes bug fixes,
|
||||
provider/API compatibility, release/updater, security and platform stability
|
||||
only, on `main-v2`.
|
||||
-->
|
||||
|
||||
## Summary
|
||||
|
||||
-
|
||||
@@ -26,3 +20,25 @@ the source-to-target contract here: easing syntax, time units, callbacks,
|
||||
cancellation, reduced-motion behavior, and failure fallback. Verification must
|
||||
assert that the target API was actually called; a mock that silently skips it
|
||||
does not count.
|
||||
|
||||
## Documentation impact
|
||||
|
||||
Documentation-impact: TODO
|
||||
|
||||
For changes to user-visible CLI, Desktop, configuration, provider, permission,
|
||||
or tool behavior, use one of:
|
||||
|
||||
- `Documentation-impact: updated - <what changed>` and update `docs/*.md`.
|
||||
- `Documentation-impact: none - <why the embedded documentation remains correct>`.
|
||||
|
||||
## Cache impact
|
||||
|
||||
Cache-impact: TODO
|
||||
Cache-guard: TODO
|
||||
System-prompt-review: N/A
|
||||
|
||||
For cache-sensitive changes, fill these lines before requesting review:
|
||||
|
||||
- `Cache-impact`: `none`, `low`, `medium`, or `high`, plus the reason.
|
||||
- `Cache-guard`: the focused guard test/command added or run, or why an existing guard covers the change.
|
||||
- `System-prompt-review`: required reviewer/approval note when provider-visible system prompt, memory prefix, output style, or skill index behavior changes.
|
||||
@@ -0,0 +1,49 @@
|
||||
name: Cache impact guard
|
||||
|
||||
on:
|
||||
pull_request_target:
|
||||
branches: [main-v2]
|
||||
types: [opened, synchronize, reopened, edited, ready_for_review]
|
||||
# The guard only cares about the prompt/tool surfaces listed in
|
||||
# scripts/check-cache-impact.sh; PRs confined to clearly unrelated paths
|
||||
# can never trip it, so skip the run entirely. Keep the ignore list
|
||||
# conservative: when in doubt, run.
|
||||
paths-ignore:
|
||||
- 'docs/**'
|
||||
- 'site/**'
|
||||
- 'release-notes/**'
|
||||
- 'workers/**'
|
||||
- 'benchmarks/**'
|
||||
- 'npm/**'
|
||||
- '*.md'
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: read
|
||||
|
||||
concurrency:
|
||||
group: cache-impact-${{ github.event.pull_request.number }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
cache-impact:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
ref: ${{ github.event.pull_request.base.sha }}
|
||||
|
||||
- name: Collect PR changed files
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
PR_NUMBER: ${{ github.event.pull_request.number }}
|
||||
REPOSITORY: ${{ github.repository }}
|
||||
run: |
|
||||
gh api --paginate "repos/${REPOSITORY}/pulls/${PR_NUMBER}/files" \
|
||||
--jq '.[].filename' > "$RUNNER_TEMP/cache-impact-files.txt"
|
||||
|
||||
- name: Check cache-sensitive PR metadata
|
||||
env:
|
||||
PR_BODY: ${{ github.event.pull_request.body }}
|
||||
CACHE_IMPACT_CHANGED_FILES_FILE: ${{ runner.temp }}/cache-impact-files.txt
|
||||
run: bash scripts/check-cache-impact.sh
|
||||
+260
-107
@@ -1,14 +1,10 @@
|
||||
name: CI
|
||||
|
||||
# studio is a parallel release line, not a feature branch: it carries the
|
||||
# kernel changes Studio is built on, so it needs the same kernel gates main-v2
|
||||
# has. Studio's own surface (desktop/next, desktop/frontend-next) exists only
|
||||
# there and is covered by studio.yml.
|
||||
on:
|
||||
push:
|
||||
branches: [main-v2, studio]
|
||||
branches: [main-v2]
|
||||
pull_request:
|
||||
branches: [main-v2, studio]
|
||||
branches: [main-v2]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -18,7 +14,7 @@ concurrency:
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
# Cheap path gate for pull requests. PRs confined to docs/release-notes
|
||||
# Cheap path gate for pull requests. PRs confined to docs/site/release-notes
|
||||
# (or top-level Markdown) skip the heavy Go jobs, and PRs that cannot affect
|
||||
# the desktop module skip the desktop jobs. Job-level `if` reports skipped,
|
||||
# which satisfies the required status checks (lint, race, test) — a
|
||||
@@ -32,15 +28,15 @@ jobs:
|
||||
outputs:
|
||||
code: ${{ steps.filter.outputs.code }}
|
||||
desktop: ${{ steps.filter.outputs.desktop }}
|
||||
site: ${{ steps.filter.outputs.site }}
|
||||
sdk: ${{ steps.filter.outputs.sdk }}
|
||||
scripts: ${{ steps.filter.outputs.scripts }}
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- id: filter
|
||||
run: |
|
||||
code=true; desktop=true; sdk=true; scripts=true
|
||||
code=true; desktop=true; site=true; sdk=true
|
||||
base=""
|
||||
if [ "${{ github.event_name }}" = "pull_request" ]; then
|
||||
base="${{ github.event.pull_request.base.sha }}"
|
||||
@@ -50,37 +46,21 @@ jobs:
|
||||
if [ -n "$base" ] && git cat-file -e "$base^{commit}" 2>/dev/null; then
|
||||
files=$(git diff --name-only "$base" HEAD)
|
||||
if [ -n "$files" ]; then
|
||||
code=false; desktop=false; sdk=false; scripts=false
|
||||
code=false; desktop=false; site=false; sdk=false
|
||||
# Root-module CI: desktop/ is a separate module, so only the
|
||||
# clearly unrelated paths below can skip it.
|
||||
if echo "$files" | grep -qvE '^(docs/|release-notes/|desktop/|workers/|[^/]+\.md$)'; then code=true; fi
|
||||
if echo "$files" | grep -qvE '^(docs/|site/|release-notes/|desktop/|workers/|[^/]+\.md$)'; then code=true; fi
|
||||
# desktop/ imports the root kernel via `replace reasonix => ../`,
|
||||
# so only this clearly-unrelated set is safe to skip.
|
||||
if echo "$files" | grep -qvE '^(docs/|release-notes/|workers/|benchmarks/|npm/|[^/]+\.md$)'; then desktop=true; fi
|
||||
if echo "$files" | grep -qvE '^(docs/|site/|release-notes/|workers/|benchmarks/|npm/|[^/]+\.md$)'; then desktop=true; fi
|
||||
if echo "$files" | grep -q '^site/'; then site=true; fi
|
||||
# sdk/go is a nested module invisible to root `go test ./...`;
|
||||
# its DTOs are generated from internal/ext/extension/protocol, so
|
||||
# its DTOs are generated from internal/extension/protocol, so
|
||||
# both paths must trigger the sdk job.
|
||||
if echo "$files" | grep -qE '^(sdk/|internal/ext/extension/)'; then sdk=true; fi
|
||||
# The release-note tests render every committed Studio note; npm/ holds the publish tests.
|
||||
if echo "$files" | grep -qE '^(scripts/|npm/|release-notes/studio/)'; then scripts=true; fi
|
||||
if echo "$files" | grep -qE '^(sdk/|internal/extension/)'; then sdk=true; fi
|
||||
fi
|
||||
fi
|
||||
{ echo "code=$code"; echo "desktop=$desktop"; echo "sdk=$sdk"; echo "scripts=$scripts"; } >> "$GITHUB_OUTPUT"
|
||||
|
||||
release-notes-scripts:
|
||||
needs: changes
|
||||
if: always() && (github.event_name != 'pull_request' || needs.changes.outputs.scripts != 'false')
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
|
||||
with:
|
||||
node-version: "22"
|
||||
- run: node --test scripts/release-credits.test.mjs scripts/markdown-refs.test.mjs scripts/studio-release-notes.test.mjs npm/publish.test.mjs
|
||||
{ echo "code=$code"; echo "desktop=$desktop"; echo "site=$site"; echo "sdk=$sdk"; } >> "$GITHUB_OUTPUT"
|
||||
|
||||
# The ruleset requires the per-OS check names (test (ubuntu-latest) etc.).
|
||||
# A matrix job skipped at job level reports no per-leg checks at all, so
|
||||
@@ -101,10 +81,10 @@ jobs:
|
||||
RUN_STEPS: ${{ github.event_name != 'pull_request' || needs.changes.outputs.code != 'false' }}
|
||||
steps:
|
||||
- if: env.RUN_STEPS == 'true'
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- if: env.RUN_STEPS == 'true'
|
||||
uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: go.mod
|
||||
cache: true
|
||||
@@ -144,21 +124,6 @@ jobs:
|
||||
if: env.RUN_STEPS == 'true'
|
||||
run: go build ./...
|
||||
|
||||
# The trajectory analyzer is the last consumer of the todo_progress frame,
|
||||
# so a rename anywhere upstream has to fail somewhere. Standard library
|
||||
# only; the runner's python3 is enough.
|
||||
- name: test (trajectory tools)
|
||||
if: env.RUN_STEPS == 'true' && runner.os != 'Windows'
|
||||
run: python3 -m unittest discover -s tools/trajectory -t tools/trajectory
|
||||
|
||||
# A hardware exception on a Windows host whose CPU saves a large XSTATE
|
||||
# (Intel AMX) can corrupt the Go heap (golang/go#81238). The runner pool
|
||||
# mixes hosts, so a runtime crash is only attributable with the CPU on record.
|
||||
- name: runner CPU
|
||||
if: env.RUN_STEPS == 'true' && runner.os == 'Windows'
|
||||
shell: pwsh
|
||||
run: Get-CimInstance Win32_Processor | Select-Object -ExpandProperty Name
|
||||
|
||||
- name: test
|
||||
if: env.RUN_STEPS == 'true' && runner.os != 'Windows'
|
||||
env:
|
||||
@@ -176,52 +141,44 @@ jobs:
|
||||
# remains independently bounded by Go's per-package timeout below.
|
||||
#
|
||||
# That timeout is 8m, not 3m: across four consecutive runs of the same
|
||||
# code, internal/assembly/boot measured 133s, 146s, 162s and then hit 180s, and
|
||||
# internal/runtime/agent 82s, 92s, 144s and 180s — a 2.2x spread from runner
|
||||
# code, internal/boot measured 133s, 146s, 162s and then hit 180s, and
|
||||
# internal/agent 82s, 92s, 144s and 180s — a 2.2x spread from runner
|
||||
# variance alone, with four heavy packages sharing four vCPUs. At 3m the
|
||||
# step failed on slow runners without a code change; a hang still trips
|
||||
# the 15 minute job budget above.
|
||||
#
|
||||
# Every root-module `go test` that runs on Windows passes -count=1. With
|
||||
# the result cache on, go test records each file and environment access
|
||||
# a test binary makes, and before and after the run resolves every
|
||||
# recorded path outside the module through filepath.EvalSymlinks.
|
||||
# internal/assembly/boot records about 1.9M accesses; on a Windows runner
|
||||
# that bookkeeping costs minutes per pass while nothing is printed.
|
||||
# internal/base/testenv holds every such step to it.
|
||||
- name: test (Windows smoke)
|
||||
if: env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request'
|
||||
timeout-minutes: 15
|
||||
env:
|
||||
REASONIX_RELEASE_CACHE_GUARD: "1"
|
||||
WINDOWS_SANDBOX_WAIT_MS: "20000"
|
||||
run: go test -count=1 -p 4 -timeout=8m ./internal/runtime/agent/... ./internal/platform/appidentity/... ./internal/assembly/boot/... ./internal/state/checkpoint/... ./internal/frontend/cli/... ./internal/session/control/... ./internal/base/filelock/... ./internal/base/fileutil/... ./internal/ext/hook/... ./internal/state/instruction/... ./internal/ext/mcplaunch/... ./internal/platform/notify/... ./internal/base/proc/... ./internal/platform/remote/... ./internal/platform/repair/... ./internal/safety/sandbox/... ./internal/base/sysproxy/... ./internal/state/workspacelease/... ./cmd/...
|
||||
run: go test -p 4 -timeout=8m ./internal/agent/... ./internal/appidentity/... ./internal/boot/... ./internal/checkpoint/... ./internal/cli/... ./internal/control/... ./internal/desktoplauncher/... ./internal/filelock/... ./internal/fileutil/... ./internal/hook/... ./internal/instruction/... ./internal/mcplaunch/... ./internal/notify/... ./internal/proc/... ./internal/remote/... ./internal/repair/... ./internal/sandbox/... ./internal/sysproxy/... ./internal/workspacelease/... ./cmd/...
|
||||
|
||||
# The general Windows PR smoke list intentionally omits internal/contract/tool.
|
||||
# The general Windows PR smoke list intentionally omits internal/tool.
|
||||
# Keep the session-temp portability contract covered without widening the
|
||||
# leg to every tool test: two real PowerShell launches must share the
|
||||
# injected TMPDIR/TMP/TEMP directory.
|
||||
- name: test (Windows session temp)
|
||||
if: env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request'
|
||||
timeout-minutes: 3
|
||||
run: go test -count=1 -timeout=2m -run '^TestBashSharesSessionTempAcrossCalls$' ./internal/tools/builtin
|
||||
run: go test -timeout=2m -run '^TestBashSharesSessionTempAcrossCalls$' ./internal/tool/builtin
|
||||
|
||||
# Shell execution contract: PowerShell identity, Chinese workspace paths,
|
||||
# UTF-8 output, and ExitCode retention must gate PRs on native Windows.
|
||||
# Keep this focused (not full ./internal/contract/tool) so the smoke budget holds.
|
||||
# Keep this focused (not full ./internal/tool) so the smoke budget holds.
|
||||
- name: test (Windows shell execution contract)
|
||||
if: env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request'
|
||||
timeout-minutes: 5
|
||||
run: go test -count=1 -timeout=3m -run '^TestBashPowerShellExecuteDetailedContract$|^TestBashPowerShell51PreflightRejectsAndAndDetailed$|^TestBashPowerShellOutputIsUTF8$|^TestBashPowerShellSurfacesNonZeroExit$|^TestBashPowerShellRejectsChaining$' ./internal/tools/builtin
|
||||
run: go test -timeout=3m -run '^TestBashPowerShellExecuteDetailedContract$|^TestBashPowerShell51PreflightRejectsAndAndDetailed$|^TestBashPowerShellOutputIsUTF8$|^TestBashPowerShellSurfacesNonZeroExit$|^TestBashPowerShellRejectsChaining$' ./internal/tool/builtin
|
||||
|
||||
# Full push suite still uses the same 8m package budget as Windows smoke:
|
||||
# agent/boot/control already hit 180s package timeouts under runner load,
|
||||
# so 3m fails on slow machines without a code regression. It runs with
|
||||
# -count=1 for the reason given at the smoke step. A hang still dies here
|
||||
# rather than on a package.
|
||||
# so 3m fails on slow machines without a code regression. Keep the step
|
||||
# above normal full-suite wall time (~8-12m) so a hang still dies on the
|
||||
# job budget rather than a false package timeout.
|
||||
- name: test (full)
|
||||
if: env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name != 'pull_request'
|
||||
timeout-minutes: 30
|
||||
timeout-minutes: 20
|
||||
env:
|
||||
# Run the prompt-cache prefix-stability guard (TestCacheHit*) in CI: a
|
||||
# regression there silently tanks the cache hit rate the project is
|
||||
@@ -230,16 +187,22 @@ jobs:
|
||||
# Bound sandbox helper children in Windows tests so a failed OS-level
|
||||
# launch cannot pin the Actions step after Go's package timeout fires.
|
||||
WINDOWS_SANDBOX_WAIT_MS: "20000"
|
||||
run: go test -count=1 -p 4 -timeout=8m ./...
|
||||
run: go test -p 4 -timeout=8m ./...
|
||||
|
||||
- name: test (Scoop desktop launch)
|
||||
if: env.RUN_STEPS == 'true' && runner.os == 'Windows'
|
||||
timeout-minutes: 10
|
||||
shell: powershell
|
||||
run: .\scripts\test-scoop-desktop-launch.ps1
|
||||
|
||||
race:
|
||||
needs: changes
|
||||
if: always() && (github.event_name != 'pull_request' || needs.changes.outputs.code != 'false')
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
- uses: actions/checkout@v7
|
||||
|
||||
- uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7
|
||||
- uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: go.mod
|
||||
cache: true
|
||||
@@ -266,7 +229,7 @@ jobs:
|
||||
if: github.event_name == 'pull_request'
|
||||
env:
|
||||
REASONIX_RELEASE_CACHE_GUARD: "1"
|
||||
run: go test -race ./internal/runtime/agent/... ./internal/ext/plugin/... ./internal/tools/jobs/... ./internal/base/proc/... ./internal/safety/sandbox/... ./internal/base/filelock/... ./internal/contract/eventwire/... ./internal/platform/remote/... ./internal/ext/extension/... ./internal/assembly/boot/... ./internal/session/control/... ./internal/contract/tool/...
|
||||
run: go test -race ./internal/agent/... ./internal/plugin/... ./internal/jobs/... ./internal/proc/... ./internal/sandbox/... ./internal/filelock/... ./internal/eventwire/... ./internal/remote/... ./internal/extension/... ./internal/boot/... ./internal/control/... ./internal/tool/...
|
||||
|
||||
- name: test -race (full)
|
||||
if: github.event_name != 'pull_request'
|
||||
@@ -275,7 +238,7 @@ jobs:
|
||||
run: go test -race ./...
|
||||
|
||||
# sdk/go is a nested stdlib-only module invisible to root `go test ./...`.
|
||||
# Its DTOs are generated from internal/ext/extension/protocol, and the
|
||||
# Its DTOs are generated from internal/extension/protocol, and the
|
||||
# host-side conformance tests spawn the SDK example, so the job runs the
|
||||
# same three-OS matrix as the root tests.
|
||||
sdk:
|
||||
@@ -293,10 +256,10 @@ jobs:
|
||||
working-directory: sdk/go
|
||||
steps:
|
||||
- if: env.RUN_STEPS == 'true'
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- if: env.RUN_STEPS == 'true'
|
||||
uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: sdk/go/go.mod
|
||||
|
||||
@@ -341,14 +304,30 @@ jobs:
|
||||
run:
|
||||
working-directory: desktop
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
- uses: actions/checkout@v7
|
||||
|
||||
- uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7
|
||||
- uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: desktop/go.mod
|
||||
cache: true
|
||||
cache-dependency-path: desktop/go.sum
|
||||
|
||||
- uses: pnpm/[email protected]
|
||||
with:
|
||||
version: 10
|
||||
run_install: false
|
||||
|
||||
- uses: actions/setup-node@v7
|
||||
with:
|
||||
node-version: "24"
|
||||
cache: pnpm
|
||||
cache-dependency-path: desktop/frontend/pnpm-lock.yaml
|
||||
|
||||
- name: Install Wails CLI
|
||||
run: |
|
||||
echo "$(go env GOPATH)/bin" >> "$GITHUB_PATH"
|
||||
go install "github.com/wailsapp/wails/v2/cmd/wails@$(cat "$GITHUB_WORKSPACE/.wails-version")"
|
||||
|
||||
- name: gofmt
|
||||
run: |
|
||||
unformatted=$(gofmt -l .)
|
||||
@@ -367,11 +346,37 @@ jobs:
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# WebKitGTK 4.0 toolchain (pinned to ubuntu-22.04; no webkit2_41 tag).
|
||||
- name: Install Linux build deps
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y gcc libgtk-3-dev libwebkit2gtk-4.0-dev xvfb
|
||||
|
||||
- name: Build frontend
|
||||
run: |
|
||||
wails generate module
|
||||
pnpm --dir frontend install --frozen-lockfile
|
||||
PLAYWRIGHT_BROWSERS_PATH=.pw-browsers pnpm --dir frontend exec playwright install --with-deps chromium
|
||||
pnpm --dir frontend build
|
||||
|
||||
- name: Test desktop frontend
|
||||
run: |
|
||||
pnpm --dir frontend test:terminal
|
||||
pnpm --dir frontend test:task-monitor
|
||||
pnpm --dir frontend test:workspace
|
||||
pnpm --dir frontend test:motion
|
||||
pnpm --dir frontend test:motion-browser
|
||||
pnpm --dir frontend test:transcript
|
||||
pnpm --dir frontend test:transcript-browser
|
||||
|
||||
- name: Test usage statistics frontend
|
||||
run: pnpm --dir frontend test:usage-stats
|
||||
|
||||
- name: vet
|
||||
run: go vet ./...
|
||||
|
||||
- name: golangci-lint
|
||||
uses: golangci/golangci-lint-action@ba0d7d2ec06a0ea1cb5fa41b2e4a3ab91d21278a # v9
|
||||
uses: golangci/golangci-lint-action@v9
|
||||
with:
|
||||
version: v2.12.2
|
||||
working-directory: desktop
|
||||
@@ -383,13 +388,46 @@ jobs:
|
||||
- name: test
|
||||
run: go test ./...
|
||||
|
||||
# The update helper replaces a running application under pkexec, so its
|
||||
# own concurrency is CI-blocked rather than author-verified locally.
|
||||
- name: WebKitGTK 4.0 native recovery smoke
|
||||
run: xvfb-run -a env WEBKIT_DISABLE_COMPOSITING_MODE=1 go test -tags reasonix_webkit_smoke . -run '^TestWebKitNativeRecoverySmoke$'
|
||||
|
||||
# The extension work added new shared-state paths to the desktop
|
||||
# runtime (tab/rebuild fences, extension UI). Race-sweep the module so
|
||||
# regressions are CI-blocked, not just author-verified locally.
|
||||
- name: test -race
|
||||
run: go test -race ./...
|
||||
|
||||
# Ubuntu 24.04 ships WebKitGTK 4.1 only. Keep this capability build separate
|
||||
# from the 22.04/4.0 leg so neither runtime becomes a distro whitelist.
|
||||
desktop-linux-webkit41:
|
||||
needs: changes
|
||||
if: always() && (github.event_name != 'pull_request' || needs.changes.outputs.desktop != 'false')
|
||||
runs-on: ubuntu-24.04
|
||||
defaults:
|
||||
run:
|
||||
working-directory: desktop
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
|
||||
- uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: desktop/go.mod
|
||||
cache: true
|
||||
cache-dependency-path: desktop/go.sum
|
||||
|
||||
- name: Install WebKitGTK 4.1 build dependencies
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y gcc libgtk-3-dev libwebkit2gtk-4.1-dev xvfb
|
||||
|
||||
- name: Build and test WebKitGTK 4.1 observer
|
||||
run: go test -tags webkit2_41 ./...
|
||||
|
||||
- name: WebKitGTK 4.1 native recovery smoke
|
||||
run: xvfb-run -a env WEBKIT_DISABLE_COMPOSITING_MODE=1 go test -tags 'webkit2_41 reasonix_webkit_smoke' . -run '^TestWebKitNativeRecoverySmoke$'
|
||||
|
||||
# desktop/ is a separate module, so the root macOS matrix above does not
|
||||
# compile it against a real macOS toolchain.
|
||||
# compile or exercise the native PTY implementation.
|
||||
desktop-macos:
|
||||
needs: changes
|
||||
if: always() && (github.event_name != 'pull_request' || needs.changes.outputs.desktop != 'false')
|
||||
@@ -398,26 +436,31 @@ jobs:
|
||||
run:
|
||||
working-directory: desktop
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
- uses: actions/checkout@v7
|
||||
|
||||
- uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7
|
||||
- uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: desktop/go.mod
|
||||
cache: true
|
||||
cache-dependency-path: desktop/go.sum
|
||||
|
||||
- name: test
|
||||
run: go test -race ./...
|
||||
- name: Test integrated terminal and PTY lifecycle
|
||||
run: go test -race -run 'Test(ResolveTerminal|Terminal|EmptyTerminal|UnixTerminalProcess)' .
|
||||
|
||||
# desktop/*_darwin.go is the one build-tag set the lint job cannot reach:
|
||||
# the main-thread watchdog is cgo, so it only type-checks with a real
|
||||
# macOS toolchain.
|
||||
- name: golangci-lint
|
||||
uses: golangci/golangci-lint-action@ba0d7d2ec06a0ea1cb5fa41b2e4a3ab91d21278a # v9
|
||||
uses: golangci/golangci-lint-action@v9
|
||||
with:
|
||||
version: v2.12.2
|
||||
working-directory: desktop
|
||||
args: --timeout=5m
|
||||
|
||||
# The update helper's versioned-install and uninstall paths are Windows-only
|
||||
# (//go:build windows), so the ubuntu leg above never compiles them.
|
||||
# Desktop project-root matching is case-insensitive only on Windows
|
||||
# (sameDesktopPath folds case when os.PathSeparator is '\'), so the
|
||||
# regression tests for that contract are named *OnWindows and can never
|
||||
# run on the ubuntu leg above.
|
||||
desktop-windows:
|
||||
needs: changes
|
||||
if: always() && (github.event_name != 'pull_request' || needs.changes.outputs.desktop != 'false')
|
||||
@@ -427,17 +470,79 @@ jobs:
|
||||
shell: bash
|
||||
working-directory: desktop
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
- uses: actions/checkout@v7
|
||||
|
||||
- uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7
|
||||
- uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: desktop/go.mod
|
||||
cache: true
|
||||
cache-dependency-path: desktop/go.sum
|
||||
|
||||
- name: test (update helper and uninstaller)
|
||||
- uses: pnpm/[email protected]
|
||||
with:
|
||||
version: 10
|
||||
run_install: false
|
||||
|
||||
- uses: actions/setup-node@v7
|
||||
with:
|
||||
node-version: "24"
|
||||
cache: pnpm
|
||||
cache-dependency-path: desktop/frontend/pnpm-lock.yaml
|
||||
|
||||
- name: Install Wails CLI
|
||||
run: |
|
||||
echo "$(go env GOPATH)/bin" >> "$GITHUB_PATH"
|
||||
go install "github.com/wailsapp/wails/v2/cmd/wails@$(cat "$GITHUB_WORKSPACE/.wails-version")"
|
||||
|
||||
# go:embed of frontend/dist needs a built frontend before the package
|
||||
# compiles, same as the ubuntu desktop leg.
|
||||
- name: Build frontend
|
||||
run: |
|
||||
wails generate module
|
||||
pnpm --dir frontend install --frozen-lockfile
|
||||
pnpm --dir frontend build
|
||||
|
||||
- name: Test native motion contracts
|
||||
run: pnpm --dir frontend test:motion
|
||||
|
||||
- name: Test WebView2 native smoke state machine
|
||||
shell: pwsh
|
||||
run: ../scripts/test-webview2-native-smoke.ps1 -SelfTest
|
||||
|
||||
# Build the real Wails executable on every desktop PR and exercise the
|
||||
# exact production startup path in Microsoft's installed WebView2 runtime.
|
||||
# The frontend and bindings were already built above, so this skips only
|
||||
# duplicate build work, not the native shell.
|
||||
- name: Build Wails executable for native startup smoke
|
||||
timeout-minutes: 10
|
||||
run: wails build -clean -s -skipbindings -nopackage -platform windows/amd64 -webview2 embed
|
||||
|
||||
- name: Smoke-test Wails/WebView2 native startup
|
||||
shell: pwsh
|
||||
timeout-minutes: 2
|
||||
run: |
|
||||
../scripts/test-webview2-native-smoke.ps1 `
|
||||
-ExecutablePath "build/bin/reasonix-desktop.exe" `
|
||||
-TimeoutSeconds 60 `
|
||||
-HealthySeconds 5
|
||||
|
||||
- name: test (Windows desktop and update helper)
|
||||
timeout-minutes: 15
|
||||
run: go test ./...
|
||||
run: |
|
||||
go test ./...
|
||||
(cd third_party/go-webview2 && go test ./pkg/edge)
|
||||
|
||||
# Installer packaging is packaging validation, not a code gate: run it
|
||||
# on pushes to main-v2 (the release pipeline builds installers again
|
||||
# anyway), but let pull requests stop after the test step.
|
||||
- name: Install NSIS
|
||||
if: github.event_name != 'pull_request'
|
||||
run: pwsh -NoProfile -File ../scripts/install-nsis.ps1
|
||||
|
||||
- name: Build Windows installer and portable archive
|
||||
if: github.event_name != 'pull_request'
|
||||
timeout-minutes: 20
|
||||
run: ../scripts/desktop-build.sh windows/amd64 v0.0.0-ci canary
|
||||
|
||||
# repolint scans desktop/ and sdk/ too, so this job also runs for diffs the
|
||||
# `code` filter would otherwise skip.
|
||||
@@ -446,20 +551,39 @@ jobs:
|
||||
if: always() && (github.event_name != 'pull_request' || needs.changes.outputs.code != 'false' || needs.changes.outputs.desktop != 'false' || needs.changes.outputs.sdk != 'false')
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
- uses: actions/checkout@v7
|
||||
|
||||
- uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7
|
||||
- uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: go.mod
|
||||
cache: true
|
||||
|
||||
- if: github.event_name != 'pull_request' || needs.changes.outputs.desktop != 'false'
|
||||
uses: pnpm/[email protected]
|
||||
with:
|
||||
version: 10
|
||||
run_install: false
|
||||
|
||||
- if: github.event_name != 'pull_request' || needs.changes.outputs.desktop != 'false'
|
||||
uses: actions/setup-node@v7
|
||||
with:
|
||||
node-version: "24"
|
||||
cache: pnpm
|
||||
cache-dependency-path: desktop/frontend/pnpm-lock.yaml
|
||||
|
||||
- name: required native motion contracts
|
||||
if: github.event_name != 'pull_request' || needs.changes.outputs.desktop != 'false'
|
||||
run: |
|
||||
pnpm --dir desktop/frontend install --frozen-lockfile
|
||||
pnpm --dir desktop/frontend test:motion
|
||||
|
||||
- name: repo standards
|
||||
run: go run ./tools/repolint
|
||||
|
||||
# The paths the project declared sensitive, held to the coverage they
|
||||
# already have. It runs only those packages' tests, not the full suite.
|
||||
- name: sensitive-path coverage
|
||||
run: go run ./tools/covergate
|
||||
- name: wails version pin
|
||||
run: |
|
||||
bash scripts/check-wails-pin.sh
|
||||
bash scripts/check-wails-pin.test.sh
|
||||
|
||||
# `make lint-install` reads the same file, so a local run and this job
|
||||
# cannot drift onto different linter versions.
|
||||
@@ -467,7 +591,7 @@ jobs:
|
||||
run: echo "version=$(cat .golangci-version)" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: golangci-lint
|
||||
uses: golangci/golangci-lint-action@ba0d7d2ec06a0ea1cb5fa41b2e4a3ab91d21278a # v9
|
||||
uses: golangci/golangci-lint-action@v9
|
||||
with:
|
||||
version: ${{ steps.golangci.outputs.version }}
|
||||
args: --timeout=5m
|
||||
@@ -491,9 +615,38 @@ jobs:
|
||||
run: |
|
||||
go run github.com/rhysd/actionlint/cmd/[email protected] \
|
||||
-ignore 'label "windows-11-arm" is unknown' \
|
||||
.github/workflows/release-studio.yml \
|
||||
.github/workflows/studio-certum-signing-smoke.yml \
|
||||
.github/workflows/release-stable.yml \
|
||||
.github/workflows/prepare-release-notes.yml \
|
||||
.github/workflows/release-stable-trigger.yml \
|
||||
.github/workflows/release.yml \
|
||||
.github/workflows/release-npm.yml \
|
||||
.github/workflows/release-desktop.yml \
|
||||
.github/workflows/release-verify-issues.yml \
|
||||
.github/workflows/docs-impact.yml \
|
||||
.github/workflows/ci.yml
|
||||
bash scripts/release-workflows.test.sh
|
||||
node scripts/check-single-release-public-contract.mjs
|
||||
|
||||
# The site/ auth client has security-sensitive redirect-validation logic
|
||||
# (safeNext) covered by node:test unit tests. Those tests use only Node
|
||||
# builtins, so no `npm install` is needed — run them directly on every PR so
|
||||
# a regression in redirect validation fails the build instead of shipping.
|
||||
site:
|
||||
needs: changes
|
||||
if: always() && (github.event_name != 'pull_request' || needs.changes.outputs.site != 'false')
|
||||
runs-on: ubuntu-latest
|
||||
defaults:
|
||||
run:
|
||||
working-directory: site
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
|
||||
- uses: actions/setup-node@v7
|
||||
with:
|
||||
node-version: "24"
|
||||
|
||||
- name: test
|
||||
run: npm test
|
||||
|
||||
govulncheck:
|
||||
needs: changes
|
||||
@@ -501,9 +654,9 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
continue-on-error: true # informational — stdlib vulns need a Go patch release
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
- uses: actions/checkout@v7
|
||||
|
||||
- uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7
|
||||
- uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: go.mod
|
||||
cache: true
|
||||
@@ -519,9 +672,9 @@ jobs:
|
||||
if: always() && (github.event_name != 'pull_request' || needs.changes.outputs.code != 'false')
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
- uses: actions/checkout@v7
|
||||
|
||||
- uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7
|
||||
- uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: go.mod
|
||||
cache: true
|
||||
@@ -530,7 +683,7 @@ jobs:
|
||||
run: go test -coverprofile=coverage.out -covermode=atomic ./...
|
||||
|
||||
- name: upload coverage
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: coverage-report
|
||||
path: coverage.out
|
||||
|
||||
@@ -1,35 +0,0 @@
|
||||
name: Close linked issues
|
||||
|
||||
# GitHub closes an issue named by "Fixes #N" only when the pull request merges
|
||||
# into the default branch. This does the same for the 1.x line. It reads the
|
||||
# merged pull request's description and never checks out its code.
|
||||
on:
|
||||
pull_request_target:
|
||||
types: [closed]
|
||||
branches: [main-v2]
|
||||
|
||||
permissions:
|
||||
issues: write
|
||||
|
||||
jobs:
|
||||
close:
|
||||
if: github.event.pull_request.merged == true
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9
|
||||
with:
|
||||
script: |
|
||||
const pr = context.payload.pull_request;
|
||||
const { owner, repo } = context.repo;
|
||||
// GitHub's own closing-keyword grammar, applied to this repository's issues only.
|
||||
const keyword = /\b(?:close[sd]?|fix(?:e[sd])?|resolve[sd]?)\s*:?\s+#(\d+)\b/gi;
|
||||
const numbers = [...new Set([...(pr.body ?? '').matchAll(keyword)].map((m) => Number(m[1])))];
|
||||
const sha = (pr.merge_commit_sha ?? '').slice(0, 9);
|
||||
for (const issue_number of numbers) {
|
||||
const { data } = await github.rest.issues.get({ owner, repo, issue_number });
|
||||
if (data.pull_request || data.state !== 'open') continue;
|
||||
await github.rest.issues.createComment({ owner, repo, issue_number,
|
||||
body: `Fixed on the 1.x line by #${pr.number} (${sha}). It ships in the next 1.x release.` });
|
||||
await github.rest.issues.update({ owner, repo, issue_number, state: 'closed', state_reason: 'completed' });
|
||||
core.info(`closed #${issue_number}`);
|
||||
}
|
||||
@@ -4,25 +4,23 @@ on:
|
||||
# Docs/release-notes/root-Markdown changes carry no analyzable code; the
|
||||
# weekly cron keeps the full scan cadence regardless.
|
||||
push:
|
||||
branches: ["main-v2", "studio"]
|
||||
branches: ["main-v2"]
|
||||
paths-ignore:
|
||||
- 'docs/**'
|
||||
- 'release-notes/**'
|
||||
- '*.md'
|
||||
pull_request:
|
||||
branches: ["main-v2", "studio"]
|
||||
branches: ["main-v2"]
|
||||
paths-ignore:
|
||||
- 'docs/**'
|
||||
- 'release-notes/**'
|
||||
- '*.md'
|
||||
# A schedule runs only from the default branch, so the weekly scan checks out
|
||||
# both lines itself: 1.x is in maintenance, and security is part of it.
|
||||
schedule:
|
||||
- cron: "27 3 * * 1"
|
||||
|
||||
jobs:
|
||||
analyze:
|
||||
name: Analyze (${{ matrix.language }}${{ matrix.branch && format(', {0}', matrix.branch) || '' }})
|
||||
name: Analyze (${{ matrix.language }})
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
security-events: write
|
||||
@@ -32,32 +30,21 @@ jobs:
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
language: [go, javascript-typescript, actions]
|
||||
# Empty means "the ref this event is about"; a scheduled run names each line.
|
||||
branch: ${{ fromJSON(github.event_name == 'schedule' && '["main-v2", "studio"]' || '[""]') }}
|
||||
include:
|
||||
- language: go
|
||||
build-mode: autobuild
|
||||
- language: javascript-typescript
|
||||
build-mode: none
|
||||
- language: actions
|
||||
build-mode: none
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
with:
|
||||
ref: ${{ matrix.branch }}
|
||||
- id: head
|
||||
if: matrix.branch != ''
|
||||
run: echo "sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT"
|
||||
- uses: actions/checkout@v7
|
||||
- name: Initialize CodeQL
|
||||
uses: github/codeql-action/init@2892aa5e19bbd11bc0cff5427e3b750a04d9e3c2 # v4.38.2
|
||||
uses: github/codeql-action/init@v4.37.4
|
||||
with:
|
||||
languages: ${{ matrix.language }}
|
||||
build-mode: ${{ matrix.language == 'go' && 'autobuild' || 'none' }}
|
||||
build-mode: ${{ matrix.build-mode }}
|
||||
- name: Perform CodeQL Analysis
|
||||
if: matrix.branch == ''
|
||||
uses: github/codeql-action/analyze@2892aa5e19bbd11bc0cff5427e3b750a04d9e3c2 # v4.38.2
|
||||
uses: github/codeql-action/[email protected]
|
||||
with:
|
||||
category: "/language:${{ matrix.language }}"
|
||||
# Results from a checked-out line are filed under that line's ref, not the
|
||||
# default branch the schedule happened to run from.
|
||||
- name: Perform CodeQL Analysis (${{ matrix.branch }})
|
||||
if: matrix.branch != ''
|
||||
uses: github/codeql-action/analyze@2892aa5e19bbd11bc0cff5427e3b750a04d9e3c2 # v4.38.2
|
||||
with:
|
||||
category: "/language:${{ matrix.language }}"
|
||||
ref: refs/heads/${{ matrix.branch }}
|
||||
sha: ${{ steps.head.outputs.sha }}
|
||||
@@ -19,12 +19,12 @@ jobs:
|
||||
deploy:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
- uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413 # v6.1.0
|
||||
- uses: actions/checkout@v7
|
||||
- uses: pnpm/action-setup@v6.0.9
|
||||
with:
|
||||
version: 10
|
||||
run_install: false
|
||||
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
|
||||
- uses: actions/setup-node@v7
|
||||
with:
|
||||
node-version: "24"
|
||||
cache: pnpm
|
||||
|
||||
@@ -19,8 +19,8 @@ jobs:
|
||||
deploy:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
|
||||
- uses: actions/checkout@v7
|
||||
- uses: actions/setup-node@v7
|
||||
with:
|
||||
node-version: '22'
|
||||
cache: npm
|
||||
|
||||
@@ -19,12 +19,12 @@ jobs:
|
||||
deploy:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
- uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413 # v6.1.0
|
||||
- uses: actions/checkout@v7
|
||||
- uses: pnpm/action-setup@v6.0.9
|
||||
with:
|
||||
version: 10
|
||||
run_install: false
|
||||
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
|
||||
- uses: actions/setup-node@v7
|
||||
with:
|
||||
node-version: "24"
|
||||
cache: pnpm
|
||||
|
||||
@@ -1,49 +0,0 @@
|
||||
name: Deploy Web Studio
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [studio]
|
||||
paths:
|
||||
- "desktop/frontend-next/**"
|
||||
- ".github/workflows/deploy-web-studio.yml"
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: web-studio-production
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
deploy:
|
||||
runs-on: ubuntu-latest
|
||||
defaults:
|
||||
run:
|
||||
working-directory: desktop/frontend-next
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
|
||||
- uses: pnpm/action-setup@0ebf47130e4866e96fce0953f49152a61190b271 # v6.0.9
|
||||
with:
|
||||
version: 10
|
||||
run_install: false
|
||||
|
||||
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
|
||||
with:
|
||||
node-version: "24"
|
||||
cache: pnpm
|
||||
cache-dependency-path: desktop/frontend-next/pnpm-lock.yaml
|
||||
|
||||
- name: Install
|
||||
run: pnpm install --frozen-lockfile
|
||||
|
||||
- name: Build
|
||||
env:
|
||||
VITE_REMOTE_HOME: https://reasonix.io/remote/
|
||||
run: pnpm build
|
||||
|
||||
- name: Deploy
|
||||
env:
|
||||
CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
|
||||
CLOUDFLARE_ACCOUNT_ID: ${{ secrets.R2_ACCOUNT_ID }}
|
||||
run: npx --yes [email protected] deploy --config wrangler.jsonc
|
||||
@@ -0,0 +1,39 @@
|
||||
name: Documentation impact guard
|
||||
|
||||
on:
|
||||
pull_request_target:
|
||||
branches: [main-v2]
|
||||
types: [opened, synchronize, reopened, edited, ready_for_review]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: read
|
||||
|
||||
concurrency:
|
||||
group: docs-impact-${{ github.event.pull_request.number }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
docs-impact:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
# Execute only the trusted base-branch checker; never run contributor code
|
||||
# from a pull_request_target workflow.
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
ref: ${{ github.event.pull_request.base.sha }}
|
||||
|
||||
- name: Collect PR changed files
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
PR_NUMBER: ${{ github.event.pull_request.number }}
|
||||
REPOSITORY: ${{ github.repository }}
|
||||
run: |
|
||||
gh api --paginate "repos/${REPOSITORY}/pulls/${PR_NUMBER}/files" \
|
||||
--jq '.[].filename' > "$RUNNER_TEMP/docs-impact-files.txt"
|
||||
|
||||
- name: Check documentation impact declaration
|
||||
env:
|
||||
PR_BODY: ${{ github.event.pull_request.body }}
|
||||
DOCS_IMPACT_CHANGED_FILES_FILE: ${{ runner.temp }}/docs-impact-files.txt
|
||||
run: bash scripts/check-docs-impact.sh
|
||||
@@ -32,7 +32,7 @@ jobs:
|
||||
environment: e2e-bot
|
||||
steps:
|
||||
- name: Acknowledge
|
||||
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9
|
||||
uses: actions/github-script@v9
|
||||
with:
|
||||
script: |
|
||||
await github.rest.reactions.createForIssueComment({
|
||||
@@ -42,16 +42,16 @@ jobs:
|
||||
|
||||
# Default-branch checkout: this is where the harness (cmd/e2ebench), the
|
||||
# suite, and a run --metrics-capable agent live.
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7
|
||||
- uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: go.mod
|
||||
cache: true
|
||||
|
||||
- uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7
|
||||
- uses: actions/setup-python@v7
|
||||
with:
|
||||
python-version: '3.12'
|
||||
|
||||
|
||||
@@ -20,7 +20,7 @@ jobs:
|
||||
classify:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9
|
||||
- uses: actions/github-script@v9
|
||||
env:
|
||||
DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }}
|
||||
with:
|
||||
@@ -35,7 +35,7 @@ jobs:
|
||||
const body = (issue.body || '').slice(0, 4000);
|
||||
|
||||
const system = [
|
||||
'You categorize GitHub issues for Reasonix, a Go-based AI coding agent with an Electron desktop app and a terminal UI.',
|
||||
'You categorize GitHub issues for Reasonix, a Go-based AI coding agent with a Wails desktop app and a terminal UI.',
|
||||
'Pick labels ONLY from these fixed sets. Never invent labels.',
|
||||
'area (0-2, the affected subsystem):',
|
||||
' agent: core agent loop / tool-calling / reasoning',
|
||||
@@ -43,7 +43,7 @@ jobs:
|
||||
' config: configuration, setup wizard, .toml/.env',
|
||||
' updater: auto-update, installer, release packaging',
|
||||
' provider: model providers, model selection/switching',
|
||||
' desktop: Electron desktop GUI',
|
||||
' desktop: Wails desktop GUI',
|
||||
' tui: terminal UI / CLI',
|
||||
' skills: skills system',
|
||||
' rendering: terminal rendering / flicker / repaint',
|
||||
|
||||
@@ -1,116 +0,0 @@
|
||||
name: Issue replies
|
||||
|
||||
# Posts maintainer replies as github-actions[bot], so a batch of triage answers
|
||||
# spends the repository's quota instead of a person's content-creation limit.
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
replies:
|
||||
description: 'JSON array of {"issue": 123, "body": "...", "close": true | "not_planned"} or {"discussion": 123, "body": "...", "close": "resolved" | "outdated" | "duplicate"}'
|
||||
required: true
|
||||
type: string
|
||||
interval:
|
||||
description: Seconds between entries
|
||||
required: false
|
||||
default: '20'
|
||||
type: string
|
||||
|
||||
permissions:
|
||||
issues: write
|
||||
pull-requests: write
|
||||
discussions: write
|
||||
|
||||
concurrency:
|
||||
group: issue-replies
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
reply:
|
||||
runs-on: win-gpu
|
||||
timeout-minutes: 180
|
||||
steps:
|
||||
- uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9
|
||||
env:
|
||||
REPLIES: ${{ inputs.replies }}
|
||||
INTERVAL: ${{ inputs.interval }}
|
||||
with:
|
||||
script: |
|
||||
const replies = JSON.parse(process.env.REPLIES);
|
||||
if (!Array.isArray(replies)) throw new Error('replies must be a JSON array');
|
||||
const { owner, repo } = context.repo;
|
||||
const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
||||
const failed = [];
|
||||
// Only a rate limit is worth waiting out; any other refusal is final.
|
||||
const rateLimited = (error) => {
|
||||
const headers = error.response?.headers ?? {};
|
||||
return error.status === 429 ||
|
||||
(error.status === 403 && (headers['retry-after'] !== undefined || headers['x-ratelimit-remaining'] === '0'));
|
||||
};
|
||||
// Discussions have no REST comment API; they are addressed by node id.
|
||||
const replyToDiscussion = async (number, body) => {
|
||||
const q = await github.graphql(
|
||||
`query($owner:String!,$repo:String!,$n:Int!){repository(owner:$owner,name:$repo){discussion(number:$n){id comments(last:100){nodes{body author{login}}}}}}`,
|
||||
{ owner, repo, n: number });
|
||||
const d = q.repository.discussion;
|
||||
if (!d) throw new Error('no such discussion');
|
||||
if (d.comments.nodes.some((c) => c.author?.login === 'github-actions' && c.body === body)) return false;
|
||||
await github.graphql(`mutation($id:ID!,$body:String!){addDiscussionComment(input:{discussionId:$id,body:$body}){comment{id}}}`, { id: d.id, body });
|
||||
return true;
|
||||
};
|
||||
const closeReasons = { resolved: 'RESOLVED', outdated: 'OUTDATED', duplicate: 'DUPLICATE' };
|
||||
const closeDiscussion = async (number, reason) => {
|
||||
const q = await github.graphql(
|
||||
`query($owner:String!,$repo:String!,$n:Int!){repository(owner:$owner,name:$repo){discussion(number:$n){id closed}}}`,
|
||||
{ owner, repo, n: number });
|
||||
const d = q.repository.discussion;
|
||||
if (d.closed) return;
|
||||
await github.graphql(`mutation($id:ID!,$r:DiscussionCloseReason!){closeDiscussion(input:{discussionId:$id,reason:$r}){discussion{id}}}`, { id: d.id, r: reason });
|
||||
};
|
||||
for (const { issue, discussion, body, close } of replies) {
|
||||
if (Number.isInteger(discussion) && typeof body === 'string' && body.trim() !== '') {
|
||||
try {
|
||||
const posted = await replyToDiscussion(discussion, body);
|
||||
if (close !== undefined && close !== false) {
|
||||
const reason = closeReasons[close];
|
||||
if (!reason) throw new Error(`unknown close reason ${close}`);
|
||||
await closeDiscussion(discussion, reason);
|
||||
}
|
||||
core.info(`discussion #${discussion}: ${posted ? 'replied' : 'already replied'}${close ? ', closed' : ''}`);
|
||||
} catch (error) {
|
||||
failed.push(`discussion ${discussion}: ${error.message}`);
|
||||
}
|
||||
await sleep(Math.max(1, Number(process.env.INTERVAL) || 20) * 1000);
|
||||
continue;
|
||||
}
|
||||
if (!Number.isInteger(issue) || typeof body !== 'string' || body.trim() === '') {
|
||||
failed.push(`${issue}: malformed entry`);
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
const comments = await github.paginate(github.rest.issues.listComments, { owner, repo, issue_number: issue, per_page: 100 });
|
||||
const posted = comments.some((c) => c.user?.type === 'Bot' && c.body === body);
|
||||
if (!posted) {
|
||||
for (let attempt = 0; ; attempt++) {
|
||||
try {
|
||||
await github.rest.issues.createComment({ owner, repo, issue_number: issue, body });
|
||||
break;
|
||||
} catch (error) {
|
||||
if (attempt >= 5 || !rateLimited(error)) throw error;
|
||||
core.warning(`#${issue}: ${error.message}; retrying in 2 minutes`);
|
||||
await sleep(120_000);
|
||||
}
|
||||
}
|
||||
}
|
||||
if (close === true || close === 'not_planned') {
|
||||
const { data } = await github.rest.issues.get({ owner, repo, issue_number: issue });
|
||||
if (data.state === 'open') {
|
||||
await github.rest.issues.update({ owner, repo, issue_number: issue, state: 'closed', state_reason: close === 'not_planned' ? 'not_planned' : 'completed' });
|
||||
}
|
||||
}
|
||||
core.info(`#${issue}: ${posted ? 'already replied' : 'replied'}${close ? ', closed' : ''}`);
|
||||
} catch (error) {
|
||||
failed.push(`${issue}: ${error.message}`);
|
||||
}
|
||||
await sleep(Math.max(1, Number(process.env.INTERVAL) || 20) * 1000);
|
||||
}
|
||||
if (failed.length) core.setFailed(`not done:\n${failed.join('\n')}`);
|
||||
@@ -2,8 +2,7 @@ name: Label issue version
|
||||
|
||||
# Issue forms can't let a reporter set a label directly (that needs triage rights),
|
||||
# so the bug/feature forms carry a "Version line" dropdown and this workflow reads
|
||||
# the submitted value and applies the matching label on their behalf:
|
||||
# 0.x -> v1 (TypeScript), 1.x -> v2 (main-v2), 2.x -> v3 (studio).
|
||||
# the submitted value and applies the matching v1/v2 label on their behalf.
|
||||
on:
|
||||
issues:
|
||||
types: [opened, edited]
|
||||
@@ -19,46 +18,46 @@ jobs:
|
||||
label:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9
|
||||
- uses: actions/github-script@v9
|
||||
with:
|
||||
script: |
|
||||
const body = context.payload.issue.body || '';
|
||||
const sectionOf = (name) =>
|
||||
body.split(/^###\s+/m).find((s) => new RegExp(`^${name}`, 'i').test(s)) || '';
|
||||
const labelOf = (major) => ({ 0: 'v1', 1: 'v2', 2: 'v3' })[major];
|
||||
|
||||
const m = sectionOf('Version line').match(/\b([012])\.x\b/);
|
||||
const m = sectionOf('Version line').match(/\bv([12])\b/i);
|
||||
if (!m) {
|
||||
core.info('No version line found; nothing to label.');
|
||||
return;
|
||||
}
|
||||
let choice = labelOf(Number(m[1]));
|
||||
let choice = 'v' + m[1];
|
||||
|
||||
// The dropdown is a choice, and its first entry gets picked by
|
||||
// reporters who never looked. `Exact version` is pasted from
|
||||
// `reasonix --version` or Studio, so when it parses it is the better
|
||||
// evidence. Anything that does not parse (a commit sha, a branch
|
||||
// name) or names no known line leaves the dropdown to decide.
|
||||
// The dropdown is a choice, and v2 sits first, so it gets picked by
|
||||
// reporters who are actually on 0.x. `Exact version` is pasted from
|
||||
// `reasonix --version`, so when it parses it is the better evidence:
|
||||
// 0.x is the TypeScript line, 1.x the Go rewrite. Anything that does
|
||||
// not parse (a commit sha, "main-v2") leaves the dropdown to decide.
|
||||
const exact = sectionOf('Exact version').match(/\b(\d+)\.(\d+)\.(\d+)/);
|
||||
const fromVersion = exact && labelOf(Number(exact[1]));
|
||||
if (fromVersion && fromVersion !== choice) {
|
||||
core.info(`Version line says ${choice} but ${exact[0]} says ${fromVersion}; trusting the version.`);
|
||||
choice = fromVersion;
|
||||
if (exact) {
|
||||
const fromVersion = Number(exact[1]) === 0 ? 'v1' : 'v2';
|
||||
if (fromVersion !== choice) {
|
||||
core.info(`Version line says ${choice} but ${exact[0]} says ${fromVersion}; trusting the version.`);
|
||||
choice = fromVersion;
|
||||
}
|
||||
}
|
||||
const other = choice === 'v2' ? 'v1' : 'v2';
|
||||
await github.rest.issues.addLabels({
|
||||
...context.repo,
|
||||
issue_number: context.issue.number,
|
||||
labels: [choice],
|
||||
});
|
||||
// Keep the line labels mutually exclusive in case an edit flipped the choice.
|
||||
for (const other of ['v1', 'v2', 'v3'].filter((l) => l !== choice)) {
|
||||
try {
|
||||
await github.rest.issues.removeLabel({
|
||||
...context.repo,
|
||||
issue_number: context.issue.number,
|
||||
name: other,
|
||||
});
|
||||
} catch (e) {
|
||||
core.info(`No ${other} label to remove.`);
|
||||
}
|
||||
// Keep v1/v2 mutually exclusive in case an edit flipped the choice.
|
||||
try {
|
||||
await github.rest.issues.removeLabel({
|
||||
...context.repo,
|
||||
issue_number: context.issue.number,
|
||||
name: other,
|
||||
});
|
||||
} catch (e) {
|
||||
core.info(`No ${other} label to remove.`);
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
name: Deploy site
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main-v2]
|
||||
paths: ['site/**', 'release-notes/**', '.github/workflows/pages.yml']
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
pages: write
|
||||
id-token: write
|
||||
|
||||
concurrency:
|
||||
group: pages
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
build:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
- uses: actions/setup-node@v7
|
||||
with:
|
||||
node-version: '22'
|
||||
cache: npm
|
||||
cache-dependency-path: site/package-lock.json
|
||||
- name: Build
|
||||
working-directory: site
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: |
|
||||
npm ci
|
||||
npm run build
|
||||
- uses: actions/upload-pages-artifact@v5
|
||||
with:
|
||||
path: site/dist
|
||||
|
||||
deploy:
|
||||
needs: build
|
||||
runs-on: ubuntu-latest
|
||||
environment:
|
||||
name: github-pages
|
||||
url: ${{ steps.deploy.outputs.page_url }}
|
||||
steps:
|
||||
- id: deploy
|
||||
uses: actions/deploy-pages@v5
|
||||
@@ -23,6 +23,6 @@ jobs:
|
||||
label:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/labeler@bf12e9b00b37c5c0ca2b87b79b2daf7891dbda13 # v7
|
||||
- uses: actions/labeler@v7
|
||||
with:
|
||||
sync-labels: false
|
||||
@@ -20,7 +20,7 @@ jobs:
|
||||
label:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9
|
||||
- uses: actions/github-script@v9
|
||||
with:
|
||||
script: |
|
||||
const base = context.payload.pull_request.base.ref;
|
||||
|
||||
@@ -0,0 +1,101 @@
|
||||
name: Prepare release
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
version:
|
||||
description: "Official version without a tag prefix (for example 1.20.0)"
|
||||
required: true
|
||||
type: string
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
pull-requests: write
|
||||
|
||||
jobs:
|
||||
draft:
|
||||
name: generate reviewable bilingual draft
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- uses: actions/setup-node@v7
|
||||
with:
|
||||
node-version: "22"
|
||||
|
||||
- name: Select the review branch
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
VERSION: ${{ inputs.version }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
git config user.name "github-actions[bot]"
|
||||
git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
|
||||
git fetch origin main-v2
|
||||
|
||||
if [[ ! "$VERSION" =~ ^(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)$ ]]; then
|
||||
echo "::error::version must be MAJOR.MINOR.PATCH"
|
||||
exit 1
|
||||
fi
|
||||
branch="release-notes/v${VERSION}"
|
||||
if git ls-remote --exit-code --heads origin "$branch" >/dev/null 2>&1; then
|
||||
git fetch origin "refs/heads/$branch:refs/remotes/origin/$branch"
|
||||
git checkout -b "$branch" --track "origin/$branch"
|
||||
git merge --no-edit origin/main-v2
|
||||
else
|
||||
git checkout -b "$branch"
|
||||
fi
|
||||
echo "RELEASE_NOTES_BRANCH=$branch" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Generate and validate release notes
|
||||
env:
|
||||
DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }}
|
||||
DEEPSEEK_MODEL: deepseek-v4-pro
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
VERSION: ${{ inputs.version }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
node scripts/generate-release-notes.mjs --version "$VERSION"
|
||||
node scripts/release-notes.mjs validate
|
||||
node --test scripts/release-notes.test.mjs
|
||||
node scripts/release-notes.mjs render --version "$VERSION" --output /tmp/release-notes.md
|
||||
{
|
||||
echo "## Review draft for v${VERSION#v}"
|
||||
echo
|
||||
cat /tmp/release-notes.md
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
- name: Open or update the review PR
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
VERSION: ${{ inputs.version }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
git add release-notes/releases.json
|
||||
if ! git diff --cached --quiet; then
|
||||
git commit -m "docs(release): prepare v${VERSION#v} notes" \
|
||||
-m "Summary:
|
||||
Generate a bilingual, product-focused draft from merged pull request metadata. Reuse the selected release-bound PR when one is available.
|
||||
|
||||
Verification:
|
||||
Validate the catalog, citations, bilingual fields, and rendered GitHub release notes before committing."
|
||||
fi
|
||||
git push -u origin "$RELEASE_NOTES_BRANCH"
|
||||
if ! gh pr list --head "$RELEASE_NOTES_BRANCH" --state open --json number --jq 'length > 0' | grep -q true; then
|
||||
if ! gh pr create \
|
||||
--base main-v2 \
|
||||
--head "$RELEASE_NOTES_BRANCH" \
|
||||
--title "docs(release): 准备 v${VERSION#v} 中英更新日志" \
|
||||
--body-file /tmp/release-notes.md; then
|
||||
echo "::warning::GitHub Actions could not open the PR; the reviewed branch is preserved."
|
||||
{
|
||||
echo "## Manual PR handoff"
|
||||
echo
|
||||
echo '```sh'
|
||||
echo "gh pr create --repo ${{ github.repository }} --base main-v2 --head $RELEASE_NOTES_BRANCH --fill"
|
||||
echo '```'
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
fi
|
||||
fi
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,26 @@
|
||||
name: Release notes
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- "release-notes/**"
|
||||
- "scripts/release-notes*.mjs"
|
||||
- "site/src/pages/changelog/**"
|
||||
- "site/src/components/ChangelogPage.astro"
|
||||
- "desktop/frontend/src/App.tsx"
|
||||
- "desktop/frontend/src/components/SettingsPanel.tsx"
|
||||
- "desktop/frontend/src/components/UpdateBanner.tsx"
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
validate:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
- uses: actions/setup-node@v7
|
||||
with:
|
||||
node-version: "22"
|
||||
- run: node scripts/release-notes.mjs validate
|
||||
- run: node --test scripts/release-notes.test.mjs
|
||||
@@ -0,0 +1,206 @@
|
||||
name: Release npm
|
||||
|
||||
# npm line. Stable releases are called by release-stable.yml after its single
|
||||
# GitHub environment approval. Direct prerelease tag publication is disabled;
|
||||
# manual workflow_dispatch recovers only an approved Stable version from an
|
||||
# existing tag. Historical canary and next package identities remain readable,
|
||||
# but normal publication advances only the official release.
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
channel:
|
||||
description: "Standalone npm recovery channel"
|
||||
required: true
|
||||
default: stable
|
||||
type: choice
|
||||
options:
|
||||
- stable
|
||||
base_version:
|
||||
description: "Stable version to recover exactly"
|
||||
required: true
|
||||
type: string
|
||||
tag:
|
||||
description: "stable: existing npm tag to publish (for example npm-v1.18.0)"
|
||||
required: false
|
||||
type: string
|
||||
workflow_call:
|
||||
inputs:
|
||||
channel:
|
||||
description: "Publish channel selected by the approved release orchestrator"
|
||||
required: true
|
||||
type: string
|
||||
base_version:
|
||||
description: "Version to publish"
|
||||
required: true
|
||||
type: string
|
||||
tag:
|
||||
description: "Existing npm tag to check out for stable publication"
|
||||
required: false
|
||||
default: ""
|
||||
type: string
|
||||
approved_cli_tag:
|
||||
description: "Stable CLI tag recorded by the approved orchestrator"
|
||||
required: true
|
||||
type: string
|
||||
approved_sha:
|
||||
description: "Immutable commit recorded by the approved orchestrator"
|
||||
required: true
|
||||
type: string
|
||||
orchestrated:
|
||||
description: "True only when called by an approved release orchestrator"
|
||||
required: false
|
||||
default: false
|
||||
type: boolean
|
||||
orchestrator:
|
||||
description: "Trusted release orchestrator (legacy Preview calls remain readable)"
|
||||
required: false
|
||||
default: stable
|
||||
type: string
|
||||
preview_number:
|
||||
description: "Legacy Preview ordinal for old workflow-call compatibility"
|
||||
required: false
|
||||
default: ""
|
||||
type: string
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
# Serialize every publisher for one npm dist-tag. Historical canary calls
|
||||
# remain ordered even though normal publication now uses Stable only.
|
||||
group: release-npm-${{ inputs.channel || 'next' }}
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
orchestration-guard:
|
||||
name: verify approved orchestrator
|
||||
if: ${{ inputs.orchestrated }}
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
fetch-depth: 0
|
||||
ref: ${{ github.sha }}
|
||||
- name: Verify caller and approved release ref
|
||||
env:
|
||||
ACTUAL_CALLER_WORKFLOW_REF: ${{ github.workflow_ref }}
|
||||
EXPECTED_CALLER_WORKFLOW_REF: ${{ format('{0}/.github/workflows/release-{1}.yml@{2}', github.repository, inputs.orchestrator, github.ref) }}
|
||||
CALLER_EVENT_NAME: ${{ github.event_name }}
|
||||
CALLER_REF: ${{ github.ref }}
|
||||
CALLER_REF_PROTECTED: ${{ github.ref_protected }}
|
||||
CALLER_WORKFLOW_SHA: ${{ github.workflow_sha }}
|
||||
CALLER_SHA: ${{ github.sha }}
|
||||
APPROVED_CLI_TAG: ${{ inputs.approved_cli_tag }}
|
||||
APPROVED_SHA: ${{ inputs.approved_sha }}
|
||||
APPROVED_CHANNEL: ${{ inputs.orchestrator }}
|
||||
RELEASE_TAG: ${{ inputs.approved_cli_tag }}
|
||||
VERIFY_RELEASE_CHECKOUT: false
|
||||
run: |
|
||||
bash scripts/verify-release-authorization.sh
|
||||
bash scripts/verify-release-tag.sh
|
||||
|
||||
release-gate:
|
||||
name: approve standalone npm release
|
||||
if: ${{ !inputs.orchestrated }}
|
||||
runs-on: ubuntu-latest
|
||||
environment: release
|
||||
steps:
|
||||
- env:
|
||||
RELEASE_TAG: ${{ inputs.tag || github.ref_name }}
|
||||
run: echo "Approved standalone npm release $RELEASE_TAG"
|
||||
|
||||
cache-guard:
|
||||
name: cache hit guard
|
||||
needs: [orchestration-guard, release-gate]
|
||||
if: ${{ always() && !cancelled() && ((inputs.orchestrated && needs.orchestration-guard.result == 'success') || (!inputs.orchestrated && needs.release-gate.result == 'success')) }}
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
ref: ${{ inputs.approved_sha || inputs.tag || github.ref }}
|
||||
- uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: go.mod
|
||||
cache: true
|
||||
- run: ./scripts/cache-guard.sh
|
||||
- name: Verify embedded documentation identity
|
||||
env:
|
||||
DOCS_BUILD_VERSION: v${{ inputs.base_version }}
|
||||
DOCS_SOURCE_REVISION: ${{ inputs.approved_sha }}
|
||||
run: |
|
||||
if [ ! -f scripts/verify-embedded-docs.sh ]; then
|
||||
echo "Legacy candidate predates the embedded docs contract; skipping."
|
||||
exit 0
|
||||
fi
|
||||
revision="$DOCS_SOURCE_REVISION"
|
||||
if [ -z "$revision" ]; then revision="$(git rev-parse HEAD)"; fi
|
||||
bash scripts/verify-embedded-docs.sh "$DOCS_BUILD_VERSION" "$revision"
|
||||
|
||||
npm:
|
||||
name: publish npm packages
|
||||
needs: cache-guard
|
||||
if: ${{ always() && !cancelled() && needs.cache-guard.result == 'success' }}
|
||||
runs-on: ubuntu-latest
|
||||
# Orchestrated releases have already passed their GitHub environment
|
||||
# approval. Direct prereleases and manual Stable recovery pass release-gate.
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
ref: ${{ inputs.approved_sha || inputs.tag || github.ref }}
|
||||
- name: Load approved standalone recovery control plane
|
||||
if: ${{ !inputs.orchestrated }}
|
||||
env:
|
||||
RECOVERY_CONTROL_SHA: ${{ github.workflow_sha }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
git fetch --no-tags --depth=1 origin "$RECOVERY_CONTROL_SHA"
|
||||
git restore --source="$RECOVERY_CONTROL_SHA" -- \
|
||||
npm/publish.mjs \
|
||||
scripts/finalize-npm-official-release.mjs
|
||||
- uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: go.mod
|
||||
cache: true
|
||||
- uses: actions/setup-node@v7
|
||||
with:
|
||||
node-version: '22'
|
||||
registry-url: 'https://registry.npmjs.org'
|
||||
# Stable publication uses the exact npm-v* tag. build.mjs strips the
|
||||
# leading `npm-`/`v`; historical Preview inputs remain only for old
|
||||
# workflow-call compatibility and are not reachable from a public entry.
|
||||
- name: Resolve version
|
||||
id: ver
|
||||
env:
|
||||
EVENT_NAME: ${{ github.event_name }}
|
||||
IN_ORCHESTRATED: ${{ inputs.orchestrated }}
|
||||
IN_CHANNEL: ${{ inputs.channel }}
|
||||
IN_BASE_VERSION: ${{ inputs.base_version }}
|
||||
IN_TAG: ${{ inputs.tag }}
|
||||
REF_NAME: ${{ github.ref_name }}
|
||||
RUN_NUMBER: ${{ github.run_number }}
|
||||
IN_PREVIEW_NUMBER: ${{ inputs.preview_number }}
|
||||
run: bash scripts/resolve-npm-release.sh
|
||||
- name: Revalidate approved release ref
|
||||
if: ${{ inputs.orchestrated }}
|
||||
env:
|
||||
RELEASE_TAG: ${{ inputs.approved_cli_tag }}
|
||||
APPROVED_SHA: ${{ inputs.approved_sha }}
|
||||
run: bash scripts/verify-release-tag.sh
|
||||
- name: Publish or recover immutable npm packages
|
||||
env:
|
||||
VERSION_ARG: ${{ steps.ver.outputs.arg }}
|
||||
NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
|
||||
run: node npm/build.mjs "$VERSION_ARG" --publish
|
||||
- name: Align legacy aliases with the official release
|
||||
if: ${{ inputs.channel == 'stable' }}
|
||||
env:
|
||||
VERSION: ${{ inputs.base_version }}
|
||||
APPROVED_SHA: ${{ inputs.approved_sha }}
|
||||
NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
candidate="$APPROVED_SHA"
|
||||
if [ -z "$candidate" ]; then candidate="$(git rev-parse HEAD)"; fi
|
||||
EXPECTED_SHA="$candidate" node scripts/finalize-npm-official-release.mjs "$VERSION"
|
||||
@@ -0,0 +1,42 @@
|
||||
name: Relay stable release
|
||||
run-name: Relay stable release ${{ github.ref_name }}
|
||||
|
||||
# The tag event itself has a tag-shaped origin. Relay the immutable tag name to
|
||||
# release-stable.yml on protected main-v2 so the production SignPath policy can
|
||||
# allow exactly one branch name without trusting wildcard tag-like branches.
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
- 'v*'
|
||||
- '!v*-*'
|
||||
|
||||
permissions:
|
||||
actions: write
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
relay:
|
||||
name: dispatch protected stable control plane
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Dispatch release-stable.yml on main-v2
|
||||
uses: actions/github-script@v9
|
||||
env:
|
||||
RELEASE_TAG: ${{ github.ref_name }}
|
||||
CONTROL_PLANE_REF: ${{ github.event.repository.default_branch }}
|
||||
with:
|
||||
script: |
|
||||
if (process.env.CONTROL_PLANE_REF !== 'main-v2') {
|
||||
core.setFailed(`Unexpected default branch: ${process.env.CONTROL_PLANE_REF}`);
|
||||
return;
|
||||
}
|
||||
await github.rest.actions.createWorkflowDispatch({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
workflow_id: 'release-stable.yml',
|
||||
ref: process.env.CONTROL_PLANE_REF,
|
||||
inputs: {
|
||||
tag: process.env.RELEASE_TAG,
|
||||
allow_recovery: 'false',
|
||||
},
|
||||
});
|
||||
@@ -0,0 +1,278 @@
|
||||
name: Release stable
|
||||
run-name: Release stable ${{ inputs.tag || github.ref_name }}
|
||||
|
||||
# Stable publication has exactly one GitHub environment gate. Push the CLI,
|
||||
# npm, and desktop tags atomically; release-stable-trigger.yml relays the vX.Y.Z
|
||||
# tag to this workflow on protected main-v2. Preflight verifies that all three
|
||||
# tags point to the reviewed Notes candidate on main-v2 history, and one GitHub
|
||||
# approval releases every surface. After approval, a zero-publication Windows
|
||||
# preflight verifies both architectures and signing stages before publication.
|
||||
# Keeping the control-plane ref on main-v2 lets SignPath restrict
|
||||
# production signing to that one protected origin instead of trusting wildcard
|
||||
# tag-like branch names. Manual recovery uses the same fixed control plane while
|
||||
# preserving an older tagged candidate on main-v2 history.
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
tag:
|
||||
description: "Existing stable CLI tag to recover (for example v1.18.0)"
|
||||
required: true
|
||||
type: string
|
||||
publish_cli:
|
||||
description: "Recover the CLI/Homebrew channel"
|
||||
required: false
|
||||
default: true
|
||||
type: boolean
|
||||
publish_npm:
|
||||
description: "Recover the npm channel"
|
||||
required: false
|
||||
default: true
|
||||
type: boolean
|
||||
publish_desktop:
|
||||
description: "Recover the Desktop/R2 channel"
|
||||
required: false
|
||||
default: true
|
||||
type: boolean
|
||||
allow_recovery:
|
||||
description: "Allow an existing stable tag on main-v2 history (manual recovery only)"
|
||||
required: false
|
||||
default: true
|
||||
type: boolean
|
||||
|
||||
concurrency:
|
||||
group: stable-release-${{ inputs.tag || github.ref_name }}
|
||||
cancel-in-progress: false
|
||||
|
||||
# Reusable release workflows can only reduce caller permissions, so the
|
||||
# orchestrator grants the union needed by CLI/Desktop publication.
|
||||
permissions:
|
||||
actions: write
|
||||
contents: write
|
||||
|
||||
jobs:
|
||||
preflight:
|
||||
name: validate stable release set
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
actions: read
|
||||
contents: read
|
||||
outputs:
|
||||
version: ${{ steps.release.outputs.version }}
|
||||
cli_tag: ${{ steps.release.outputs.cli_tag }}
|
||||
npm_tag: ${{ steps.release.outputs.npm_tag }}
|
||||
desktop_tag: ${{ steps.release.outputs.desktop_tag }}
|
||||
sha: ${{ steps.release.outputs.sha }}
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
fetch-depth: 0
|
||||
# A recovery dispatch uses the fixed workflow/scripts from protected
|
||||
# main-v2. Publishers still check out the immutable approved tag SHA.
|
||||
ref: ${{ github.sha }}
|
||||
- uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: go.mod
|
||||
cache: true
|
||||
- uses: actions/setup-node@v7
|
||||
with:
|
||||
node-version: "22"
|
||||
- name: Resolve stable release
|
||||
id: release
|
||||
env:
|
||||
RELEASE_TAG: ${{ inputs.tag || github.ref_name }}
|
||||
ALLOW_STABLE_RECOVERY: ${{ inputs.allow_recovery }}
|
||||
run: bash scripts/resolve-stable-release.sh
|
||||
- name: Revalidate normal release candidate and exact push CI
|
||||
if: ${{ !inputs.allow_recovery }}
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
RELEASE_REPOSITORY: ${{ github.repository }}
|
||||
RELEASE_CI_WAIT_SECONDS: 60
|
||||
RELEASE_VERSION: ${{ steps.release.outputs.version }}
|
||||
RELEASE_SHA: ${{ steps.release.outputs.sha }}
|
||||
run: |
|
||||
bash scripts/validate-stable-candidate.sh "$RELEASE_VERSION" "$RELEASE_SHA"
|
||||
bash scripts/verify-release-push-ci.sh "$RELEASE_SHA"
|
||||
- name: Validate reviewed release notes
|
||||
run: node scripts/release-notes.mjs render --version "${{ steps.release.outputs.cli_tag }}" --output /tmp/release-notes.md
|
||||
# Recovery builds deliberately check out the immutable tagged candidate,
|
||||
# which can predate its reviewed release-note entry. Carry the exact file
|
||||
# validated by this protected control-plane job into both publishers.
|
||||
- name: Upload reviewed release notes
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: orchestrator-reviewed-release-notes
|
||||
path: /tmp/release-notes.md
|
||||
if-no-files-found: error
|
||||
retention-days: 1
|
||||
- name: Cache hit guard
|
||||
run: ./scripts/cache-guard.sh
|
||||
|
||||
authorize:
|
||||
name: approve stable release
|
||||
needs: preflight
|
||||
runs-on: ubuntu-latest
|
||||
environment: release
|
||||
permissions:
|
||||
contents: read
|
||||
outputs:
|
||||
version: ${{ steps.approved.outputs.version }}
|
||||
cli_tag: ${{ steps.approved.outputs.cli_tag }}
|
||||
npm_tag: ${{ steps.approved.outputs.npm_tag }}
|
||||
desktop_tag: ${{ steps.approved.outputs.desktop_tag }}
|
||||
sha: ${{ steps.approved.outputs.sha }}
|
||||
steps:
|
||||
- name: Record approved release
|
||||
id: approved
|
||||
env:
|
||||
VERSION: ${{ needs.preflight.outputs.version }}
|
||||
CLI_TAG: ${{ needs.preflight.outputs.cli_tag }}
|
||||
NPM_TAG: ${{ needs.preflight.outputs.npm_tag }}
|
||||
DESKTOP_TAG: ${{ needs.preflight.outputs.desktop_tag }}
|
||||
RELEASE_SHA: ${{ needs.preflight.outputs.sha }}
|
||||
run: |
|
||||
{
|
||||
echo "version=$VERSION"
|
||||
echo "cli_tag=$CLI_TAG"
|
||||
echo "npm_tag=$NPM_TAG"
|
||||
echo "desktop_tag=$DESKTOP_TAG"
|
||||
echo "sha=$RELEASE_SHA"
|
||||
} >> "$GITHUB_OUTPUT"
|
||||
echo "Approved stable release $VERSION at $RELEASE_SHA"
|
||||
|
||||
cli:
|
||||
name: publish CLI and Homebrew
|
||||
needs: [authorize, signpath-preflight]
|
||||
if: ${{ always() && !cancelled() && needs.authorize.result == 'success' && (needs.signpath-preflight.result == 'success' || needs.signpath-preflight.result == 'skipped') && (github.event_name != 'workflow_dispatch' || inputs.publish_cli) }}
|
||||
uses: ./.github/workflows/release.yml
|
||||
with:
|
||||
tag: ${{ needs.authorize.outputs.cli_tag }}
|
||||
approved_cli_tag: ${{ needs.authorize.outputs.cli_tag }}
|
||||
approved_sha: ${{ needs.authorize.outputs.sha }}
|
||||
orchestrated: true
|
||||
secrets: inherit
|
||||
|
||||
npm:
|
||||
name: publish npm
|
||||
needs: [authorize, signpath-preflight]
|
||||
if: ${{ always() && !cancelled() && needs.authorize.result == 'success' && (needs.signpath-preflight.result == 'success' || needs.signpath-preflight.result == 'skipped') && (github.event_name != 'workflow_dispatch' || inputs.publish_npm) }}
|
||||
uses: ./.github/workflows/release-npm.yml
|
||||
with:
|
||||
channel: stable
|
||||
base_version: ${{ needs.authorize.outputs.version }}
|
||||
tag: ${{ needs.authorize.outputs.npm_tag }}
|
||||
approved_cli_tag: ${{ needs.authorize.outputs.cli_tag }}
|
||||
approved_sha: ${{ needs.authorize.outputs.sha }}
|
||||
orchestrated: true
|
||||
secrets: inherit
|
||||
|
||||
signpath-preflight:
|
||||
name: verify stable SignPath control plane
|
||||
needs: authorize
|
||||
if: ${{ github.event_name != 'workflow_dispatch' || inputs.publish_desktop }}
|
||||
uses: ./.github/workflows/release-desktop.yml
|
||||
with:
|
||||
channel: stable
|
||||
tag: ${{ needs.authorize.outputs.desktop_tag }}
|
||||
approved_cli_tag: ${{ needs.authorize.outputs.cli_tag }}
|
||||
approved_sha: ${{ needs.authorize.outputs.sha }}
|
||||
orchestrated: true
|
||||
signing_preflight: true
|
||||
secrets: inherit
|
||||
|
||||
desktop:
|
||||
name: publish desktop
|
||||
needs: [authorize, signpath-preflight]
|
||||
if: ${{ always() && !cancelled() && needs.authorize.result == 'success' && needs.signpath-preflight.result == 'success' && (github.event_name != 'workflow_dispatch' || inputs.publish_desktop) }}
|
||||
uses: ./.github/workflows/release-desktop.yml
|
||||
with:
|
||||
channel: stable
|
||||
tag: ${{ needs.authorize.outputs.desktop_tag }}
|
||||
approved_cli_tag: ${{ needs.authorize.outputs.cli_tag }}
|
||||
approved_sha: ${{ needs.authorize.outputs.sha }}
|
||||
orchestrated: true
|
||||
signing_preflight_verified: true
|
||||
secrets: inherit
|
||||
|
||||
postflight:
|
||||
name: verify stable release artifacts
|
||||
needs: [authorize, cli, npm, desktop]
|
||||
if: ${{ always() && !cancelled() }}
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
actions: write
|
||||
contents: write
|
||||
steps:
|
||||
- name: Require every publisher to succeed
|
||||
env:
|
||||
PUBLISH_CLI: ${{ github.event_name != 'workflow_dispatch' || inputs.publish_cli }}
|
||||
PUBLISH_NPM: ${{ github.event_name != 'workflow_dispatch' || inputs.publish_npm }}
|
||||
PUBLISH_DESKTOP: ${{ github.event_name != 'workflow_dispatch' || inputs.publish_desktop }}
|
||||
CLI_RESULT: ${{ needs.cli.result }}
|
||||
NPM_RESULT: ${{ needs.npm.result }}
|
||||
DESKTOP_RESULT: ${{ needs.desktop.result }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
for channel in cli npm desktop; do
|
||||
selected_var="PUBLISH_${channel^^}"
|
||||
result_var="${channel^^}_RESULT"
|
||||
selected="${!selected_var}"
|
||||
result="${!result_var}"
|
||||
if [ "$selected" != "true" ]; then
|
||||
echo "$channel recovery skipped; public postflight will still verify it"
|
||||
continue
|
||||
fi
|
||||
if [ "$result" != "success" ]; then
|
||||
echo "::error::$channel stable publisher result is $result, expected success"
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
# Postflight belongs to the trusted control plane, not the old build
|
||||
# candidate, which may predate this verifier.
|
||||
ref: ${{ github.sha }}
|
||||
- uses: actions/setup-node@v7
|
||||
with:
|
||||
node-version: "22"
|
||||
- name: Verify public artifacts and npm latest
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
RELEASE_REPOSITORY: ${{ github.repository }}
|
||||
RELEASE_VERSION: ${{ needs.authorize.outputs.version }}
|
||||
CLI_TAG: ${{ needs.authorize.outputs.cli_tag }}
|
||||
DESKTOP_TAG: ${{ needs.authorize.outputs.desktop_tag }}
|
||||
run: bash scripts/verify-stable-release-artifacts.sh
|
||||
- name: Publish exact Stable release record
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
VERSION: ${{ needs.authorize.outputs.version }}
|
||||
CLI_TAG: ${{ needs.authorize.outputs.cli_tag }}
|
||||
RELEASE_SHA: ${{ needs.authorize.outputs.sha }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if ! node -e '
|
||||
const catalog = require("./release-notes/releases.json");
|
||||
const release = catalog.releases.find((item) => item.version === process.env.VERSION);
|
||||
process.exit(release?.status === "reviewed" ? 0 : 1);
|
||||
'; then
|
||||
echo "Legacy Stable notes do not require a publication marker"
|
||||
exit 0
|
||||
fi
|
||||
node scripts/release-event.mjs generate \
|
||||
--version "$VERSION" --sha "$RELEASE_SHA" \
|
||||
--published-at "$(gh api "repos/${{ github.repository }}/releases/tags/$CLI_TAG" --jq .published_at)" \
|
||||
--output /tmp/release-event.json
|
||||
existing="$(mktemp -d)"
|
||||
if gh release download "$CLI_TAG" --pattern release-event.json --dir "$existing" 2>/dev/null; then
|
||||
cmp -s /tmp/release-event.json "$existing/release-event.json" || {
|
||||
echo "::error::published release-event.json differs from the approved Stable event"
|
||||
exit 1
|
||||
}
|
||||
else
|
||||
gh release upload "$CLI_TAG" /tmp/release-event.json
|
||||
fi
|
||||
- name: Refresh public changelog
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
run: gh workflow run pages.yml --ref main-v2
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,62 @@
|
||||
name: Request verification on released issues
|
||||
|
||||
# A PR that writes "Fixes #N" auto-closes its issue on merge. One that writes
|
||||
# "Refs"/"Related"/"Addresses" does not, so the report stays open waiting for a
|
||||
# follow-up nobody remembers to send after the release. This sends it.
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
tag:
|
||||
description: Release tag to announce (e.g. desktop-v1.18.0)
|
||||
required: true
|
||||
type: string
|
||||
dry_run:
|
||||
description: Print the targets without commenting
|
||||
required: false
|
||||
default: true
|
||||
type: boolean
|
||||
release:
|
||||
types: [published]
|
||||
push:
|
||||
branches:
|
||||
- main-v2
|
||||
paths:
|
||||
- '.github/workflows/release-verify-issues.yml'
|
||||
- 'scripts/release-verify-issues.mjs'
|
||||
- 'scripts/release-verify-issues.test.mjs'
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
issues: write
|
||||
|
||||
concurrency:
|
||||
group: release-verify-issues-${{ github.event.inputs.tag || github.event.release.tag_name || github.ref }}
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
notify:
|
||||
if: github.repository == 'esengine/DeepSeek-Reasonix'
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- uses: actions/setup-node@v7
|
||||
with:
|
||||
node-version: '22'
|
||||
- name: Test notifier
|
||||
run: node --test scripts/release-verify-issues.test.mjs
|
||||
# A push that only touched the script has no release to announce; the test
|
||||
# above is the whole point of that trigger.
|
||||
- name: Request verification
|
||||
if: github.event_name != 'push'
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
TAG: ${{ github.event.inputs.tag || github.event.release.tag_name }}
|
||||
DRY_RUN: ${{ github.event.inputs.dry_run || 'false' }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
args=(--tag "$TAG")
|
||||
if [ "$DRY_RUN" = "true" ]; then args+=(--dry-run); fi
|
||||
node scripts/release-verify-issues.mjs "${args[@]}"
|
||||
@@ -0,0 +1,581 @@
|
||||
name: Release
|
||||
|
||||
# Native CLI binary line. Official releases are called by the protected Stable
|
||||
# orchestrator. Manual dispatch remains available only for official recovery;
|
||||
# historical Preview inputs below are workflow-call compatibility, not a public
|
||||
# publication entrypoint.
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
channel:
|
||||
description: "Standalone CLI recovery channel"
|
||||
required: true
|
||||
default: stable
|
||||
type: choice
|
||||
options: [stable]
|
||||
tag:
|
||||
description: "Existing official CLI tag (for example v1.18.0)"
|
||||
required: true
|
||||
type: string
|
||||
workflow_call:
|
||||
inputs:
|
||||
channel:
|
||||
description: "Native CLI release channel selected by the approved orchestrator"
|
||||
required: false
|
||||
default: stable
|
||||
type: string
|
||||
tag:
|
||||
description: "Existing CLI tag selected by the approved release orchestrator"
|
||||
required: true
|
||||
type: string
|
||||
approved_cli_tag:
|
||||
description: "Stable CLI tag recorded by the approved orchestrator"
|
||||
required: true
|
||||
type: string
|
||||
approved_sha:
|
||||
description: "Immutable commit recorded by the approved orchestrator"
|
||||
required: true
|
||||
type: string
|
||||
orchestrated:
|
||||
description: "True only when called by an approved release orchestrator"
|
||||
required: false
|
||||
default: false
|
||||
type: boolean
|
||||
orchestrator:
|
||||
description: "Trusted release orchestrator (legacy Preview calls remain readable)"
|
||||
required: false
|
||||
default: stable
|
||||
type: string
|
||||
allow_preview_recovery:
|
||||
description: "Legacy compatibility for already-created Preview runs"
|
||||
required: false
|
||||
default: false
|
||||
type: boolean
|
||||
|
||||
permissions:
|
||||
contents: write # create the release and upload archives
|
||||
|
||||
concurrency:
|
||||
# A channel pointer is a monotonic public state machine. Serialize all
|
||||
# publishers for the same channel so an older recovery run cannot pass its
|
||||
# read-before-write window after a newer release has published.
|
||||
group: release-cli-${{ inputs.channel || 'stable' }}
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
resolve:
|
||||
name: resolve CLI release
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
tag: ${{ steps.release.outputs.tag }}
|
||||
version: ${{ steps.release.outputs.version }}
|
||||
base_version: ${{ steps.release.outputs.base_version }}
|
||||
notes_version: ${{ steps.release.outputs.notes_version }}
|
||||
channel: ${{ steps.release.outputs.channel }}
|
||||
prerelease: ${{ steps.release.outputs.prerelease }}
|
||||
sha: ${{ steps.candidate.outputs.sha }}
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
fetch-depth: 0
|
||||
ref: ${{ github.sha }}
|
||||
- name: Resolve channel and tag
|
||||
id: release
|
||||
env:
|
||||
EVENT_NAME: ${{ github.event_name }}
|
||||
IN_ORCHESTRATED: ${{ inputs.orchestrated }}
|
||||
IN_CHANNEL: ${{ inputs.channel }}
|
||||
IN_TAG: ${{ inputs.tag }}
|
||||
REF_NAME: ${{ github.ref_name }}
|
||||
CALLER_REF: ${{ github.ref }}
|
||||
CALLER_REF_PROTECTED: ${{ github.ref_protected }}
|
||||
run: bash scripts/resolve-cli-release.sh
|
||||
- name: Record immutable candidate
|
||||
id: candidate
|
||||
env:
|
||||
RELEASE_TAG: ${{ steps.release.outputs.tag }}
|
||||
RELEASE_CHANNEL: ${{ steps.release.outputs.channel }}
|
||||
IN_ORCHESTRATED: ${{ inputs.orchestrated }}
|
||||
IN_ORCHESTRATOR: ${{ inputs.orchestrator }}
|
||||
ALLOW_PREVIEW_RECOVERY: ${{ inputs.allow_preview_recovery }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
git fetch origin main-v2
|
||||
sha="$(git rev-parse "$RELEASE_TAG^{commit}")"
|
||||
if ! git merge-base --is-ancestor "$sha" origin/main-v2; then
|
||||
echo "::error::$RELEASE_TAG points to $sha, which is not on main-v2 history"
|
||||
exit 1
|
||||
fi
|
||||
if [ "$ALLOW_PREVIEW_RECOVERY" = "true" ]; then
|
||||
if [ "$IN_ORCHESTRATED" != "true" ] || [ "$IN_ORCHESTRATOR" != "preview" ] || [ "$RELEASE_CHANNEL" != "preview" ]; then
|
||||
echo "::error::Preview recovery requires the approved Preview orchestrator"
|
||||
exit 1
|
||||
fi
|
||||
elif [ "$RELEASE_CHANNEL" = "preview" ] && [ "$sha" != "$(git rev-parse origin/main-v2)" ]; then
|
||||
echo "::error::CLI Preview must tag current main-v2; $RELEASE_TAG points to $sha"
|
||||
exit 1
|
||||
fi
|
||||
echo "sha=$sha" >> "$GITHUB_OUTPUT"
|
||||
- name: Verify existing protected tag
|
||||
env:
|
||||
RELEASE_TAG: ${{ steps.release.outputs.tag }}
|
||||
APPROVED_SHA: ${{ steps.candidate.outputs.sha }}
|
||||
VERIFY_RELEASE_CHECKOUT: false
|
||||
run: bash scripts/verify-release-tag.sh
|
||||
|
||||
orchestration-guard:
|
||||
name: verify approved orchestrator
|
||||
needs: resolve
|
||||
if: ${{ inputs.orchestrated }}
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
fetch-depth: 0
|
||||
ref: ${{ github.sha }}
|
||||
- name: Verify caller and approved release ref
|
||||
env:
|
||||
ACTUAL_CALLER_WORKFLOW_REF: ${{ github.workflow_ref }}
|
||||
EXPECTED_CALLER_WORKFLOW_REF: ${{ format('{0}/.github/workflows/release-{1}.yml@{2}', github.repository, inputs.orchestrator, github.ref) }}
|
||||
CALLER_EVENT_NAME: ${{ github.event_name }}
|
||||
CALLER_REF: ${{ github.ref }}
|
||||
CALLER_REF_PROTECTED: ${{ github.ref_protected }}
|
||||
CALLER_WORKFLOW_SHA: ${{ github.workflow_sha }}
|
||||
CALLER_SHA: ${{ github.sha }}
|
||||
APPROVED_CLI_TAG: ${{ inputs.approved_cli_tag }}
|
||||
APPROVED_SHA: ${{ inputs.approved_sha }}
|
||||
APPROVED_CHANNEL: ${{ inputs.orchestrator }}
|
||||
RELEASE_TAG: ${{ inputs.tag }}
|
||||
VERIFY_RELEASE_CHECKOUT: false
|
||||
run: |
|
||||
bash scripts/verify-release-authorization.sh
|
||||
bash scripts/verify-release-tag.sh
|
||||
|
||||
release-gate:
|
||||
name: approve standalone CLI release
|
||||
needs: resolve
|
||||
if: ${{ !inputs.orchestrated }}
|
||||
runs-on: ubuntu-latest
|
||||
environment: release
|
||||
steps:
|
||||
- env:
|
||||
RELEASE_TAG: ${{ needs.resolve.outputs.tag }}
|
||||
RELEASE_CHANNEL: ${{ needs.resolve.outputs.channel }}
|
||||
run: echo "Approved standalone CLI $RELEASE_CHANNEL release $RELEASE_TAG"
|
||||
|
||||
cache-guard:
|
||||
name: cache hit guard
|
||||
needs: [resolve, orchestration-guard, release-gate]
|
||||
if: ${{ always() && !cancelled() && needs.resolve.result == 'success' && ((inputs.orchestrated && needs.orchestration-guard.result == 'success') || (!inputs.orchestrated && needs.release-gate.result == 'success')) }}
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
ref: ${{ needs.resolve.outputs.sha }}
|
||||
- uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: go.mod
|
||||
cache: true
|
||||
- run: ./scripts/cache-guard.sh
|
||||
- name: Verify embedded documentation identity
|
||||
env:
|
||||
DOCS_BUILD_VERSION: ${{ needs.resolve.outputs.tag }}
|
||||
DOCS_SOURCE_REVISION: ${{ needs.resolve.outputs.sha }}
|
||||
run: |
|
||||
if [ ! -f scripts/verify-embedded-docs.sh ]; then
|
||||
echo "Legacy candidate predates the embedded docs contract; skipping."
|
||||
exit 0
|
||||
fi
|
||||
bash scripts/verify-embedded-docs.sh "$DOCS_BUILD_VERSION" "$DOCS_SOURCE_REVISION"
|
||||
|
||||
goreleaser:
|
||||
name: archives + checksums + homebrew tap
|
||||
needs: [resolve, cache-guard]
|
||||
if: ${{ always() && !cancelled() && needs.cache-guard.result == 'success' }}
|
||||
runs-on: ubuntu-latest
|
||||
# The Stable caller has already passed the single GitHub release approval.
|
||||
# Official standalone recovery passes release-gate above. This job therefore
|
||||
# must not add a second GitHub environment approval.
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
fetch-depth: 0
|
||||
ref: ${{ needs.resolve.outputs.sha }}
|
||||
# Recovery may build an immutable tag that predates the current recovery
|
||||
# policy. Keep product sources pinned above, but execute publication
|
||||
# decisions from the protected workflow commit.
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
fetch-depth: 0
|
||||
path: release-control
|
||||
ref: ${{ github.workflow_sha }}
|
||||
- name: Isolate release-control checkout from product git state
|
||||
run: |
|
||||
set -euo pipefail
|
||||
git_common_dir="$(git rev-parse --path-format=absolute --git-common-dir)"
|
||||
exclude_file="$git_common_dir/info/exclude"
|
||||
if ! grep -qxF '/release-control/' "$exclude_file"; then
|
||||
printf '%s\n' '/release-control/' >> "$exclude_file"
|
||||
fi
|
||||
git check-ignore -q release-control/
|
||||
dirty="$(git status --porcelain --untracked-files=all)"
|
||||
if [ -n "$dirty" ]; then
|
||||
printf 'product checkout is dirty before release:\n%s\n' "$dirty" >&2
|
||||
exit 1
|
||||
fi
|
||||
- uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: go.mod
|
||||
cache: true
|
||||
- uses: actions/setup-node@v7
|
||||
with:
|
||||
node-version: "22"
|
||||
- name: Download orchestrator-reviewed release notes
|
||||
if: ${{ inputs.orchestrated }}
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: orchestrator-reviewed-release-notes
|
||||
path: /tmp/orchestrator-reviewed-release-notes
|
||||
- name: Use orchestrator-reviewed release notes
|
||||
if: ${{ inputs.orchestrated }}
|
||||
run: |
|
||||
test -s /tmp/orchestrator-reviewed-release-notes/release-notes.md
|
||||
cp /tmp/orchestrator-reviewed-release-notes/release-notes.md /tmp/release-notes.md
|
||||
- name: Render reviewed release notes
|
||||
if: ${{ !inputs.orchestrated }}
|
||||
env:
|
||||
RELEASE_TAG: ${{ needs.resolve.outputs.notes_version }}
|
||||
run: node scripts/release-notes.mjs render --version "$RELEASE_TAG" --output /tmp/release-notes.md
|
||||
- name: Revalidate approved release ref
|
||||
env:
|
||||
RELEASE_TAG: ${{ needs.resolve.outputs.tag }}
|
||||
APPROVED_SHA: ${{ needs.resolve.outputs.sha }}
|
||||
run: bash scripts/verify-release-tag.sh
|
||||
- name: Decide whether CLI artifacts need publication
|
||||
id: publication
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
TAG: ${{ needs.resolve.outputs.tag }}
|
||||
CHANNEL: ${{ needs.resolve.outputs.channel }}
|
||||
PRERELEASE: ${{ needs.resolve.outputs.prerelease }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
validation_channel="$CHANNEL"
|
||||
if [ "$CHANNEL" = "stable" ] && [ "$PRERELEASE" = "true" ]; then
|
||||
validation_channel=any
|
||||
fi
|
||||
release_json=/tmp/existing-cli-release.json
|
||||
release_error=/tmp/existing-cli-release.error
|
||||
checksums=/tmp/existing-cli-release-SHA256SUMS
|
||||
if gh api "repos/${{ github.repository }}/releases/tags/$TAG" \
|
||||
>"$release_json" 2>"$release_error"; then
|
||||
gh release download "$TAG" -R "${{ github.repository }}" \
|
||||
--pattern SHA256SUMS --output "$checksums"
|
||||
decision="$(
|
||||
bash release-control/scripts/decide-cli-release-publication.sh \
|
||||
"$validation_channel" "$TAG" "${{ github.repository }}" \
|
||||
"$release_json" "$checksums"
|
||||
)"
|
||||
echo "existing CLI release $TAG is complete and checksum-bound; reusing it"
|
||||
elif grep -Eiq 'HTTP 404|Not Found' "$release_error"; then
|
||||
decision="$(
|
||||
bash release-control/scripts/decide-cli-release-publication.sh \
|
||||
"$validation_channel" "$TAG" "${{ github.repository }}" - -
|
||||
)"
|
||||
echo "CLI release $TAG does not exist; GoReleaser will publish it"
|
||||
else
|
||||
cat "$release_error" >&2
|
||||
exit 1
|
||||
fi
|
||||
test "$decision" = "publish" -o "$decision" = "reuse"
|
||||
echo "decision=$decision" >> "$GITHUB_OUTPUT"
|
||||
- uses: goreleaser/goreleaser-action@v7
|
||||
if: ${{ steps.publication.outputs.decision == 'publish' }}
|
||||
with:
|
||||
version: '~> v2'
|
||||
args: release --clean
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
HOMEBREW_TAP_TOKEN: ${{ needs.resolve.outputs.channel == 'stable' && secrets.HOMEBREW_TAP_TOKEN || '' }}
|
||||
# workflow_dispatch recovery runs have a branch-shaped GITHUB_REF even
|
||||
# though checkout is on the release tag. Pin GoReleaser explicitly, and
|
||||
# avoid ambiguity from the three release tags sharing one commit.
|
||||
GORELEASER_CURRENT_TAG: ${{ needs.resolve.outputs.tag }}
|
||||
|
||||
- name: Publish product release notes
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
TAG: ${{ needs.resolve.outputs.tag }}
|
||||
run: gh release edit "$TAG" --notes-file /tmp/release-notes.md
|
||||
|
||||
- name: Publish CLI release metadata to R2
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
TAG: ${{ needs.resolve.outputs.tag }}
|
||||
NOTES_TAG: ${{ needs.resolve.outputs.notes_version }}
|
||||
HAS_R2: ${{ secrets.R2_ACCESS_KEY_ID != '' && secrets.R2_SECRET_ACCESS_KEY != '' && secrets.R2_ACCOUNT_ID != '' && secrets.R2_BUCKET != '' }}
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
|
||||
AWS_DEFAULT_REGION: auto
|
||||
R2_ACCOUNT_ID: ${{ secrets.R2_ACCOUNT_ID }}
|
||||
R2_BUCKET: ${{ secrets.R2_BUCKET }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ "$HAS_R2" != "true" ]; then
|
||||
echo "R2 secrets not configured; skipping CLI release metadata"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
channel=""
|
||||
if [[ "$TAG" =~ ^v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)$ ]]; then
|
||||
channel="stable"
|
||||
elif [[ "$TAG" =~ ^v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)-preview\.(0|[1-9][0-9]*)$ ]]; then
|
||||
channel="preview"
|
||||
else
|
||||
echo "internal CLI release $TAG; publishing only immutable metadata"
|
||||
fi
|
||||
|
||||
required_assets='[
|
||||
"reasonix-darwin-amd64.tar.gz",
|
||||
"reasonix-darwin-arm64.tar.gz",
|
||||
"reasonix-linux-amd64.tar.gz",
|
||||
"reasonix-linux-arm64.tar.gz",
|
||||
"reasonix-windows-amd64.zip",
|
||||
"reasonix-windows-arm64.zip",
|
||||
"SHA256SUMS"
|
||||
]'
|
||||
gh api "repos/${{ github.repository }}/releases/tags/$TAG" > /tmp/cli-release.raw.json
|
||||
jq --arg tag "$TAG" --arg notes_tag "$NOTES_TAG" --argjson required "$required_assets" '
|
||||
if .tag_name != $tag then error("release tag mismatch") else . end |
|
||||
if .draft then error("draft release cannot be published") else . end |
|
||||
. as $release |
|
||||
($release.assets | map({key: .name, value: .}) | from_entries) as $assets |
|
||||
if ($required | all(. as $name | $assets[$name] != null))
|
||||
then {
|
||||
tag_name: $release.tag_name,
|
||||
prerelease: $release.prerelease,
|
||||
html_url: $release.html_url,
|
||||
release_notes_url: ("https://reasonix.io/changelog/" + $notes_tag + "/"),
|
||||
assets: [
|
||||
$required[] as $name |
|
||||
$assets[$name] |
|
||||
{
|
||||
name: .name,
|
||||
browser_download_url: .browser_download_url,
|
||||
size: .size
|
||||
}
|
||||
]
|
||||
}
|
||||
else error("release is missing one or more required CLI assets")
|
||||
end
|
||||
' /tmp/cli-release.raw.json > /tmp/cli-release.json
|
||||
if [ -n "$channel" ]; then
|
||||
bash scripts/validate-cli-release-manifest.sh \
|
||||
"$channel" "$TAG" "${{ github.repository }}" /tmp/cli-release.json "$NOTES_TAG"
|
||||
else
|
||||
bash scripts/validate-cli-release-manifest.sh \
|
||||
any "$TAG" "${{ github.repository }}" /tmp/cli-release.json "$NOTES_TAG"
|
||||
fi
|
||||
|
||||
endpoint="https://${R2_ACCOUNT_ID}.r2.cloudflarestorage.com"
|
||||
validation_channel="${channel:-any}"
|
||||
immutable_key="cli/releases/${TAG}/latest.json"
|
||||
immutable_error="$(mktemp)"
|
||||
if aws s3 cp "s3://${R2_BUCKET}/${immutable_key}" /tmp/cli-release.immutable.json \
|
||||
--endpoint-url "$endpoint" 2>"$immutable_error"; then
|
||||
bash scripts/validate-cli-release-manifest.sh \
|
||||
"legacy-${validation_channel}" "$TAG" "${{ github.repository }}" \
|
||||
/tmp/cli-release.immutable.json "$NOTES_TAG"
|
||||
if ! bash scripts/compare-cli-release-manifests.sh \
|
||||
/tmp/cli-release.json /tmp/cli-release.immutable.json; then
|
||||
echo "::error::immutable CLI release metadata for $TAG already exists with different content"
|
||||
exit 1
|
||||
fi
|
||||
echo "immutable CLI release metadata for $TAG already exists; preserving it"
|
||||
elif grep -Eiq '404|NoSuchKey|Not Found' "$immutable_error"; then
|
||||
aws s3 cp /tmp/cli-release.json "s3://${R2_BUCKET}/${immutable_key}" \
|
||||
--endpoint-url "$endpoint" \
|
||||
--content-type "application/json; charset=utf-8" \
|
||||
--cache-control "public, max-age=31536000, immutable"
|
||||
else
|
||||
cat "$immutable_error" >&2
|
||||
exit 1
|
||||
fi
|
||||
rm -f "$immutable_error"
|
||||
|
||||
aws s3 cp "s3://${R2_BUCKET}/${immutable_key}" /tmp/cli-release.immutable.json \
|
||||
--endpoint-url "$endpoint"
|
||||
bash scripts/validate-cli-release-manifest.sh \
|
||||
"legacy-${validation_channel}" "$TAG" "${{ github.repository }}" \
|
||||
/tmp/cli-release.immutable.json "$NOTES_TAG"
|
||||
bash scripts/compare-cli-release-manifests.sh \
|
||||
/tmp/cli-release.json /tmp/cli-release.immutable.json
|
||||
|
||||
if [ -z "$channel" ]; then
|
||||
echo "internal CLI release $TAG; Stable and Preview pointers remain unchanged"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
current_tag=""
|
||||
pointer_error="$(mktemp)"
|
||||
if aws s3 cp "s3://${R2_BUCKET}/cli/${channel}/latest.json" /tmp/cli-release.pointer.json \
|
||||
--endpoint-url "$endpoint" 2>"$pointer_error"; then
|
||||
current_tag="$(jq -er '.tag_name | strings' /tmp/cli-release.pointer.json)"
|
||||
bash scripts/validate-cli-release-manifest.sh \
|
||||
"legacy-${channel}" "$current_tag" "${{ github.repository }}" \
|
||||
/tmp/cli-release.pointer.json "$current_tag"
|
||||
elif grep -Eiq '404|NoSuchKey|Not Found' "$pointer_error"; then
|
||||
echo "CLI $channel pointer does not exist yet"
|
||||
else
|
||||
cat "$pointer_error" >&2
|
||||
exit 1
|
||||
fi
|
||||
rm -f "$pointer_error"
|
||||
|
||||
pointer_manifest=-
|
||||
if [ -n "$current_tag" ]; then
|
||||
pointer_manifest=/tmp/cli-release.pointer.json
|
||||
fi
|
||||
pointer_decision="$(
|
||||
bash scripts/decide-cli-pointer-update.sh \
|
||||
"$channel" /tmp/cli-release.json "$pointer_manifest"
|
||||
)"
|
||||
if [ "$pointer_decision" = "skip" ]; then
|
||||
echo "CLI $channel pointer remains ${current_tag:-unset}; candidate $TAG is not newer and needs no repair"
|
||||
exit 0
|
||||
fi
|
||||
aws s3 cp /tmp/cli-release.json "s3://${R2_BUCKET}/cli/${channel}/latest.json" \
|
||||
--endpoint-url "$endpoint" \
|
||||
--content-type "application/json; charset=utf-8" \
|
||||
--cache-control "public, max-age=300, stale-if-error=86400"
|
||||
|
||||
aws s3 cp "s3://${R2_BUCKET}/cli/${channel}/latest.json" /tmp/cli-release.pointer.json \
|
||||
--endpoint-url "$endpoint"
|
||||
bash scripts/validate-cli-release-manifest.sh \
|
||||
"$channel" "$TAG" "${{ github.repository }}" \
|
||||
/tmp/cli-release.pointer.json "$NOTES_TAG"
|
||||
cmp -s /tmp/cli-release.json /tmp/cli-release.pointer.json
|
||||
echo "CLI $channel pointer -> $TAG"
|
||||
|
||||
- name: Attach desktop manifest compatibility asset
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
TAG: ${{ needs.resolve.outputs.tag }}
|
||||
HAS_R2: ${{ secrets.R2_ACCESS_KEY_ID != '' && secrets.R2_SECRET_ACCESS_KEY != '' && secrets.R2_ACCOUNT_ID != '' && secrets.R2_BUCKET != '' }}
|
||||
R2_ACCOUNT_ID: ${{ secrets.R2_ACCOUNT_ID }}
|
||||
R2_BUCKET: ${{ secrets.R2_BUCKET }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
case "$TAG" in
|
||||
*-*)
|
||||
echo "prerelease $TAG — GitHub latest does not move here; skipping desktop manifest compatibility asset"
|
||||
exit 0
|
||||
;;
|
||||
esac
|
||||
if [ "$HAS_R2" != "true" ]; then
|
||||
echo "R2 secrets not configured; skipping desktop manifest compatibility asset"
|
||||
exit 0
|
||||
fi
|
||||
# dl.reasonix.io serves 403 to GitHub Actions egress IPs (Cloudflare bot
|
||||
# protection), so read the manifest over the authenticated S3 API instead
|
||||
# of the public edge.
|
||||
aws configure set aws_access_key_id "${{ secrets.R2_ACCESS_KEY_ID }}"
|
||||
aws configure set aws_secret_access_key "${{ secrets.R2_SECRET_ACCESS_KEY }}"
|
||||
aws configure set region auto
|
||||
aws s3 cp "s3://${R2_BUCKET}/latest/latest.json" latest.raw.json \
|
||||
--endpoint-url "https://${R2_ACCOUNT_ID}.r2.cloudflarestorage.com"
|
||||
jq '.download_page = "https://reasonix.io/?download=desktop#start"' latest.raw.json > latest.json
|
||||
jq -e '
|
||||
([.platforms[] | (.url, .sig)] |
|
||||
all(type == "string" and startswith("https://dl.reasonix.io/") and (contains("/releases/latest/") | not)))
|
||||
' latest.json >/dev/null
|
||||
gh release upload "$TAG" latest.json --clobber
|
||||
|
||||
# The compatibility asset exists for pre-v1.16 desktop updaters that
|
||||
# still poll GitHub's repository-wide latest URL. Desktop releases now
|
||||
# own that Latest badge, but this check still exercises the public fallback
|
||||
# path exactly the way those clients fetch it: anonymously, over the public
|
||||
# edge, with a Go client UA. Unlike dl.reasonix.io (whose bot protection
|
||||
# 403s Actions egress — see the R2 note above), GitHub serves its own
|
||||
# runners, so this can hard-fail. #5826/#5858 shipped a broken update check
|
||||
# for weeks precisely because nothing exercised the public path. Retries
|
||||
# cover the release CDN propagating the freshly uploaded asset.
|
||||
- name: Smoke public compatibility manifest
|
||||
env:
|
||||
TAG: ${{ needs.resolve.outputs.tag }}
|
||||
HAS_R2: ${{ secrets.R2_ACCESS_KEY_ID != '' && secrets.R2_SECRET_ACCESS_KEY != '' && secrets.R2_ACCOUNT_ID != '' && secrets.R2_BUCKET != '' }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
case "$TAG" in
|
||||
*-*)
|
||||
echo "prerelease $TAG — no compatibility asset uploaded; skipping"
|
||||
exit 0
|
||||
;;
|
||||
esac
|
||||
if [ "$HAS_R2" != "true" ]; then
|
||||
echo "R2 secrets not configured; no compatibility asset uploaded; skipping"
|
||||
exit 0
|
||||
fi
|
||||
url="https://github.com/${{ github.repository }}/releases/latest/download/latest.json"
|
||||
for attempt in 1 2 3 4 5 6; do
|
||||
if curl -fsSL -A "Go-http-client/2.0" -o /tmp/compat-latest.json "$url"; then
|
||||
jq -e '(.version | type == "string") and (.platforms | type == "object")' /tmp/compat-latest.json >/dev/null
|
||||
echo "public compatibility manifest OK (desktop version $(jq -r .version /tmp/compat-latest.json))"
|
||||
exit 0
|
||||
fi
|
||||
echo "attempt $attempt failed; retrying in 10s"
|
||||
sleep 10
|
||||
done
|
||||
echo "::error::public compatibility manifest unreachable at $url"
|
||||
exit 1
|
||||
|
||||
# A stable CLI release must never leave the npm line behind: v1.17.5
|
||||
# shipped as binaries/Homebrew while npm `latest` still pointed at 0.53.2
|
||||
# (#5822) — every `npm update -g` user was silently downgraded to a
|
||||
# months-old version, and nothing noticed because the npm line
|
||||
# (release-npm.yml, `npm-vX.Y.Z` tags) is triggered independently and the
|
||||
# stable npm tag was simply never pushed. release-npm.yml's own verify
|
||||
# step only guards runs that happen; this guard catches the run that
|
||||
# DIDN'T.
|
||||
#
|
||||
# Two distinct states, two responses (the approved orchestrator starts the
|
||||
# CLI and npm reusable workflows concurrently, and npm dist-tags propagate
|
||||
# asynchronously, so "tag pushed but latest not moved yet" is a NORMAL
|
||||
# mid-release state, not a failure):
|
||||
# - npm-v<version> tag missing -> hard fail. This is the #5822 gap:
|
||||
# nobody pushed the npm release at all.
|
||||
# - tag pushed, latest lagging -> poll briefly, then WARN and pass.
|
||||
# The npm job may still be publishing; release-npm.yml's verify step
|
||||
# owns asserting the dist-tag lands.
|
||||
- name: Check npm latest dist-tag freshness
|
||||
env:
|
||||
TAG: ${{ needs.resolve.outputs.tag }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
case "$TAG" in
|
||||
*-*)
|
||||
echo "prerelease $TAG — npm latest does not move on prereleases; skipping"
|
||||
exit 0
|
||||
;;
|
||||
esac
|
||||
version="${TAG#v}"
|
||||
if ! git ls-remote --exit-code origin "refs/tags/npm-v$version" >/dev/null; then
|
||||
echo "::error::the npm-v$version tag was never pushed — the npm channel is being left behind and 'npm update -g' users will be downgraded to the old 'latest'. Push it: git tag npm-v$version ${TAG} && git push origin npm-v$version (or 'npm dist-tag add reasonix@$version latest' for an already-published version)."
|
||||
exit 1
|
||||
fi
|
||||
for attempt in 1 2 3 4 5 6; do
|
||||
got="$(npm view reasonix dist-tags.latest 2>/dev/null || true)"
|
||||
if [ -n "$got" ]; then
|
||||
newest="$(printf '%s\n%s\n' "$got" "$version" | sort -V | tail -1)"
|
||||
if [ "$newest" = "$got" ]; then
|
||||
echo "npm latest -> $got (>= $version) OK"
|
||||
exit 0
|
||||
fi
|
||||
fi
|
||||
echo "npm latest -> ${got:-<unreadable>}, want >= $version (attempt $attempt)"
|
||||
sleep 10
|
||||
done
|
||||
echo "::warning::npm-v$version is pushed but npm 'latest' is still ${got:-<unreadable>} — the concurrent npm publish is likely still running or propagating. Monitor the npm job; its verify step asserts the dist-tag lands."
|
||||
exit 0
|
||||
@@ -24,7 +24,7 @@ jobs:
|
||||
if: github.event.comment.user.type != 'Bot' && github.event.comment.author_association == 'NONE'
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9
|
||||
- uses: actions/github-script@v9
|
||||
with:
|
||||
script: |
|
||||
const EMOJI = /[\u{1F300}-\u{1FAFF}\u{2600}-\u{27BF}\u{2B00}-\u{2BFF}\u{FE0F}]/gu;
|
||||
|
||||
@@ -40,10 +40,10 @@ jobs:
|
||||
if: github.repository == 'esengine/DeepSeek-Reasonix'
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
|
||||
- uses: actions/setup-node@v7
|
||||
with:
|
||||
node-version: '22'
|
||||
- name: Test sweeper
|
||||
|
||||
@@ -1,117 +0,0 @@
|
||||
name: Studio Certum signing smoke test
|
||||
|
||||
# Signs two throwaway probes with the certificate release-studio.yml would use,
|
||||
# and publishes nothing. It reports the signer's subject, which is the value
|
||||
# STUDIO_SIGNING_SUBJECT must hold.
|
||||
on:
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: certum-signing
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
sign:
|
||||
if: github.repository == 'esengine/DeepSeek-Reasonix' && github.ref == 'refs/heads/studio'
|
||||
runs-on: windows-2022
|
||||
environment: studio-release
|
||||
timeout-minutes: 20
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7
|
||||
with:
|
||||
go-version-file: go.mod
|
||||
cache: false
|
||||
|
||||
- name: Check signing configuration
|
||||
shell: pwsh
|
||||
env:
|
||||
CERTUM_USERNAME: ${{ secrets.CERTUM_USERNAME }}
|
||||
CERTUM_OTP_URI: ${{ secrets.CERTUM_OTP_URI }}
|
||||
CERTUM_KEY_ID: ${{ secrets.CERTUM_KEY_ID }}
|
||||
run: |
|
||||
$ErrorActionPreference = 'Stop'
|
||||
if ([string]::IsNullOrWhiteSpace($env:CERTUM_USERNAME)) { throw 'Missing secret CERTUM_USERNAME' }
|
||||
if ($env:CERTUM_OTP_URI -notmatch '^otpauth://totp/') { throw 'Secret CERTUM_OTP_URI must be a complete TOTP URI' }
|
||||
if ($env:CERTUM_KEY_ID -notmatch '^[0-9a-fA-F]{40}$') { throw 'Secret CERTUM_KEY_ID must be the 40-character SHA-1 certificate thumbprint' }
|
||||
|
||||
- name: Build open-source signing probes
|
||||
shell: pwsh
|
||||
run: |
|
||||
$ErrorActionPreference = 'Stop'
|
||||
# This minimal probe is covered by the repository's MIT license.
|
||||
New-Item -ItemType Directory payload | Out-Null
|
||||
'package main; func main() { println("Reasonix Studio Certum signing probe") }' | Set-Content probe.go
|
||||
$env:GOOS = 'windows'
|
||||
$env:GOARCH = 'amd64'
|
||||
$env:CGO_ENABLED = '0'
|
||||
go build -trimpath -o payload/probe.exe probe.go
|
||||
if ($LASTEXITCODE -ne 0) { throw 'Probe compilation failed' }
|
||||
'probe.exe' | Set-Content payload/signing-files.txt
|
||||
Copy-Item payload/probe.exe installer-probe.exe
|
||||
|
||||
- name: Connect to SimplySign cloud certificate
|
||||
uses: ./.github/actions/setup-certum
|
||||
with:
|
||||
username: ${{ secrets.CERTUM_USERNAME }}
|
||||
otp-uri: ${{ secrets.CERTUM_OTP_URI }}
|
||||
thumbprint: ${{ secrets.CERTUM_KEY_ID }}
|
||||
|
||||
- name: Sign and verify payload and installer probes
|
||||
shell: pwsh
|
||||
env:
|
||||
CERTUM_KEY_ID: ${{ secrets.CERTUM_KEY_ID }}
|
||||
run: |
|
||||
./scripts/sign-certum.ps1 -PayloadDirectory payload
|
||||
./scripts/sign-certum.ps1 -FilePath installer-probe.exe
|
||||
|
||||
# The session signs for any process on the runner while it is up.
|
||||
- name: Disconnect from Certum
|
||||
if: always()
|
||||
shell: pwsh
|
||||
run: Get-Process -Name '*SimplySign*' -ErrorAction SilentlyContinue | Stop-Process -Force
|
||||
|
||||
# With the switch on, an unset subject fails here as it would in a
|
||||
# release; with it off, the value is reported so it can be recorded.
|
||||
- name: Report and check the signer
|
||||
shell: pwsh
|
||||
env:
|
||||
SIGNING_ENABLED: ${{ vars.STUDIO_SIGNING_ENABLED }}
|
||||
EXPECTED_SUBJECT: ${{ vars.STUDIO_SIGNING_SUBJECT }}
|
||||
run: |
|
||||
$ErrorActionPreference = 'Stop'
|
||||
$failed = $false
|
||||
foreach ($file in @('payload/probe.exe', 'installer-probe.exe')) {
|
||||
$signature = Get-AuthenticodeSignature -LiteralPath $file
|
||||
$certificate = $signature.SignerCertificate
|
||||
$stamp = if ($signature.TimeStamperCertificate) { $signature.TimeStamperCertificate.Subject } else { '(none)' }
|
||||
@("### $file", '', "- subject: ``$($certificate.Subject)``", "- issuer: ``$($certificate.Issuer)``",
|
||||
"- thumbprint: ``$($certificate.Thumbprint)``", "- timestamp: ``$stamp``", '') >> $env:GITHUB_STEP_SUMMARY
|
||||
if (-not $signature.TimeStamperCertificate) {
|
||||
Write-Output "::error title=studio-signing.timestamp-missing::$file carries no timestamp"
|
||||
$failed = $true
|
||||
}
|
||||
if ([string]::IsNullOrWhiteSpace($certificate.Issuer)) {
|
||||
Write-Output "::error title=studio-signing.issuer-missing::$file has a signer certificate without an issuer"
|
||||
$failed = $true
|
||||
}
|
||||
if ([string]::IsNullOrWhiteSpace($env:EXPECTED_SUBJECT)) {
|
||||
if ($env:SIGNING_ENABLED -eq 'true') {
|
||||
Write-Output "::error title=studio-signing.subject-unset::STUDIO_SIGNING_SUBJECT is empty while STUDIO_SIGNING_ENABLED=true"
|
||||
$failed = $true
|
||||
} else {
|
||||
Write-Output "::warning::STUDIO_SIGNING_SUBJECT is not set; record '$($certificate.Subject)'"
|
||||
}
|
||||
} elseif ($certificate.Subject -cne $env:EXPECTED_SUBJECT) {
|
||||
Write-Output "::error title=studio-signing.unexpected-signer::STUDIO_SIGNING_SUBJECT is '$($env:EXPECTED_SUBJECT)' but $file is signed by '$($certificate.Subject)'"
|
||||
$failed = $true
|
||||
}
|
||||
}
|
||||
if ($failed) { exit 1 }
|
||||
'Certum payload and installer signing, signer, trust and timestamps passed.' >> $env:GITHUB_STEP_SUMMARY
|
||||
@@ -1,107 +0,0 @@
|
||||
name: Studio Windows self-test
|
||||
|
||||
# An unsigned Windows installer for trying a branch on a real machine before it
|
||||
# is released. It runs on the intranet runner, so it is dispatch-only: the
|
||||
# repository is public, and nothing a pull request writes may run there.
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
ref:
|
||||
description: Branch, tag or commit to build
|
||||
required: false
|
||||
default: studio
|
||||
type: string
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: studio-selftest-windows
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
build:
|
||||
runs-on: win-gpu
|
||||
timeout-minutes: 90
|
||||
steps:
|
||||
# The machine's PATH finds WSL's bash before Git's, and a bash step run in
|
||||
# WSL sees neither this checkout's toolchain nor Windows paths. cmd, since
|
||||
# the machine's execution policy refuses the script a powershell step is.
|
||||
- name: Put Git Bash first on PATH
|
||||
shell: cmd
|
||||
run: '>>"%GITHUB_PATH%" echo C:\Program Files\Git\bin'
|
||||
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
with:
|
||||
ref: ${{ inputs.ref }}
|
||||
|
||||
- uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7
|
||||
with:
|
||||
go-version-file: go.mod
|
||||
cache: true
|
||||
cache-dependency-path: go.sum
|
||||
|
||||
- uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413 # v6.1.0
|
||||
with:
|
||||
version: 10
|
||||
run_install: false
|
||||
|
||||
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
|
||||
with:
|
||||
node-version: "24"
|
||||
cache: pnpm
|
||||
cache-dependency-path: |
|
||||
desktop/frontend-next/pnpm-lock.yaml
|
||||
desktop/electron/pnpm-lock.yaml
|
||||
|
||||
# A patch above the newest release, as a prerelease: it ranks ahead of
|
||||
# what is published, so the build does not offer itself that release as
|
||||
# an update, and behind the next one, which it does offer.
|
||||
- name: Choose the version
|
||||
id: version
|
||||
shell: bash
|
||||
run: |
|
||||
latest="$(git ls-remote --tags origin 'refs/tags/studio-v*' | sed -n 's#.*refs/tags/studio-v\([0-9.]*\)$#\1#p' | sort -V | tail -1)"
|
||||
test -n "$latest" || { echo "::error::no studio-v* release tag"; exit 1; }
|
||||
IFS=. read -r major minor patch <<< "$latest"
|
||||
version="$major.$minor.$((patch + 1))-selftest.${{ github.run_number }}"
|
||||
echo "version=$version" >> "$GITHUB_OUTPUT"
|
||||
echo "Version: \`$version\` from \`$(git rev-parse --short HEAD)\`" >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
# The intranet's route to the module proxy drops connections now and then.
|
||||
# What arrives stays in this machine's module cache, so a retry resumes.
|
||||
- name: Fetch Go modules
|
||||
shell: bash
|
||||
run: for i in 1 2 3 4 5; do go mod download && exit 0; sleep 10; done; exit 1
|
||||
|
||||
- name: Build the SPA
|
||||
working-directory: desktop/frontend-next
|
||||
shell: bash
|
||||
run: pnpm install --frozen-lockfile && pnpm build
|
||||
|
||||
- name: Build the kernel
|
||||
working-directory: desktop/electron
|
||||
shell: bash
|
||||
env:
|
||||
REASONIX_VERSION: v${{ steps.version.outputs.version }}
|
||||
run: pnpm install --frozen-lockfile && pnpm run build:host
|
||||
|
||||
- name: Build the bundle
|
||||
working-directory: desktop/electron
|
||||
shell: bash
|
||||
run: pnpm exec electron-builder --dir -c.extraMetadata.version="${{ steps.version.outputs.version }}"
|
||||
|
||||
- name: Package the installer
|
||||
working-directory: desktop/electron
|
||||
shell: bash
|
||||
run: |
|
||||
pnpm exec electron-builder --prepackaged ../../dist/win-unpacked --win nsis \
|
||||
-c.extraMetadata.version="${{ steps.version.outputs.version }}"
|
||||
sha256sum ../../dist/ReasonixStudio-windows-amd64-installer.exe | tee -a "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: studio-selftest-windows-${{ steps.version.outputs.version }}
|
||||
path: dist/ReasonixStudio-windows-amd64-installer.exe
|
||||
if-no-files-found: error
|
||||
retention-days: 7
|
||||
@@ -1,303 +0,0 @@
|
||||
name: Studio
|
||||
|
||||
# Studio's own surface — desktop/frontend-next (the SPA) and the Electron shell
|
||||
# that serves it — exists only on the studio branch, so it cannot live in ci.yml
|
||||
# without failing on main-v2 where neither directory is present. ci.yml covers
|
||||
# the kernel; this file covers the frontend build, the desktop module's Go side,
|
||||
# and the packaging contract on real artifacts.
|
||||
on:
|
||||
push:
|
||||
branches: [studio]
|
||||
pull_request:
|
||||
branches: [studio]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: studio-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
frontend:
|
||||
name: frontend-next
|
||||
runs-on: ubuntu-latest
|
||||
defaults:
|
||||
run:
|
||||
working-directory: desktop/frontend-next
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
|
||||
- uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413 # v6.1.0
|
||||
with:
|
||||
version: 10
|
||||
run_install: false
|
||||
|
||||
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
|
||||
with:
|
||||
node-version: "24"
|
||||
cache: pnpm
|
||||
cache-dependency-path: desktop/frontend-next/pnpm-lock.yaml
|
||||
|
||||
- name: Install
|
||||
run: pnpm install --frozen-lockfile
|
||||
|
||||
- name: typecheck
|
||||
run: pnpm typecheck
|
||||
|
||||
# The reducer and the event port carry the two pieces of logic a type
|
||||
# error cannot reach: what a card means when a turn is interrupted, and
|
||||
# what the client does about a frame that never arrived.
|
||||
- name: test
|
||||
run: pnpm test
|
||||
|
||||
# What the UI census holds. The gate reads structure — which inputs reach
|
||||
# a mutating endpoint, and which of those the product names an action for
|
||||
# — so UNDECLARED_MUTATION == 0 is a fact about the tree, not a lint over
|
||||
# how anything is spelled.
|
||||
- name: census invariants
|
||||
run: pnpm census > /dev/null
|
||||
|
||||
# And that the analyzer still judges the way it did. _fx is a corpus
|
||||
# written to exercise the rules, so its reports move only when a judgement
|
||||
# moves; the product's name every root by file and line and are stale the
|
||||
# moment a button moves, which is why they are not frozen.
|
||||
- name: census fixture reports have not drifted
|
||||
run: pnpm census:verify
|
||||
|
||||
# `build` runs tsc -b before vite, so this also proves the project
|
||||
# references resolve — typecheck alone uses --noEmit and does not.
|
||||
- name: build
|
||||
run: pnpm build
|
||||
|
||||
# The published bundle must not carry MockPort's scripted fixtures: a
|
||||
# release that cannot reach the kernel has to say so, not act out a fake
|
||||
# session. main.tsx guards the import behind import.meta.env.DEV, and this
|
||||
# is what proves the guard still holds.
|
||||
- name: no fixtures in the bundle
|
||||
run: |
|
||||
if grep -rl 网关瞬时态 dist/ 2>/dev/null; then
|
||||
echo "::error::MockPort fixtures reached the production bundle - check the import.meta.env.DEV guard in main.tsx"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
desktop:
|
||||
name: desktop (${{ matrix.os }})
|
||||
runs-on: ${{ matrix.os }}
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
os: [ubuntu-22.04, windows-latest, macos-latest]
|
||||
defaults:
|
||||
run:
|
||||
working-directory: desktop
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
|
||||
- uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7
|
||||
with:
|
||||
go-version-file: desktop/go.mod
|
||||
cache: true
|
||||
cache-dependency-path: desktop/go.sum
|
||||
|
||||
# The desktop module's Go tests: the elevated update helper, the
|
||||
# release-side tools and the Windows uninstall planner. The code this
|
||||
# gate covers is what replaces a running application on a user's disk.
|
||||
# The module is CGO-free since the Wails shell retired, so no platform
|
||||
# carries build dependencies of its own.
|
||||
# Same command as `make studio-test`; this job is already in desktop/.
|
||||
- name: go test
|
||||
run: go test ./...
|
||||
|
||||
# The host sits in the main module and nothing else builds it, so its
|
||||
# dependency closure is checked here: it must stay CGO-free and
|
||||
# platform-neutral or the Electron bundle cannot ship it. Cross-compiling
|
||||
# is host-independent, so once is enough.
|
||||
- name: host stays pure Go and platform-neutral
|
||||
if: runner.os == 'Linux'
|
||||
working-directory: .
|
||||
run: |
|
||||
for os in windows linux darwin; do
|
||||
CGO_ENABLED=0 GOOS=$os GOARCH=amd64 go build -o /dev/null ./cmd/reasonix-studio-host
|
||||
done
|
||||
|
||||
# The packaging contract, on real artifacts rather than on the config that
|
||||
# describes them. Unsigned on purpose: signing and notarization are a
|
||||
# distribution concern and belong to the release workflow, and treating their
|
||||
# absence as "packaging unverified" is what left macOS and Linux resting on a
|
||||
# cross-compile while only Windows had ever produced a package.
|
||||
package:
|
||||
name: package (${{ matrix.os }})
|
||||
runs-on: ${{ matrix.os }}
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- os: macos-latest
|
||||
arch: arm64
|
||||
platform: darwin
|
||||
- os: ubuntu-22.04
|
||||
arch: amd64
|
||||
platform: linux
|
||||
- os: windows-latest
|
||||
arch: amd64
|
||||
platform: windows
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
|
||||
- uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7
|
||||
with:
|
||||
go-version-file: desktop/go.mod
|
||||
cache: true
|
||||
cache-dependency-path: desktop/go.sum
|
||||
|
||||
- uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413 # v6.1.0
|
||||
with:
|
||||
version: 10
|
||||
run_install: false
|
||||
|
||||
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
|
||||
with:
|
||||
node-version: "24"
|
||||
cache: pnpm
|
||||
cache-dependency-path: desktop/electron/pnpm-lock.yaml
|
||||
|
||||
# electron-builder shells out to dpkg-deb through fakeroot for the .deb.
|
||||
- name: Install Linux packaging deps
|
||||
if: runner.os == 'Linux'
|
||||
run: sudo apt-get update && sudo apt-get install -y fakeroot
|
||||
|
||||
- name: Build the SPA
|
||||
working-directory: desktop/frontend-next
|
||||
run: pnpm install --frozen-lockfile && pnpm build
|
||||
|
||||
- name: Build the kernel the shell speaks to
|
||||
working-directory: desktop/electron
|
||||
run: pnpm install --frozen-lockfile && pnpm run build:host
|
||||
|
||||
# The .deb's absolute-path payload. A no-op off Linux, so it is not
|
||||
# guarded here: the one place it must not be forgotten is the one place
|
||||
# it does anything.
|
||||
- name: Stage the Linux package payload
|
||||
working-directory: desktop/electron
|
||||
run: pnpm run build:linux-payload
|
||||
|
||||
# No identity on a runner, and none wanted here: auto-discovery would make
|
||||
# this job fail for the one reason it is not testing.
|
||||
- name: Package
|
||||
working-directory: desktop/electron
|
||||
env:
|
||||
CSC_IDENTITY_AUTO_DISCOVERY: "false"
|
||||
run: pnpm run build:dist
|
||||
|
||||
# The release vocabulary is GOARCH. electron-builder's own is not, and the
|
||||
# rename that reconciles them runs here rather than in the manifest.
|
||||
- name: Artifacts carry the release architecture
|
||||
run: |
|
||||
ls -la dist
|
||||
if ls dist | grep -q -- '-x64'; then
|
||||
echo "::error::an artifact still names x64; studio-manifest reads names as GOARCH"
|
||||
exit 1
|
||||
fi
|
||||
if ! ls dist | grep -q -- "-${{ matrix.platform }}-${{ matrix.arch }}"; then
|
||||
echo "::error::no artifact named for ${{ matrix.platform }}-${{ matrix.arch }}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# The update channel is the signed manifest. A second answer to which
|
||||
# version is current is not a convenience. The .blockmap is the same
|
||||
# channel's differential index and carries the release prefix, so it also
|
||||
# inflates the count the release uses to refuse a partial publish.
|
||||
- name: No electron-updater metadata
|
||||
run: |
|
||||
if ls dist | grep -qiE '^(latest|beta|alpha).*\.yml$'; then
|
||||
echo "::error::electron-updater metadata was published beside the manifest channel"
|
||||
exit 1
|
||||
fi
|
||||
if ls dist | grep -q '\.blockmap$'; then
|
||||
echo "::error::a .blockmap ships under the release prefix; set differentialPackage: false"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# A binary inside app.asar cannot be spawned and the kernel serves the SPA
|
||||
# off disk, so both have to be resources. Found rather than asserted at a
|
||||
# path, because the unpacked directory is named by the builder.
|
||||
- name: The kernel and the page are resources, not archive members
|
||||
run: |
|
||||
host=$(find dist -type f -path '*/bin/reasonix-studio-host*' ! -path '*app.asar*' | head -1)
|
||||
page=$(find dist -type f -path '*/frontend-next/dist/index.html' ! -path '*app.asar*' | head -1)
|
||||
echo "kernel: ${host:-<missing>}"
|
||||
echo "page: ${page:-<missing>}"
|
||||
test -n "$host" -a -n "$page"
|
||||
|
||||
# Read off the package rather than off the config that describes it. Both
|
||||
# of these went wrong the same way once: electron-builder names a package
|
||||
# after package.json unless told, and fpm records the owner it finds on
|
||||
# disk. Either one leaves a Studio that cannot update itself on Linux and
|
||||
# says nothing about it until a user asks.
|
||||
- name: The .deb is Studio's own package and carries what updates it
|
||||
if: runner.os == 'Linux'
|
||||
run: |
|
||||
deb=$(ls dist/ReasonixStudio-linux-*.deb)
|
||||
name=$(dpkg-deb -f "$deb" Package)
|
||||
if [ "$name" != "reasonix-studio" ]; then
|
||||
echo "::error::the package is $name; dpkg holds that beside the installed reasonix-studio instead of upgrading it, and update.StudioLine asks for it by name"
|
||||
exit 1
|
||||
fi
|
||||
dpkg-deb -c "$deb" > contents.txt
|
||||
# The path is the polkit action's exec.path annotation, so it is the
|
||||
# only one authorization is granted for; root because polkit will not
|
||||
# run a helper the invoking user could have written. Either spelling
|
||||
# of root: dpkg-deb prints the numeric ids it cannot map, and which
|
||||
# one it prints is a property of the machine reading the package
|
||||
# rather than of the package.
|
||||
if ! grep -qE '^-rwxr-xr-x (root/root|0/0) +[0-9]+ .* \./usr/lib/reasonix-studio/reasonix-studio-update-helper$' contents.txt; then
|
||||
echo "::error::no root-owned executable update helper at the path the polkit action names"
|
||||
grep reasonix-studio-update-helper contents.txt || echo "(not in the package at all)"
|
||||
exit 1
|
||||
fi
|
||||
if ! grep -q './usr/share/polkit-1/actions/io.reasonix.studio.update.policy$' contents.txt; then
|
||||
echo "::error::the polkit action is not in the package, so pkexec has nothing to authorize against"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# The Electron shell sets the package's glibc floor; a Go binary linked
|
||||
# with cgo would raise it to the runner's own.
|
||||
- name: The .deb's Go binaries are statically linked
|
||||
if: runner.os == 'Linux'
|
||||
run: |
|
||||
deb=$(ls dist/ReasonixStudio-linux-*.deb)
|
||||
dpkg-deb -x "$deb" "$RUNNER_TEMP/deb"
|
||||
host=$(find "$RUNNER_TEMP/deb" -type f -path '*/resources/bin/reasonix-studio-host')
|
||||
test -n "$host"
|
||||
node desktop/electron/packaging/elf.js "$host" "$RUNNER_TEMP/deb/usr/lib/reasonix-studio/reasonix-studio-update-helper"
|
||||
|
||||
# Recorded, not asserted: the real bundle layout, so later work argues
|
||||
# from what electron-builder produced rather than from its documentation.
|
||||
- name: Record the macOS bundle layout
|
||||
if: runner.os == 'macOS'
|
||||
run: |
|
||||
app=$(find dist -maxdepth 2 -name '*.app' | head -1)
|
||||
echo "app: $app"
|
||||
ls "$app/Contents"
|
||||
ls "$app/Contents/Resources" | head -20
|
||||
ls "$app/Contents/Frameworks" | head -20
|
||||
|
||||
# The end of the contract: not that the name parses, but that the manifest
|
||||
# keys the artifact under the platform this runner actually is. The tool
|
||||
# writes latest.json into the directory and logs to stdout, so the check
|
||||
# reads the file -- captured output would match its own progress line.
|
||||
- name: studio-manifest resolves the platform this runner is
|
||||
working-directory: desktop
|
||||
env:
|
||||
GITHUB_REPOSITORY: ${{ github.repository }}
|
||||
run: |
|
||||
go run ./cmd/studio-manifest ../dist v0.0.0-ci v0.0.0-ci
|
||||
cat ../dist/latest.json
|
||||
if ! grep -q '"${{ matrix.platform }}-${{ matrix.arch }}"' ../dist/latest.json; then
|
||||
echo "::error::the manifest has no entry for ${{ matrix.platform }}-${{ matrix.arch }}"
|
||||
exit 1
|
||||
fi
|
||||
@@ -13,8 +13,8 @@ jobs:
|
||||
update:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
|
||||
- uses: actions/checkout@v7
|
||||
- uses: actions/setup-node@v7
|
||||
with:
|
||||
node-version: '22'
|
||||
- name: Update README contributor tables
|
||||
@@ -22,10 +22,9 @@ jobs:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: node scripts/update-acknowledgments.mjs
|
||||
- name: Open pull request
|
||||
uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1 # v8
|
||||
uses: peter-evans/create-pull-request@v8
|
||||
with:
|
||||
# GITHUB_TOKEN-authored pull requests never trigger CI, so required checks would never report.
|
||||
token: ${{ secrets.ACKNOWLEDGMENTS_TOKEN || secrets.GITHUB_TOKEN }}
|
||||
token: ${{ secrets.GITHUB_TOKEN }}
|
||||
branch: chore/update-acknowledgments
|
||||
delete-branch: true
|
||||
title: 'Update acknowledgments contributors / 更新致谢贡献者'
|
||||
|
||||
@@ -24,10 +24,10 @@ jobs:
|
||||
if: github.repository == 'esengine/DeepSeek-Reasonix'
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
|
||||
- uses: actions/setup-node@v7
|
||||
with:
|
||||
node-version: '22'
|
||||
- name: Test chart generator
|
||||
|
||||
@@ -1,29 +0,0 @@
|
||||
name: 1.x nightly monitoring
|
||||
|
||||
# A schedule runs only from the default branch. These monitors measure the 1.x
|
||||
# desktop app, which lives on main-v2 alone, so their nightly runs are started
|
||||
# from here on that branch. Their push and pull_request gates run on main-v2 as
|
||||
# before.
|
||||
on:
|
||||
schedule:
|
||||
- cron: "17 3 * * *"
|
||||
- cron: "43 4 * * *"
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
actions: write
|
||||
|
||||
jobs:
|
||||
dispatch:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: App memory screening
|
||||
if: github.event_name == 'workflow_dispatch' || github.event.schedule == '17 3 * * *'
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
run: gh workflow run app-memory.yml --repo "$GITHUB_REPOSITORY" --ref main-v2
|
||||
- name: Diagnostic overhead monitoring
|
||||
if: github.event_name == 'workflow_dispatch' || github.event.schedule == '43 4 * * *'
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
run: gh workflow run diagnostic-overhead.yml --repo "$GITHUB_REPOSITORY" --ref main-v2
|
||||
+8
-25
@@ -1,6 +1,4 @@
|
||||
# Build artifacts. A `go build` of one command writes its binary into the
|
||||
# working directory, so every main package in the root module can drop its own
|
||||
# name here; repolint's build-artifact rule holds the two lists together.
|
||||
# Build artifacts
|
||||
/bin/
|
||||
/dist/
|
||||
/stage/
|
||||
@@ -9,22 +7,11 @@
|
||||
/reasonix.exe
|
||||
/e2ebench
|
||||
/e2ebench.exe
|
||||
/context-retrieval
|
||||
/compaction
|
||||
/context-maintenance-e2e
|
||||
/runtime-resume
|
||||
/extension-protocol-gen
|
||||
/reasonix-plugin-example
|
||||
/reasonix-studio-host
|
||||
/signpath-contract
|
||||
/studio-delta
|
||||
/covergate
|
||||
/pricecheck
|
||||
/repolint
|
||||
/workgroupassay
|
||||
/write_heartbeat
|
||||
/catalog-detector
|
||||
/tool-surface-cache
|
||||
|
||||
# Website (Astro) build output
|
||||
/site/dist/
|
||||
/site/.astro/
|
||||
/site/_preview.png
|
||||
|
||||
# Go test/build scratch
|
||||
*.test
|
||||
@@ -76,6 +63,7 @@ desktop/**/.reasonix/
|
||||
Thumbs.db
|
||||
.idea/
|
||||
.vscode/
|
||||
/site/public/av/
|
||||
|
||||
benchmarks/context-maintenance-e2e/run/
|
||||
|
||||
@@ -85,12 +73,7 @@ temp/
|
||||
# Official theme art intermediate outputs / caches
|
||||
scripts/official-theme-art/out/
|
||||
scripts/official-theme-art/__pycache__/
|
||||
site/.generated/
|
||||
|
||||
# Committed pointer at REASONIX.md; overrides a global CLAUDE.md ignore rule.
|
||||
!/CLAUDE.md
|
||||
|
||||
# Go build output left in the tree by `go build ./tools/...`. The ~ form is
|
||||
# what a linker leaves behind when it cannot replace a binary that is still
|
||||
# running, which on Windows is every rebuild while Studio is open.
|
||||
*.exe
|
||||
*.exe~
|
||||
+1
-1
@@ -16,7 +16,7 @@ builds:
|
||||
ldflags:
|
||||
# Inject version identity for `reasonix version --verbose/--json` (Integration D/E).
|
||||
# buildTimeUTC is the goreleaser run time (UTC), not vcs.time.
|
||||
- -s -w -X main.version={{ .Tag }} -X main.gitCommit={{ .ShortCommit }} -X main.buildTimeUTC={{ .Date }} -X reasonix/internal/tools/productdocs.linkedVersion={{ .Tag }} -X reasonix/internal/tools/productdocs.linkedRevision={{ .Commit }}
|
||||
- -s -w -X main.version={{ .Tag }} -X main.gitCommit={{ .ShortCommit }} -X main.buildTimeUTC={{ .Date }} -X reasonix/internal/productdocs.linkedVersion={{ .Tag }} -X reasonix/internal/productdocs.linkedRevision={{ .Commit }}
|
||||
goos: [darwin, linux, windows]
|
||||
goarch: [amd64, arm64]
|
||||
|
||||
|
||||
@@ -0,0 +1,13 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<!-- Canary-only configuration. SignPath test certificates intentionally do not
|
||||
chain to a Windows trusted root, so authenticode-verify cannot be used for
|
||||
the already test-signed payload. The workflow verifies that every payload
|
||||
executable is signed and hash-identical to the rebuilt portable package.
|
||||
Stable and RC releases use windows-installer-v2 instead. -->
|
||||
<artifact-configuration xmlns="http://signpath.io/artifact-configuration/v1">
|
||||
<zip-file>
|
||||
<pe-file path="*installer*.exe">
|
||||
<authenticode-sign />
|
||||
</pe-file>
|
||||
</zip-file>
|
||||
</artifact-configuration>
|
||||
@@ -0,0 +1,30 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<!-- Sign the finished NSIS container only when the exact payload used to build it
|
||||
is present and already Authenticode-signed. GitHub Actions wraps this flat
|
||||
bundle in a zip before SignPath receives it. Keep this versioned separately
|
||||
so provisioning it cannot break releases still using windows-installer. -->
|
||||
<artifact-configuration xmlns="http://signpath.io/artifact-configuration/v1">
|
||||
<zip-file>
|
||||
<pe-file path="*installer*.exe">
|
||||
<authenticode-sign />
|
||||
</pe-file>
|
||||
<pe-file path="reasonix-desktop.exe">
|
||||
<authenticode-verify />
|
||||
</pe-file>
|
||||
<pe-file path="reasonix-guard.exe">
|
||||
<authenticode-verify />
|
||||
</pe-file>
|
||||
<pe-file path="reasonix-launcher.exe">
|
||||
<authenticode-verify />
|
||||
</pe-file>
|
||||
<pe-file path="reasonix-update-helper.exe">
|
||||
<authenticode-verify />
|
||||
</pe-file>
|
||||
<pe-file path="reasonix-cli.exe">
|
||||
<authenticode-verify />
|
||||
</pe-file>
|
||||
<pe-file path="reasonix-uninstall.exe">
|
||||
<authenticode-verify />
|
||||
</pe-file>
|
||||
</zip-file>
|
||||
</artifact-configuration>
|
||||
@@ -0,0 +1,13 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<!-- GitHub Actions wraps every uploaded artifact in a zip, so the installer arrives
|
||||
zipped even though it is a single file — match and sign the .exe inside. -->
|
||||
<artifact-configuration xmlns="http://signpath.io/artifact-configuration/v1">
|
||||
<zip-file>
|
||||
<pe-file-set>
|
||||
<include path="*installer*.exe" min-matches="1" max-matches="1" />
|
||||
<for-each>
|
||||
<authenticode-sign />
|
||||
</for-each>
|
||||
</pe-file-set>
|
||||
</zip-file>
|
||||
</artifact-configuration>
|
||||
@@ -0,0 +1,25 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<!-- Sign every executable that the NSIS installer writes to disk. GitHub Actions
|
||||
wraps the flat payload directory in this zip before SignPath receives it. -->
|
||||
<artifact-configuration xmlns="http://signpath.io/artifact-configuration/v1">
|
||||
<zip-file>
|
||||
<pe-file path="reasonix-desktop.exe">
|
||||
<authenticode-sign />
|
||||
</pe-file>
|
||||
<pe-file path="reasonix-guard.exe">
|
||||
<authenticode-sign />
|
||||
</pe-file>
|
||||
<pe-file path="reasonix-launcher.exe">
|
||||
<authenticode-sign />
|
||||
</pe-file>
|
||||
<pe-file path="reasonix-update-helper.exe">
|
||||
<authenticode-sign />
|
||||
</pe-file>
|
||||
<pe-file path="reasonix-cli.exe">
|
||||
<authenticode-sign />
|
||||
</pe-file>
|
||||
<pe-file path="reasonix-uninstall.exe">
|
||||
<authenticode-sign />
|
||||
</pe-file>
|
||||
</zip-file>
|
||||
</artifact-configuration>
|
||||
@@ -3,17 +3,17 @@ project_slug: DeepSeek-Reasonix
|
||||
signing_policy_slug: release-signing
|
||||
repository_url: https://github.com/esengine/DeepSeek-Reasonix.git
|
||||
allowed_branch_names:
|
||||
- studio
|
||||
- main-v2
|
||||
allowed_build_definitions:
|
||||
- .github/workflows/release-studio.yml
|
||||
- .github/workflows/studio-certum-signing-smoke.yml
|
||||
- .github/workflows/release-stable.yml
|
||||
- .github/workflows/release-desktop.yml
|
||||
fingerprint_files:
|
||||
- .github/workflows/release-studio.yml
|
||||
- .github/workflows/studio-certum-signing-smoke.yml
|
||||
- .github/actions/setup-certum/action.yml
|
||||
- scripts/sign-certum.ps1
|
||||
- .github/workflows/release-stable.yml
|
||||
- .github/workflows/release-stable-trigger.yml
|
||||
- .github/workflows/release-desktop.yml
|
||||
- .signpath/artifact-configurations/windows-payload.xml
|
||||
- .signpath/artifact-configurations/windows-installer-v2.xml
|
||||
- scripts/complete-signpath-request.ps1
|
||||
- scripts/package-windows-desktop.sh
|
||||
- scripts/verify-windows-authenticode.ps1
|
||||
- scripts/verify-windows-package.ps1
|
||||
- scripts/windows-signing-lib.ps1
|
||||
- desktop/electron/electron-builder.yml
|
||||
- cmd/signpath-contract/main.go
|
||||
@@ -0,0 +1 @@
|
||||
v2.13.0
|
||||
@@ -1,6 +0,0 @@
|
||||
# Reasonix
|
||||
|
||||
The project's standing instructions live in `REASONIX.md` — one file, read by
|
||||
both Reasonix and Codex. Add rules there, never here.
|
||||
|
||||
@REASONIX.md
|
||||
+362
@@ -0,0 +1,362 @@
|
||||
# Changelog
|
||||
|
||||
All notable changes to the Go line (Reasonix 1.0+) are recorded here. The legacy
|
||||
`0.x` TypeScript history lives on the [`v1`](https://github.com/esengine/DeepSeek-Reasonix/tree/v1)
|
||||
branch.
|
||||
|
||||
## Unreleased
|
||||
|
||||
### Fixed
|
||||
|
||||
- **v1.24.2 session snapshot & recovery root fix:** Keep PR #7982's WAL/CAS/lease
|
||||
safety foundation, but replace process-level "I hold a lease" ownership with a
|
||||
generation-bound `SessionWriteAuthority`. Same-revision tool-preview/load
|
||||
reshapes no longer false-diverge; recovery files are bounded to one path per
|
||||
writer/lineage; empty checkpoints heal from their own WAL; projection lineage
|
||||
rebinds across upgrade/model switch and inherits across recovery forks without
|
||||
changing provider-visible prompt bytes. Catalog upgrades to disposable
|
||||
`session-catalog/v3.sqlite` with recovery lineage roles
|
||||
(`normal|covered_copy|adopted|diverged`); covered idle copies move to the
|
||||
recoverable `.trash` using a 15-minute idle threshold applied on two early
|
||||
sweeps (at startup and ~20 minutes later), then a 24-hour threshold on the
|
||||
6-hour background ticker; independent diverged branches stay and are listed
|
||||
for user choice. v1/v2 catalogs are
|
||||
left byte-unchanged for coexistence/downgrade.
|
||||
**v1.24.1** only hid/reclaimed already-created covered copies and fixed Windows
|
||||
flash-window startup; **v1.24.2** stops the misclassification source and repairs
|
||||
existing user directories without rewriting authoritative JSONL/WAL/sidecar data.
|
||||
|
||||
- Goal now runs continuously by default: the former 16-round per-Run boundary,
|
||||
10/20/40 cross-Run quotas, default wall-clock budget, and numeric
|
||||
no-progress/Todo-stall pauses no longer stop valid work. Progress guards still
|
||||
detect repeated host outcomes and zero-evidence work, but redirect the model
|
||||
to re-plan instead of producing `goal_run_budget` or `goal_stuck`. Explicit
|
||||
`[agent].goal_token_budget`, `--max-steps`, positive time/cost budgets, manual
|
||||
pause/stop, genuine user/external blockers, and evaluator fail-closed behavior
|
||||
remain available. The Goal token budget defaults to `0` (off); resuming its
|
||||
`budget_spend` pause grants a fresh slice without clearing cumulative usage.
|
||||
Goal status reports turns, provider requests, tokens, the optional configured
|
||||
token threshold, and cumulative active work time. Bot `max_steps` also
|
||||
defaults to `0` (continuous), while positive user configuration is enforced.
|
||||
|
||||
- Removed numeric Goal pauses in existing sidecars automatically normalize to
|
||||
`running` without sending a model request. Active Goal sidecars write
|
||||
`turnsLimit: -1` as a downgrade-safe unlimited sentinel while public runtime
|
||||
APIs retain deprecated limit fields as `0`. The migration preserves unknown
|
||||
fields, todos, checkpoints, usage, evidence, and historical metadata.
|
||||
|
||||
- Goal is now the sole long-task runtime. Historical AutoResearch sidecars
|
||||
migrate transactionally into Goals with research compatibility metadata. Invalid archives block
|
||||
fail closed and remain read-only, retaining the task id and compatibility mode
|
||||
for a restart or `/goal resume` retry; successful Goal-only sidecars omit the
|
||||
old task id and write an explicit downgrade fence so previous readers cannot
|
||||
reactivate the removed runtime.
|
||||
|
||||
- Context-dependent workflow tools now share one host-side execution boundary.
|
||||
Goal, Plan sign-off, and background-job calls cannot reach permissions,
|
||||
hooks, leases, or Execute outside their owning context; mixed batches execute
|
||||
valid calls once and stop safely after one repair. Child agents also isolate
|
||||
inherited Goal, Jobs, and live memory queues, while persisted tool identity
|
||||
records the effective child schema projection.
|
||||
|
||||
- **Issue #7575:** Linux Bash under bubblewrap no longer mounts a fresh empty
|
||||
`--tmpfs /tmp` on every call. Consecutive commands in the same logical session
|
||||
now share a private temporary directory (bound at `/tmp` on Linux, exported via
|
||||
`TMPDIR`/`TMP`/`TEMP` on all platforms) without exposing the host public
|
||||
temporary root. `/new`, `/clear`, resume of another session, and branch
|
||||
switches rotate the directory; model/settings hot rebuilds keep it. Sub-agent
|
||||
runs get independent directories. Temporary files are not durable across process
|
||||
restarts.
|
||||
|
||||
### Added
|
||||
|
||||
- Added `[ui].show_turn_usage` so CLI/TUI users can hide per-request token and
|
||||
cost receipts from transcript scrollback without disabling usage accounting.
|
||||
|
||||
## [1.20.0] — 2026-08-05
|
||||
|
||||
Extension kernel, Task Monitor, and safer Goal completion.
|
||||
|
||||
Compact decision surfaces, local decision receipts, unified extension kernel,
|
||||
native Task Monitor, bounded sub-agent progress, Goal fail-closed completion,
|
||||
MiMo and DashScope Responses fixes, SSH remote access simplification, and
|
||||
multiple Desktop stability improvements.
|
||||
|
||||
### Highlights
|
||||
|
||||
- **Unified Extension Kernel and Extension Protocol v1**: Immutable runtime
|
||||
snapshots, fail-atomic reload, Plugin Manifest v1 (prompts, themes, full-trust
|
||||
code runtimes), stable JSON-RPC sidecar protocol, interceptor dispatch,
|
||||
streaming provider adapter, structured UI, and Go SDK.
|
||||
- **Native Task Monitor**: Monitor agent tasks natively in CLI and Desktop with
|
||||
lifecycle semantics and session-scoped summary view.
|
||||
- **Bounded Sub-agent Progress Forwarding**: Forward structured progress for
|
||||
`task`, `parallel_tasks`, and `fleet` without flooding the parent stream.
|
||||
Renders nested lifecycle cards in Desktop and stable per-child transcript
|
||||
slots in CLI.
|
||||
- **Goal Completion Fail-Closed**: Replace free-form Goal footer markers with a
|
||||
stable `update_goal` tool and epoch-scoped per-turn reports. Centralized
|
||||
completion logic with bounded evaluator, progress-aware budgets, and
|
||||
pause/resume controls.
|
||||
- **Ablation Subsystem Switches**: Switch subsystems off behind one shared
|
||||
vocabulary for controlled experiments. Includes planner, subagent, retrieval,
|
||||
evidence, and compaction.
|
||||
- **Benchmark Cost per Solved Task**: Report cost per solved task, tokens per
|
||||
solved, median wall time, and failure-class breakdown in e2e reports.
|
||||
- **Compact Decision Surfaces and Local Receipts**: Compact footer decision-card
|
||||
layout with bounded scroll, dense action rows, and overflow disclosure.
|
||||
Record bounded Ask, approval, and recovery decisions as local transcript
|
||||
receipts.
|
||||
- **Simplified SSH Remote Access**: Remove Remote Workbench protocol and
|
||||
stacks; reuse CLI/Serve remote model. Desktop opens per-host native web child
|
||||
windows via SSH. Keyless remote Serve setup with loopback-only page.
|
||||
- **Model Usage Charts with Primer Palette**: Replace monochrome accent ramp
|
||||
with GitHub Primer data-viz two-set categorical palette. Fix donut overflow
|
||||
on hover and keyboard accessibility.
|
||||
- **Cross-platform Extension and Task Monitor Reliability**: Make
|
||||
content-reference eviction deterministic, reject Unix and Windows absolute
|
||||
plugin paths consistently, stabilize parallel-task cancellation, and restore
|
||||
reliable Windows validation for Task Monitor and remote provider setup.
|
||||
- **MiMo and DashScope Responses Wire Alignment**: Fix multi-turn tool loops,
|
||||
reasoning round-trip, JSON output for MiMo; fix DashScope second-turn 400
|
||||
error, all-zero usage suppression, and vendor-aware cache TTL.
|
||||
- **Desktop Stability Fixes**: Recover stuck updates and legacy WebKit, contain
|
||||
macOS alias repair startup crashes, keep composer overflow stacks readable,
|
||||
and harden account verification and community flows.
|
||||
- **Remote Web Recovery After SSH Drops**: Add integration regression test for
|
||||
SSH drop, forward recovery, and window reload. Document transient outage
|
||||
behavior.
|
||||
- **CI: Auto-minimize Activity-Farming Spam Comments**: Detect and minimize
|
||||
template spam comments from non-contributor accounts based on structural
|
||||
signals.
|
||||
|
||||
### Added
|
||||
|
||||
- Added Extension Protocol v1 and the unified extension kernel: installed or
|
||||
linked sidecars can contribute tools, skills, commands, hooks, MCP servers,
|
||||
providers, interceptors, and structured UI surfaces through a versioned
|
||||
NDJSON contract and the public Go SDK. CLI, Desktop, ACP, and Serve support
|
||||
fail-atomic runtime reloads; Serve also renders extension surfaces and lists
|
||||
extension-hosted providers without exposing credentials.
|
||||
- Added the structured Goal completion protocol: the always-registered
|
||||
`update_goal` tool (continue/complete/blocked with reason and next_action)
|
||||
replaces the `[goal:*]` footer markers. The Goal FSM is now the exclusive
|
||||
cross-turn decision point and validates every complete claim against Delivery
|
||||
readiness; when the model submits no report, an independent bounded evaluator
|
||||
(recovery_model → guardian_model → main model, no tools/history, usage
|
||||
attributed to `goal-evaluator`) judges the turn once, and any evaluator
|
||||
failure pauses the goal instead of continuing silently.
|
||||
- Added Goal budget classes with safe pauses: simple 10 turns / 200k tokens,
|
||||
write 20 turns / 400k tokens, AutoResearch 40 turns / 800k tokens, and a
|
||||
4-turn no-host-verifiable-progress gate. Pauses keep all Goal state; `/goal
|
||||
resume` continues and adds one slice of the current class when the pause was
|
||||
budget-related. `/goal status` shows the full turn/token/no-progress runtime,
|
||||
and `/goal pause` manually suspends a running Goal.
|
||||
- Added the `goalRuntime` nested view to the desktop Meta, the remote protocol
|
||||
(`session/goal/pause` operation, `goalRuntime` DTO on session meta), and the
|
||||
ACP status payload; the desktop Composer goal menu shows the runtime summary
|
||||
with distinct pause/end/resume actions.
|
||||
|
||||
### Changed
|
||||
|
||||
- Delivery no longer retries final-answer readiness with hidden model messages:
|
||||
a plain Delivery run ends on the first unsatisfied final answer and surfaces
|
||||
the recovery card, while a Goal + Delivery run has the Goal FSM absorb the
|
||||
failure and continue under budget with the missing requirements as the next
|
||||
turn's prompt. Historical `[goal:*]` footers are stripped from old transcripts
|
||||
for display only and never participate in state decisions.
|
||||
- Added a **Remote SSH** module (VS Code Remote-SSH style): a user-global
|
||||
`[remote]` host config, `reasonix remote` CLI (add/list/remove/import/test/
|
||||
connect/status/forward/serve/fs) and `/remote` slash command, an SSH transport
|
||||
with trust-on-first-use host-key verification, keepalive + exponential-backoff
|
||||
reconnect, `-L`/`-R` port forwarding, and SFTP file access. `connect`
|
||||
bootstraps a persistent `reasonix serve` on the remote host and tunnels its
|
||||
loopback port so the full agent runs remotely. The desktop app adds a
|
||||
**Settings -> Remote SSH** host manager, a remote file browser/editor, a
|
||||
port-forwarding panel, and a status-bar connection chip. Linux/macOS remotes.
|
||||
- Added `reasonix serve --port-file/--token-file/--pid-file` so a supervised
|
||||
headless serve can bind an ephemeral port and read its auth token from a file
|
||||
(keeping it out of `ps`).
|
||||
- Added an authenticated, loopback-only Provider setup page for `reasonix
|
||||
serve`. A Serve whose selected Provider is missing its API key now remains
|
||||
reachable, stores the submitted key in that host's Reasonix credential file,
|
||||
and rebuilds the active controller in place without restarting Serve.
|
||||
- Added Claude Code-style searchable CLI pickers for models, providers, and
|
||||
sessions, with arrow, Vim, and `Ctrl+P` / `Ctrl+N` navigation.
|
||||
- Added `-p` / `--print`, `text`, `json`, and `stream-json` output modes for
|
||||
one-shot use and automation.
|
||||
- Added session-scoped `--allowed-tools`, repeatable `--add-dir`, Claude-compatible
|
||||
permission modes, flexible `--resume [QUERY]`, and the `--copy` resume escape
|
||||
hatch.
|
||||
- Added `/status` details for the active model, effort, cache, Git state,
|
||||
background jobs, work profile, and provider balance where available.
|
||||
- Remote SSH workspaces now open as a standalone remote web window again.
|
||||
Opening a workspace from the status bar or the Remote Server tab starts or
|
||||
reuses the remote `reasonix serve`, tunnels its loopback port, and opens the
|
||||
Serve web client in a dedicated per-host window. The remote web page uses
|
||||
the provider configuration and API keys on the **remote** host; the desktop
|
||||
no longer exposes its local providers to remote hosts. If the selected remote
|
||||
Provider is missing its API key, the window opens a setup page that saves the
|
||||
key only on that host and then opens the normal Serve UI. The Remote Workbench
|
||||
protocol, its Provider Broker, and the same-window remote projection were
|
||||
removed. Legacy mirror and provider-trust files are not deleted
|
||||
automatically; Settings -> Remote SSH shows a cleanup card when they exist.
|
||||
The hidden `remote attach-workspace`, `remote runtime-workbench`, and
|
||||
`remote workbench-build-id` commands now fail with a pointer to
|
||||
`reasonix remote connect <host> --open`.
|
||||
- Automatic Plan Mode has been retired. Plan Mode is now always entered through
|
||||
an explicit user choice, and the one-time config v5 upgrade removes legacy
|
||||
`agent.auto_plan` and `agent.auto_plan_classifier` values so upgraded users
|
||||
receive the same behavior as new users.
|
||||
- `Shift+Tab` now cycles CLI safe modes from Ask to Auto to Plan, while YOLO
|
||||
remains an independent `Ctrl+Y` toggle.
|
||||
- Model, provider, resume, and approval menus now use consistent row selection;
|
||||
slash completion, help, aliases, and dispatch share one command registry.
|
||||
- The full-screen CLI composer now uses theme-accented borders and a slim bar
|
||||
cursor by default, grows within the available terminal height, scrolls long
|
||||
drafts independently, and preserves selections across explicit image paste.
|
||||
- The persistent CLI footer now uses a responsive, theme-aware layout for
|
||||
interaction state, model, effort, localized work mode, Git identity, cache,
|
||||
context, compaction headroom, jobs, and balance. Narrow terminals move or
|
||||
compact complete groups instead of clipping labels.
|
||||
- CLI clipboard actions now separate terminal-native text paste from explicit
|
||||
image paste: `Ctrl+V` on macOS/Linux, `Alt+V` on Windows, or `/paste-image`.
|
||||
Local transcript copy verifies the native clipboard write, while SSH uses a
|
||||
clearly labelled OSC 52 fallback.
|
||||
- Runtime rebuilds after model, effort, or work-mode changes now preserve the
|
||||
conversation, session permission overrides, additional directories, and
|
||||
session lease ownership.
|
||||
- Agent execution now monitors host-observed Todo progress automatically. A
|
||||
stalled current item receives a recovery nudge after 8 tool-call rounds with
|
||||
no new completion, unique read, command, or mutation, and pauses with saved
|
||||
work after 16. Exact repeats do not renew the progress lease; real work does.
|
||||
Two-level task lists keep the single in_progress contract: the active
|
||||
sub-step is the only current item while its phase stays pending, and the
|
||||
phase becomes in_progress to sign off only after all of its sub-steps are
|
||||
completed. A level-1 sub-step with no phase header above it is rejected.
|
||||
Executor and planner rounds now use automatic progress management. Retired
|
||||
`[agent].max_steps` and `planner_max_steps` keys remain parseable for upgrades,
|
||||
but are ignored and removed by a one-time migration so stale hidden limits
|
||||
cannot truncate new behavior. One-off CLI and unattended bot limits remain.
|
||||
|
||||
### Fixed
|
||||
|
||||
- Fixed long parallel sub-agent research being silently lost when combined
|
||||
`parallel_tasks` or `fleet` answers exceeded the single-tool output limit.
|
||||
Persisted sessions now keep each child transcript independently, return a
|
||||
bounded fair preview plus stable reference for every result, and page full
|
||||
answers through the conversation-scoped `read_subagent_result` tool.
|
||||
- Fixed Remote Workbench failing with only `initialize: workbench-desktop:
|
||||
connection closed` on fresh or cross-platform SSH hosts. Desktop now proves
|
||||
the exact Host CLI Build ID, provisions the matching verified release without
|
||||
requiring remote npm, runs the managed binary explicitly, and preserves a
|
||||
safe structured bootstrap error when the remote command exits early.
|
||||
- Hardened Bash permission reuse for dynamic and indirect execution. Parameter/arithmetic expansions,
|
||||
assignments, redirects, heredocs, and globs can only be remembered as exact
|
||||
`Bash=<literal>` rules, while still using Auto's normal fallback. Nested or
|
||||
indirect execution now requires a human in interactive Ask/Auto and fails
|
||||
closed in headless Ask/Auto/DontAsk. Broad Bash rules, Guardian/hook allows,
|
||||
and the approved-plan window can no longer silently authorize that stricter
|
||||
class; YOLO remains the explicit full-access bypass and sandbox enforcement
|
||||
is unchanged.
|
||||
- Fixed Desktop sessions incorrectly locking themselves during Goal + Delivery
|
||||
mode changes, controller rebuilds, duplicate-tab restore, and background
|
||||
reattachment. Desktop now keeps one process-local runtime owner per canonical
|
||||
session, fences stale controller events by runtime epoch, blocks sends until
|
||||
that runtime is ready, and scopes single-instance ownership to
|
||||
`REASONIX_HOME` instead of the executable path. Switching saved sessions is
|
||||
now transactional: a target build, restore, or lease failure leaves the
|
||||
current controller, lease, path, mode profile, and runtime epoch untouched.
|
||||
- Stabilized the desktop rich composer caret after skill and plugin invocation
|
||||
tags. DOM→model and model→DOM selection mapping now treat invocation chips as
|
||||
zero-length atoms while still counting user text that lands inside the NBSP
|
||||
caret anchor (common on Windows WebView2), restore both selection ends, and
|
||||
recover the insertion point from a `beforeinput` snapshot when the browser
|
||||
temporarily loses selection — so mid-text edits no longer jump to the end.
|
||||
- Isolated the Windows desktop WebView2 shell from stale system proxies, so an
|
||||
exited proxy client cannot leave the embedded UI hidden during startup. If
|
||||
WebView2 still does not reach DOM-ready within 15 seconds, Reasonix now shows
|
||||
the native window with a recovery prompt instead of appearing not to launch.
|
||||
Remote Markdown images are fetched by the backend with Reasonix's proxy
|
||||
configuration instead of bypassing that proxy through the isolated WebView.
|
||||
- Restored captured-mouse right-click text paste, made composer drag selection
|
||||
copy through the verified native clipboard path, and kept non-Git footer
|
||||
telemetry left-aligned without reserving an empty data band.
|
||||
- Restored stateful MCP behavior after the v1.17.13 regression: user-added
|
||||
servers work without extra trust settings (including delivery-mode on-demand
|
||||
calls), repository-provided servers use one exact launch confirmation, and
|
||||
stdio tools reuse one persistent process so browser sessions survive across
|
||||
calls without repeated startup latency. The former trust/reverify/catalog
|
||||
management UI and CLI are removed.
|
||||
- Localized persistent-footer labels and displayed work-mode values in English,
|
||||
Simplified Chinese, and Traditional Chinese, while keeping command arguments
|
||||
stable.
|
||||
- Restored the `0.53` content boundary: model output, tool output, session
|
||||
transcripts, recovery branches, and background-job artifacts retain their
|
||||
original text instead of being rewritten by heuristic secret redaction.
|
||||
Credential masking remains in key-entry summaries and explicit diagnostic or
|
||||
session-cleanup paths. Transcript-bearing session/job sidecars are kept
|
||||
private (`0600`, with private job directories), and the retired
|
||||
`redact_tool_output` setting is removed with a one-time upgrade notice.
|
||||
|
||||
### Notes
|
||||
|
||||
- Full bilingual release notes:
|
||||
<https://reasonix.io/changelog/v1.20.0/> ·
|
||||
[GitHub release](https://github.com/esengine/DeepSeek-Reasonix/releases/tag/desktop-v1.20.0).
|
||||
- The detailed entries below accumulated on `main-v2` after 1.0.0 and shipped
|
||||
across 1.1.0–1.20.0; per-version attribution lives in the per-version release
|
||||
notes linked above.
|
||||
|
||||
## 1.1.0 – 1.19.7
|
||||
|
||||
Per-version entries for the intermediate releases are published in the
|
||||
[bilingual release notes](https://reasonix.io/changelog/) and on the
|
||||
[GitHub releases page](https://github.com/esengine/DeepSeek-Reasonix/releases).
|
||||
|
||||
## [1.0.0] — 2026-06-03
|
||||
|
||||
First stable release — a **ground-up rewrite in Go**. Not an upgrade of the `0.x`
|
||||
TypeScript line; a new codebase that becomes the default (`main-v2`).
|
||||
|
||||
### Highlights
|
||||
|
||||
- **Go kernel**: a single static binary (CGO-free), cross-compiled for
|
||||
darwin/linux/windows on amd64 + arm64. Distributed via npm (the package wraps
|
||||
the native binary), Homebrew (`esengine/reasonix` tap), and release archives;
|
||||
no Node runtime needed to run it.
|
||||
- **Agent core**: the loop, built-in tools (read/write/edit/multi_edit/glob/grep/
|
||||
ls/bash/web_fetch/todo_write), permission gate, sandboxed bash, and the
|
||||
DeepSeek prefix-cache–oriented design.
|
||||
- **Subagents**: `task` plus explore/research/review/security_review skill agents.
|
||||
- **Skills & hooks**: Claude-Code-style skills (`internal/skill`) and hooks
|
||||
(`internal/hook`), symlink-aware and slash-integrated.
|
||||
- **MCP client**: connect external servers over stdio / Streamable HTTP; reads
|
||||
`[[plugins]]` and a Claude-Code `.mcp.json`.
|
||||
- **Code intelligence via CodeGraph**: a tree-sitter symbol/call graph
|
||||
(`codegraph_*` tools) replaces embedding semantic search — no embedding service
|
||||
or API cost. Fetched into a local cache on first use (or `reasonix codegraph
|
||||
install`) and indexed in the background, so installs and startup stay fast.
|
||||
- **Plan mode** with evidence-backed step sign-off (`complete_step`).
|
||||
- **Memory**: `REASONIX.md` hierarchy + auto-memory, folded into the cache-stable
|
||||
prefix.
|
||||
- **ACP** (`reasonix acp`) and an HTTP/SSE server frontend; desktop app (Wails).
|
||||
|
||||
### Fixed
|
||||
|
||||
- **File encoding support restored** — GBK/GB18030 (and other non-UTF-8) files
|
||||
can now be read, edited, and grepped correctly. The v2 rewrite had dropped
|
||||
v1's encoding detection; files in CJK Windows charsets were silently misread
|
||||
or rejected as binary. The read/edit/write round-trip now preserves the
|
||||
original file encoding. (#2637)
|
||||
|
||||
### Notes
|
||||
|
||||
- Versions: the legacy TypeScript line stays in `0.x`; the Go line starts at
|
||||
`1.0.0`. See [docs/MIGRATING.md](docs/MIGRATING.md).
|
||||
- Release archives ship a bare binary; CodeGraph is fetched on first use. Windows
|
||||
support for the fetched runtime is unverified — install `codegraph` on PATH if
|
||||
the auto-fetch doesn't resolve there.
|
||||
|
||||
[1.20.0]: https://github.com/esengine/DeepSeek-Reasonix/releases/tag/desktop-v1.20.0
|
||||
[1.0.0]: https://github.com/esengine/DeepSeek-Reasonix/releases/tag/v1.0.0
|
||||
+42
-46
@@ -5,10 +5,11 @@ everything you need to get started.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- **Go 1.26+** — the version `go.mod` requires
|
||||
- **Go 1.25+** — the project targets the latest stable Go release
|
||||
- **Git** — for version control
|
||||
- **Node.js 24+ and pnpm 10** (optional) — only if you work on Studio
|
||||
(`desktop/`)
|
||||
- **Node.js** (optional) — only if you work on the desktop app (`desktop/`)
|
||||
- **Wails CLI** (desktop only) — run `make wails-install` so the CLI matches
|
||||
`desktop/go.mod`
|
||||
|
||||
## Getting started
|
||||
|
||||
@@ -24,22 +25,22 @@ go test ./... # runs the full test suite
|
||||
| Directory | Purpose |
|
||||
|-----------|---------|
|
||||
| `cmd/reasonix` | CLI entry point |
|
||||
| `internal/runtime/agent` | Agent loop, session, coordinator |
|
||||
| `internal/frontend/cli` | TUI, subcommands, setup wizard |
|
||||
| `internal/session/control` | Transport-agnostic controller |
|
||||
| `internal/contract/config` | TOML configuration loading |
|
||||
| `internal/tools/builtin` | Built-in tools (bash, read_file, …) |
|
||||
| `internal/contract/provider` | Model-backend abstraction |
|
||||
| `internal/model/openai` | OpenAI-compatible provider |
|
||||
| `internal/ext/plugin` | MCP client (stdio + HTTP) |
|
||||
| `internal/contract/event` | Typed event stream |
|
||||
| `internal/ext/hook` | Shell hooks (PreToolUse, …) |
|
||||
| `internal/state/memory` | REASONIX.md hierarchy + auto-memory |
|
||||
| `internal/ext/skill` | Skill discovery from Markdown |
|
||||
| `internal/safety/sandbox` | OS-level sandboxing |
|
||||
| `internal/frontend/serve` | HTTP/SSE server frontend |
|
||||
| `internal/state/checkpoint` | Snapshot-based rewind |
|
||||
| `desktop/` | Studio: the Electron shell, the SPA it serves (separate Go module) |
|
||||
| `internal/agent` | Agent loop, session, coordinator |
|
||||
| `internal/cli` | TUI, subcommands, setup wizard |
|
||||
| `internal/control` | Transport-agnostic controller |
|
||||
| `internal/config` | TOML configuration loading |
|
||||
| `internal/tool/builtin` | Built-in tools (bash, read_file, …) |
|
||||
| `internal/provider` | Model-backend abstraction |
|
||||
| `internal/provider/openai` | OpenAI-compatible provider |
|
||||
| `internal/plugin` | MCP client (stdio + HTTP) |
|
||||
| `internal/event` | Typed event stream |
|
||||
| `internal/hook` | Shell hooks (PreToolUse, …) |
|
||||
| `internal/memory` | REASONIX.md hierarchy + auto-memory |
|
||||
| `internal/skill` | Skill discovery from Markdown |
|
||||
| `internal/sandbox` | OS-level sandboxing |
|
||||
| `internal/serve` | HTTP/SSE server frontend |
|
||||
| `internal/checkpoint` | Snapshot-based rewind |
|
||||
| `desktop/` | Wails-based desktop app (separate Go module) |
|
||||
| `docs/` | Engineering spec, migration guide |
|
||||
|
||||
### Dependency direction
|
||||
@@ -79,10 +80,11 @@ REASONIX_HOME=/tmp/reasonix-dev go run ./cmd/reasonix
|
||||
# REASONIX_HOME=/tmp/reasonix-dev ./bin/reasonix
|
||||
```
|
||||
|
||||
**Studio**
|
||||
**Desktop**
|
||||
|
||||
```bash
|
||||
REASONIX_HOME=/tmp/reasonix-dev-isolated make studio
|
||||
cd desktop && wails build
|
||||
REASONIX_HOME=/tmp/reasonix-dev-isolated build/bin/reasonix-desktop
|
||||
```
|
||||
|
||||
On Windows, use `$env:REASONIX_HOME` in PowerShell or `set REASONIX_HOME=` in
|
||||
@@ -104,17 +106,24 @@ serialization, compaction, or MCP/tool registration need explicit cache review.
|
||||
For these changes:
|
||||
|
||||
- Keep system prompt changes low-frequency and require explicit review.
|
||||
- Say in the PR what the change does to the provider-visible prefix, and why
|
||||
that cost is worth paying.
|
||||
- Fill the PR body `Cache-impact:` line with `none`, `low`, `medium`, or `high`
|
||||
plus the reason.
|
||||
- Fill the PR body `Cache-guard:` line with the focused guard test/command added
|
||||
or run, or explain why an existing guard covers the change.
|
||||
- Fill `System-prompt-review:` when system prompt, memory prefix, output style,
|
||||
or skill index behavior changes.
|
||||
- Prefer focused guard tests near the changed surface; `scripts/cache-guard.sh`
|
||||
remains the broader release-level cache-hit check.
|
||||
|
||||
CI enforces this metadata for cache-sensitive paths so prompt/tool prefix churn
|
||||
is called out before review.
|
||||
|
||||
### Running tests
|
||||
|
||||
```bash
|
||||
go test ./... # all tests
|
||||
go test ./internal/runtime/agent/ -v # verbose, one package
|
||||
go test ./internal/tools/builtin/ -run TestGrep # one test
|
||||
go test ./internal/agent/ -v # verbose, one package
|
||||
go test ./internal/tool/builtin/ -run TestGrep # one test
|
||||
```
|
||||
|
||||
### Code style
|
||||
@@ -139,52 +148,39 @@ ci: add golangci-lint and govulncheck
|
||||
|
||||
## Adding a new built-in tool
|
||||
|
||||
1. Create `internal/tools/builtin/mytool.go`
|
||||
1. Create `internal/tool/builtin/mytool.go`
|
||||
2. Implement the `tool.Tool` interface: `Name()`, `Description()`, `Schema()`, `ReadOnly()`, `Execute()`
|
||||
3. Register via `func init() { tool.RegisterBuiltin(myTool{}) }`
|
||||
4. Add tests in `internal/tools/builtin/builtin_test.go` or a separate `mytool_test.go`
|
||||
4. Add tests in `internal/tool/builtin/builtin_test.go` or a separate `mytool_test.go`
|
||||
5. The tool is automatically available — `main` blank-imports `builtin`
|
||||
|
||||
## Adding a new model provider
|
||||
|
||||
(For MCP tool servers see `internal/ext/plugin` instead — that's a different layer.)
|
||||
(For MCP tool servers see `internal/plugin` instead — that's a different layer.)
|
||||
|
||||
1. Create `internal/contract/provider/myprovider/`
|
||||
1. Create `internal/provider/myprovider/`
|
||||
2. Implement `provider.Provider`: `Name()`, `Stream()`
|
||||
3. Register via `func init() { provider.Register("mykind", New) }`
|
||||
4. The provider is available from config with `kind = "mykind"`
|
||||
|
||||
## Adding i18n strings
|
||||
|
||||
1. Add the field to `internal/base/i18n/i18n.go` (`Messages` struct)
|
||||
2. Add the value in `internal/base/i18n/messages_en.go` and `messages_zh.go`
|
||||
1. Add the field to `internal/i18n/i18n.go` (`Messages` struct)
|
||||
2. Add the value in `internal/i18n/messages_en.go` and `messages_zh.go`
|
||||
3. The `TestCatalogsComplete` test will fail if you miss a locale
|
||||
|
||||
## Submitting changes
|
||||
|
||||
Reasonix ships on two lines (see the
|
||||
[version roadmap](https://github.com/esengine/DeepSeek-Reasonix/discussions/10748)),
|
||||
and each takes different changes:
|
||||
|
||||
| Line | Branch | Takes |
|
||||
| --- | --- | --- |
|
||||
| 2.x | `studio` | Features and fixes: active development |
|
||||
| 1.x | `main-v2` | Bug fixes, provider/API compatibility, release/updater, security, platform stability |
|
||||
|
||||
A fix that matters on both lines lands on `main-v2` and is reviewed for
|
||||
porting to `studio`; do not open the same PR against both.
|
||||
|
||||
1. Fork the repository
|
||||
2. Create a feature branch from the branch your change targets
|
||||
2. Create a feature branch from `main-v2`
|
||||
3. Make your changes with tests
|
||||
4. Ensure `go test ./...` passes
|
||||
5. Ensure `gofmt -l .` shows no changes
|
||||
6. Submit a pull request to that same branch
|
||||
6. Submit a pull request to `main-v2`
|
||||
|
||||
## Reporting issues
|
||||
|
||||
Open an issue on GitHub with:
|
||||
- Which line (1.x or 2.x) and the exact version
|
||||
- Steps to reproduce
|
||||
- Expected vs actual behavior
|
||||
- Go version and OS
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
VERSION := $(shell git describe --tags --always 2>/dev/null | sed 's/^studio-//' || echo dev)
|
||||
VERSION := $(shell git describe --tags --always 2>/dev/null || echo dev)
|
||||
BUILD_TIME_UTC := $(shell date -u +%Y-%m-%dT%H:%M:%SZ)
|
||||
GIT_COMMIT := $(shell git rev-parse --short=12 HEAD 2>/dev/null || echo unknown)
|
||||
LDFLAGS := -s -w \
|
||||
@@ -8,8 +8,9 @@ LDFLAGS := -s -w \
|
||||
GOEXE := $(shell go env GOEXE)
|
||||
# One pin for the Makefile and the CI lint job; see .github/workflows/ci.yml.
|
||||
GOLANGCI_VERSION := $(shell cat .golangci-version)
|
||||
WAILS_VERSION := $(shell tr -d '[:space:]' < .wails-version)
|
||||
|
||||
.PHONY: build vet fmt lint lint-go lint-install lint-cross lint-update coverage-gate coverage-gate-update pricecheck check test studio-test sdk-test sdk-test-race hooks cross clean studio
|
||||
.PHONY: build vet fmt lint lint-go lint-install lint-cross lint-update wails-install test desktop-test desktop-test-short desktop-test-times sdk-test sdk-test-race hooks cross clean
|
||||
|
||||
build:
|
||||
CGO_ENABLED=0 go build -ldflags "$(LDFLAGS)" -o bin/reasonix$(GOEXE) ./cmd/reasonix
|
||||
@@ -18,13 +19,6 @@ build:
|
||||
vet:
|
||||
go vet ./...
|
||||
|
||||
# Build Studio and launch it. The Electron shell is the one this runs: it is a
|
||||
# real GUI app on every platform without being bundled first, so there is one
|
||||
# launch path rather than a macOS one and an everywhere-else one.
|
||||
studio:
|
||||
bash desktop/electron/run-studio.sh
|
||||
|
||||
|
||||
fmt:
|
||||
gofmt -w .
|
||||
|
||||
@@ -33,6 +27,8 @@ fmt:
|
||||
# particular never surface in `go vet`.
|
||||
lint: lint-go
|
||||
go run ./tools/repolint
|
||||
bash scripts/check-wails-pin.sh
|
||||
bash scripts/check-wails-pin.test.sh
|
||||
|
||||
lint-go:
|
||||
@command -v golangci-lint >/dev/null || { echo "golangci-lint not installed; run: make lint-install"; exit 1; }
|
||||
@@ -45,33 +41,12 @@ lint-go:
|
||||
lint-install:
|
||||
CGO_ENABLED=0 go install github.com/golangci/golangci-lint/v2/cmd/golangci-lint@$(GOLANGCI_VERSION)
|
||||
|
||||
# repolint over what the working tree changed. It exists as a target rather than
|
||||
# a command to paste because the host recognizes `make check` as a verification
|
||||
# it can read the result of, while an arbitrary `go run ./tools/...` is an
|
||||
# unknown program it must assume writes — so only this form can be cited as
|
||||
# evidence that the standards gate passed.
|
||||
check:
|
||||
go run ./tools/repolint -changed
|
||||
|
||||
lint-update:
|
||||
go run ./tools/repolint -update
|
||||
|
||||
# The paths the project declared sensitive, held to the coverage they already
|
||||
# have. It runs their tests, so it is its own target rather than part of the
|
||||
# fast `make lint`. -update records the current numbers; lowering one needs
|
||||
# -allow-drop, the way widening a repolint budget does.
|
||||
coverage-gate:
|
||||
go run ./tools/covergate
|
||||
|
||||
coverage-gate-update:
|
||||
go run ./tools/covergate -update
|
||||
|
||||
# The vendors publish their rates on pages this reads and compares the table
|
||||
# against. It reaches the network, so it stays out of lint and out of CI: run it
|
||||
# before touching a rate and when a vendor announces a change. A page it cannot
|
||||
# read exits non-zero — unread never counts as agreement.
|
||||
pricecheck:
|
||||
go run ./tools/pricecheck
|
||||
wails-install:
|
||||
bash scripts/check-wails-pin.sh
|
||||
go install "github.com/wailsapp/wails/v2/cmd/wails@$(WAILS_VERSION)"
|
||||
|
||||
# Linting one GOOS leaves every //go:build windows and darwin file unchecked.
|
||||
lint-cross:
|
||||
@@ -81,15 +56,17 @@ lint-cross:
|
||||
(cd $$2 && GOOS=$$1 golangci-lint run --timeout=5m ./...) || exit 1; \
|
||||
done
|
||||
|
||||
# Every module, because a nested one is where a red test hides longest: the
|
||||
# shell's route allowlist sat broken through a release, and `go test ./...` from
|
||||
# the root cannot see it.
|
||||
test: studio-test sdk-test
|
||||
test:
|
||||
go test ./...
|
||||
|
||||
# The shell is a nested module, so the root `test` target never reaches it.
|
||||
studio-test:
|
||||
cd desktop && go test ./...
|
||||
desktop-test:
|
||||
cd desktop && go test .
|
||||
|
||||
desktop-test-short:
|
||||
cd desktop && go test -short .
|
||||
|
||||
desktop-test-times:
|
||||
cd desktop && go test -count=1 -json . | python3 ../scripts/desktop-test-times.py
|
||||
|
||||
sdk-test:
|
||||
cd sdk/go && go test ./...
|
||||
|
||||
@@ -21,7 +21,6 @@
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
<a href="https://github.com/esengine/DeepSeek-Reasonix/releases?q=studio-v&expanded=true"><img src="https://img.shields.io/github/v/release/esengine/DeepSeek-Reasonix.svg?filter=studio-v*&include_prereleases&style=flat-square&color=8250df&labelColor=161b22&label=studio%202.x" alt="Reasonix Studio 2.x"/></a>
|
||||
<a href="https://www.npmjs.com/package/reasonix"><img src="https://img.shields.io/npm/v/reasonix.svg?style=flat-square&color=cb3837&labelColor=161b22&logo=npm&logoColor=white" alt="npm version"/></a>
|
||||
<a href="https://github.com/esengine/DeepSeek-Reasonix/actions/workflows/ci.yml"><img src="https://img.shields.io/github/actions/workflow/status/esengine/DeepSeek-Reasonix/ci.yml?style=flat-square&label=ci&labelColor=161b22&logo=githubactions&logoColor=white" alt="CI"/></a>
|
||||
<a href="./LICENSE"><img src="https://img.shields.io/npm/l/reasonix.svg?style=flat-square&color=8b949e&labelColor=161b22" alt="license"/></a>
|
||||
@@ -43,36 +42,18 @@
|
||||
<p align="center"><strong>Open source · MIT · a single Go binary</strong></p>
|
||||
<h3 align="center">A coding agent you can leave running.</h3>
|
||||
<p align="center">One local engine, four ways in — terminal, desktop app, browser, or your editor over ACP. Plan mode, permissions, a workspace sandbox and per-turn checkpoints keep a long autonomous run something you can still read and undo.</p>
|
||||
<p align="center">Maintained by <strong>Huahui Yu</strong>.</p>
|
||||
|
||||
<div align="center">
|
||||
<video src="https://github.com/user-attachments/assets/ab2f3878-e224-4931-8254-060e7695cfb9" controls preload="metadata" width="560"></video>
|
||||
</div>
|
||||
|
||||
<br/>
|
||||
|
||||
> [!IMPORTANT]
|
||||
> **Community · 加入社区** — bilingual Discord for setup help (`#help` / `#求助`), workflow showcases, and feature ideas. → **<https://discord.gg/XF78rEME2D>**
|
||||
|
||||
<br/>
|
||||
|
||||
## Versions
|
||||
|
||||
Reasonix ships on two lines. See the [version roadmap announcement](https://github.com/esengine/DeepSeek-Reasonix/discussions/10748) for the reasoning behind the split.
|
||||
|
||||
| Line | Branch | Status | Get it |
|
||||
| --- | --- | --- | --- |
|
||||
| **Reasonix 2.x** | `studio` (this branch) | Active development, pre-release | [Studio releases](https://github.com/esengine/DeepSeek-Reasonix/releases?q=studio-v&expanded=true) |
|
||||
| **Reasonix 1.x** | [`main-v2`](https://github.com/esengine/DeepSeek-Reasonix/tree/main-v2) | Maintenance / stable | `npm i -g reasonix` · [desktop download](https://reasonix.io/?download=desktop#start) |
|
||||
|
||||
- **Want something stable?** Stay on 1.x. It keeps receiving bug fixes,
|
||||
provider/API compatibility, updater and security fixes; its core architecture
|
||||
is no longer being extended.
|
||||
- **Want the new architecture?** Use 2.x Reasonix Studio. It is still moving
|
||||
quickly; reports are welcome.
|
||||
|
||||
Valuable fixes, tests and behaviour from 1.x are reviewed one by one and ported
|
||||
or reimplemented on 2.x where they fit the new architecture.
|
||||
|
||||
- [Moving from 1.x to 2.x](./docs/MIGRATING.md): what the two lines share, and
|
||||
how to run both on one machine without losing sessions.
|
||||
- [2.x roadmap](./docs/ROADMAP.md): what 2.x still has to deliver, how it ships,
|
||||
and the decisions still open.
|
||||
|
||||
## Features
|
||||
|
||||
- **Config-driven.** Providers, the agent, enabled tools, and plugins are all
|
||||
@@ -92,60 +73,53 @@ or reimplemented on 2.x where they fit the new architecture.
|
||||
|
||||
## Install
|
||||
|
||||
This branch is **Reasonix 2.x**. Its desktop app is Reasonix Studio; the CLI,
|
||||
Studio, and editor integrations all run the same local Reasonix engine.
|
||||
Choose the path that matches how you want to use Reasonix. The CLI/TUI,
|
||||
desktop app, and VS Code extension all use the same local Reasonix engine.
|
||||
|
||||
### Reasonix Studio (2.x)
|
||||
### Path A: CLI / TUI
|
||||
|
||||
Download the package for your platform from the latest
|
||||
[Studio release](https://github.com/esengine/DeepSeek-Reasonix/releases?q=studio-v&expanded=true)
|
||||
(tagged `studio-v2.*`, published as pre-releases while 2.x is in active
|
||||
development):
|
||||
|
||||
| Platform | Package | Architecture |
|
||||
| --- | --- | --- |
|
||||
| macOS | `.dmg` or `.zip` | Apple Silicon (`arm64`) / Intel (`amd64`) |
|
||||
| Windows | Installer `-installer.exe` or portable `.zip` | x64 |
|
||||
| Linux | `.deb` | x64 |
|
||||
|
||||
Every package ships with a `.minisig` signature, and the release carries
|
||||
`SHA256SUMS`. Once installed, Studio updates itself in place.
|
||||
|
||||
The same release also carries the 2.x `reasonix` CLI as archives for
|
||||
`darwin|linux|windows × amd64|arm64`.
|
||||
|
||||
### Reasonix 1.x (stable)
|
||||
|
||||
The 1.x CLI installs through npm on any supported platform, or Homebrew on
|
||||
macOS:
|
||||
Install the native binary through npm on any supported platform, or use
|
||||
Homebrew on macOS:
|
||||
|
||||
```sh
|
||||
npm i -g reasonix # any OS; pulls the prebuilt native binary
|
||||
brew install esengine/reasonix/reasonix # macOS
|
||||
```
|
||||
|
||||
The 1.x desktop app is on the
|
||||
[official download page](https://reasonix.io/?download=desktop#start). Windows
|
||||
installers are code-signed with a Certum code-signing certificate.
|
||||
Prebuilt archives (`darwin|linux|windows × amd64|arm64`) and `SHA256SUMS` are on
|
||||
every [GitHub release](https://github.com/esengine/DeepSeek-Reasonix/releases).
|
||||
|
||||
### VS Code extension
|
||||
### Path B: Desktop app
|
||||
|
||||
The extension does not bundle the CLI; it starts your local `reasonix acp`
|
||||
backend and adds native chat, editor context, tool-call approvals, model
|
||||
selection, and workspace sessions. Install the 1.x CLI first.
|
||||
Use the [official download page](https://reasonix.io/?download=desktop#start)
|
||||
for the latest desktop build.
|
||||
|
||||
| Platform | Package | Architecture |
|
||||
| --- | --- | --- |
|
||||
| macOS | Universal `.dmg` or `.zip` | Apple Silicon / Intel |
|
||||
| Windows | Installer `.exe` or portable `.zip` | x64 / ARM64 |
|
||||
| Linux | `.deb` or `.tar.gz` | x64 |
|
||||
|
||||
Windows installers are code-signed through [SignPath.io](https://signpath.io/)
|
||||
with a free certificate provided by the [SignPath Foundation](https://signpath.org/).
|
||||
|
||||
### Path C: VS Code extension
|
||||
|
||||
Complete Path A first. The extension does not bundle the CLI; it starts your
|
||||
local `reasonix acp` backend and adds native chat, editor context, tool-call
|
||||
approvals, model selection, and workspace sessions.
|
||||
|
||||
- **VS Code:** [install from Visual Studio Marketplace](https://marketplace.visualstudio.com/items?itemName=SivanLiu.reasonix-agent)
|
||||
- **VSCodium / Eclipse Theia:** [install from Open VSX Registry](https://open-vsx.org/extension/SivanLiu/reasonix-agent)
|
||||
- **Extension ID:** `SivanLiu.reasonix-agent` · [source and usage guide](https://github.com/SivanCola/reasonix-vscode)
|
||||
|
||||
### Build from source
|
||||
### Path D: Build from source
|
||||
|
||||
Clone the repository; `studio` builds 2.x and `main-v2` builds 1.x:
|
||||
Clone the repository first:
|
||||
|
||||
```sh
|
||||
git clone https://github.com/esengine/DeepSeek-Reasonix.git
|
||||
cd DeepSeek-Reasonix
|
||||
git switch studio
|
||||
```
|
||||
|
||||
#### CLI
|
||||
@@ -158,27 +132,28 @@ make build # -> bin/reasonix(.exe)
|
||||
make cross # -> dist/ (darwin|linux|windows × amd64|arm64)
|
||||
```
|
||||
|
||||
#### Studio
|
||||
#### Desktop
|
||||
|
||||
Studio additionally requires **Node 24+ and pnpm 10**
|
||||
(`npm install -g pnpm@10`) for the frontend.
|
||||
The desktop build additionally requires:
|
||||
|
||||
- **Node 24+ and pnpm 10** (`npm install -g pnpm@10`) for the frontend
|
||||
- **Wails CLI** matching the shared `.wails-version` pin
|
||||
|
||||
```sh
|
||||
make studio
|
||||
make wails-install
|
||||
cd desktop
|
||||
wails build
|
||||
```
|
||||
|
||||
See the [Studio build guide](desktop/README.md#prerequisites) for platform
|
||||
See the [desktop build guide](desktop/README.md#prerequisites) for platform
|
||||
webview dependencies and Linux build tags.
|
||||
|
||||
## Quick start
|
||||
|
||||
### Reasonix Studio
|
||||
|
||||
Install and launch Studio, then connect a provider and model in the app. No CLI
|
||||
setup is needed.
|
||||
|
||||
### CLI / TUI
|
||||
|
||||
These commands are for the CLI/TUI installed through Path A:
|
||||
|
||||
```sh
|
||||
reasonix setup # configure a provider and model
|
||||
reasonix # start an interactive session
|
||||
@@ -188,6 +163,13 @@ reasonix run "implement the TODOs in main.go"
|
||||
In an interactive session, run `/init` when you want Reasonix to create project
|
||||
instructions.
|
||||
|
||||
### Desktop app
|
||||
|
||||
Download the installer for your platform from the
|
||||
[official download page](https://reasonix.io/?download=desktop#start), install
|
||||
and launch Reasonix, then configure a provider and model in the app. The CLI
|
||||
commands above are not required for the desktop app.
|
||||
|
||||
For advanced CLI usage and configuration, see the **[CLI reference](./docs/CLI.md)**,
|
||||
**[Guide](./docs/GUIDE.md)**, and
|
||||
**[configuration paths](./docs/CONFIG_PATHS.md)**.
|
||||
@@ -199,13 +181,12 @@ For advanced CLI usage and configuration, see the **[CLI reference](./docs/CLI.m
|
||||
- **Features & troubleshooting:** [Subagent profiles](./docs/SUBAGENT_PROFILES.md) ·
|
||||
[Context Engine v2](./docs/SESSION_MEMORY_RETRIEVAL.md) ·
|
||||
[Capability diagnostics](./docs/CAPABILITY_DIAGNOSTICS.md) ·
|
||||
[Recovery and updates](./docs/RECOVERY.md) ·
|
||||
[Recovery and updates](./docs/RECOVERY.md) · [Bot guide](./docs/BOT_GUIDE.md) ·
|
||||
[Checkpoints & rewind](./docs/CHECKPOINTS.md)
|
||||
- **Engineering & migration:** [Spec](./docs/SPEC.md) ·
|
||||
[Task contracts & pause policy](./docs/TASK_CONTRACT.md) ·
|
||||
[Tool contract](./docs/TOOL_CONTRACT.md) · [Moving from 1.x to 2.x](./docs/MIGRATING.md) ·
|
||||
[2.x roadmap](./docs/ROADMAP.md)
|
||||
- **Extension development:** [Skills](./docs/SKILLS.md) · [Extensions](./docs/EXTENSIONS.md) ·
|
||||
[Tool contract](./docs/TOOL_CONTRACT.md) · [Migrating from 0.x](./docs/MIGRATING.md)
|
||||
- **Extension development:** [Extensions](./docs/EXTENSIONS.md) ·
|
||||
[Plugin packages and Manifest v1](./docs/PLUGIN_PACKAGES.md) ·
|
||||
[Extension Protocol](./docs/EXTENSION_PROTOCOL.md) ·
|
||||
[Go SDK and starter](./sdk/go/README.md)
|
||||
@@ -232,10 +213,10 @@ A small list of folks whose work has shaped Reasonix the most — the current to
|
||||
| Contributor | Contributor | Contributor | Contributor |
|
||||
| --- | --- | --- | --- |
|
||||
| [**SivanCola**](https://github.com/SivanCola) | [**esengine**](https://github.com/esengine) | [**ttmouse**](https://github.com/ttmouse) | [**lifu963**](https://github.com/lifu963) |
|
||||
| **reasonix** | [**HUQIANTAO**](https://github.com/HUQIANTAO) | [**GTC2080**](https://github.com/GTC2080) | [**mchenziyi**](https://github.com/mchenziyi) |
|
||||
| [**Li-Charles-One**](https://github.com/Li-Charles-One) | **merge-order-check** | [**light-front-theory**](https://github.com/light-front-theory) | **Yan Li** |
|
||||
| [**eghrhegpe**](https://github.com/eghrhegpe) | **wufengfan** | [**Bernardxu123**](https://github.com/Bernardxu123) | [**HaoyueQin**](https://github.com/HaoyueQin) |
|
||||
| [**CVEngineer66**](https://github.com/CVEngineer66) | [**KHG420**](https://github.com/KHG420) | [**JesonChou**](https://github.com/JesonChou) | [**SuMuxi66**](https://github.com/SuMuxi66) |
|
||||
| **reasonix** | [**HUQIANTAO**](https://github.com/HUQIANTAO) | [**GTC2080**](https://github.com/GTC2080) | [**light-front-theory**](https://github.com/light-front-theory) |
|
||||
| **merge-order-check** | [**Li-Charles-One**](https://github.com/Li-Charles-One) | [**eghrhegpe**](https://github.com/eghrhegpe) | **wufengfan** |
|
||||
| [**CVEngineer66**](https://github.com/CVEngineer66) | [**dependabot\[bot\]**](https://github.com/apps/dependabot) | [**lanshi17**](https://github.com/lanshi17) | [**SuMuxi66**](https://github.com/SuMuxi66) |
|
||||
| [**CnsMaple**](https://github.com/CnsMaple) | [**cyq1017**](https://github.com/cyq1017) | [**JesonChou**](https://github.com/JesonChou) | [**XTLine**](https://github.com/XTLine) |
|
||||
<!-- reasonix-top-contributors:end -->
|
||||
|
||||
Special thanks to [**Bernardxu123**](https://github.com/Bernardxu123) for designing the project logo and intro video.
|
||||
|
||||
+60
-95
@@ -9,11 +9,11 @@
|
||||
·
|
||||
<a href="./docs/GUIDE.zh-CN.md">指南</a>
|
||||
·
|
||||
<a href="./docs/ACP.md">ACP</a>
|
||||
<a href="./docs/ACP.zh-CN.md">ACP</a>
|
||||
·
|
||||
<a href="./docs/EXTENSIONS.md">扩展开发</a>
|
||||
<a href="./docs/EXTENSIONS.zh-CN.md">扩展开发</a>
|
||||
·
|
||||
<a href="./docs/SPEC.md">规格</a>
|
||||
<a href="./docs/SPEC.zh-CN.md">规格</a>
|
||||
·
|
||||
<a href="https://esengine.github.io/DeepSeek-Reasonix/">官方网站</a>
|
||||
·
|
||||
@@ -21,7 +21,6 @@
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
<a href="https://github.com/esengine/DeepSeek-Reasonix/releases?q=studio-v&expanded=true"><img src="https://img.shields.io/github/v/release/esengine/DeepSeek-Reasonix.svg?filter=studio-v*&include_prereleases&style=flat-square&color=8250df&labelColor=161b22&label=studio%202.x" alt="Reasonix Studio 2.x"/></a>
|
||||
<a href="https://www.npmjs.com/package/reasonix"><img src="https://img.shields.io/npm/v/reasonix.svg?style=flat-square&color=cb3837&labelColor=161b22&logo=npm&logoColor=white" alt="npm version"/></a>
|
||||
<a href="https://github.com/esengine/DeepSeek-Reasonix/actions/workflows/ci.yml"><img src="https://img.shields.io/github/actions/workflow/status/esengine/DeepSeek-Reasonix/ci.yml?style=flat-square&label=ci&labelColor=161b22&logo=githubactions&logoColor=white" alt="CI"/></a>
|
||||
<a href="./LICENSE"><img src="https://img.shields.io/npm/l/reasonix.svg?style=flat-square&color=8b949e&labelColor=161b22" alt="license"/></a>
|
||||
@@ -43,30 +42,16 @@
|
||||
<p align="center"><strong>开源 · MIT · 单个 Go 二进制</strong></p>
|
||||
<h3 align="center">可以一直开着跑的编码 Agent。</h3>
|
||||
<p align="center">一套本地引擎,四个入口——终端、桌面端、浏览器,或通过 ACP 接入你的编辑器。计划模式、权限、工作区沙箱与逐轮 checkpoint,让长时间自治运行始终可读、可撤销。</p>
|
||||
<p align="center">维护者:<strong>Huahui Yu</strong>。</p>
|
||||
|
||||
<div align="center">
|
||||
<video src="https://github.com/user-attachments/assets/ab2f3878-e224-4931-8254-060e7695cfb9" controls preload="metadata" width="560"></video>
|
||||
</div>
|
||||
|
||||
<br/>
|
||||
|
||||
> [!IMPORTANT]
|
||||
> **加入社区 · Community** — 双语 Discord,提供安装答疑(`#help` / `#求助`)、工作流展示与功能想法。→ **<https://discord.gg/XF78rEME2D>**
|
||||
|
||||
## 版本
|
||||
|
||||
Reasonix 分为两条版本线,调整的原因见[版本路线公告](https://github.com/esengine/DeepSeek-Reasonix/discussions/10748)。
|
||||
|
||||
| 版本线 | 分支 | 状态 | 获取方式 |
|
||||
| --- | --- | --- | --- |
|
||||
| **Reasonix 2.x** | `studio`(当前分支) | 活跃开发,预发布 | [Studio 发布页](https://github.com/esengine/DeepSeek-Reasonix/releases?q=studio-v&expanded=true) |
|
||||
| **Reasonix 1.x** | [`main-v2`](https://github.com/esengine/DeepSeek-Reasonix/tree/main-v2) | 维护 / 稳定 | `npm i -g reasonix` · [桌面端下载](https://reasonix.io/?download=desktop#start) |
|
||||
|
||||
- **想要稳定**:继续使用 1.x。它会持续收到 Bug 修复、Provider / API 兼容、
|
||||
更新器和安全修复,但不再扩展核心架构。
|
||||
- **想体验新架构**:使用 2.x Reasonix Studio。它仍在快速迭代,欢迎反馈问题。
|
||||
|
||||
1.x 中有价值的修复、测试和行为会逐项 review,适合新架构的会迁移或重新实现到 2.x。
|
||||
|
||||
- [从 1.x 迁移到 2.x](./docs/MIGRATING.md)(英文):两条线共用哪些数据,以及如何在同一台
|
||||
机器上同时使用而不丢会话。
|
||||
- [2.x 路线图](./docs/ROADMAP.md)(英文):2.x 还要交付什么、如何发布、哪些决定尚未做出。
|
||||
|
||||
## 特性
|
||||
|
||||
- **配置驱动**:provider、agent、启用的工具、插件全部在 `reasonix.toml` 中声明,
|
||||
@@ -82,84 +67,58 @@ Reasonix 分为两条版本线,调整的原因见[版本路线公告](https://
|
||||
|
||||
## 安装
|
||||
|
||||
当前分支是 **Reasonix 2.x**,桌面端是 Reasonix Studio;CLI、Studio 和编辑器集成
|
||||
都运行同一套本地 Reasonix 引擎。
|
||||
选择适合你的使用路径。CLI/TUI、桌面端和 VS Code 扩展都使用同一套本地
|
||||
Reasonix 引擎。
|
||||
|
||||
### Reasonix Studio(2.x)
|
||||
### 路径 A:CLI / TUI
|
||||
|
||||
从最新的 [Studio 发布](https://github.com/esengine/DeepSeek-Reasonix/releases?q=studio-v&expanded=true)
|
||||
下载对应平台的安装包(标签为 `studio-v2.*`,2.x 活跃开发期间以预发布形式发布):
|
||||
|
||||
| 平台 | 安装包 | 架构 |
|
||||
| --- | --- | --- |
|
||||
| macOS | `.dmg` 或 `.zip` | Apple Silicon(`arm64`)/ Intel(`amd64`) |
|
||||
| Windows | 安装器 `-installer.exe` 或便携 `.zip` | x64 |
|
||||
| Linux | `.deb` | x64 |
|
||||
|
||||
每个安装包都附带 `.minisig` 签名,发布页另有 `SHA256SUMS`。安装后 Studio 会在
|
||||
应用内自行更新。
|
||||
|
||||
同一个发布页也提供 2.x 的 `reasonix` CLI 归档(`darwin|linux|windows × amd64|arm64`)。
|
||||
|
||||
### Reasonix 1.x(稳定版)
|
||||
|
||||
1.x CLI 在任意支持的平台上都可以通过 npm 安装,macOS 也可以使用 Homebrew:
|
||||
任意支持的平台都可以通过 npm 安装原生二进制;macOS 也可以使用 Homebrew:
|
||||
|
||||
```sh
|
||||
npm i -g reasonix # 任意系统;自动拉取对应平台的原生二进制
|
||||
brew install esengine/reasonix/reasonix # macOS
|
||||
```
|
||||
|
||||
1.x 桌面端请前往[官方下载页](https://reasonix.io/?download=desktop#start)。
|
||||
Windows 安装器使用 Certum 代码签名证书签名。
|
||||
预编译归档(`darwin|linux|windows × amd64|arm64`)和 `SHA256SUMS` 见每个
|
||||
[GitHub release](https://github.com/esengine/DeepSeek-Reasonix/releases)。
|
||||
|
||||
### VS Code 扩展
|
||||
### 路径 B:桌面端
|
||||
|
||||
扩展不内置 CLI,而是启动本机的 `reasonix acp` 后端,并提供原生聊天、编辑器
|
||||
上下文、工具调用审批、模型选择和工作区会话。请先安装 1.x CLI。
|
||||
前往[官方下载页](https://reasonix.io/?download=desktop#start)获取最新桌面版本。
|
||||
|
||||
| 平台 | 安装包 | 架构 |
|
||||
| --- | --- | --- |
|
||||
| macOS | 通用 `.dmg` 或 `.zip` | Apple Silicon / Intel |
|
||||
| Windows | 安装器 `.exe` 或便携 `.zip` | x64 / ARM64 |
|
||||
| Linux | `.deb` 或 `.tar.gz` | x64 |
|
||||
|
||||
Windows 安装器通过 [SignPath.io](https://signpath.io/) 完成代码签名,证书由
|
||||
[SignPath 基金会](https://signpath.org/) 免费提供。
|
||||
|
||||
### 路径 C:VS Code 扩展
|
||||
|
||||
请先完成路径 A。扩展不内置 CLI,而是启动本机的 `reasonix acp` 后端,
|
||||
并提供原生聊天、编辑器上下文、工具调用审批、模型选择和工作区会话。
|
||||
|
||||
- **VS Code:** [从 Visual Studio Marketplace 安装](https://marketplace.visualstudio.com/items?itemName=SivanLiu.reasonix-agent)
|
||||
- **VSCodium / Eclipse Theia:** [从 Open VSX Registry 安装](https://open-vsx.org/extension/SivanLiu/reasonix-agent)
|
||||
- **扩展 ID:** `SivanLiu.reasonix-agent` · [源码与使用说明](https://github.com/SivanCola/reasonix-vscode)
|
||||
|
||||
### 从源码构建
|
||||
|
||||
克隆仓库;`studio` 构建 2.x,`main-v2` 构建 1.x:
|
||||
### 路径 D:从源码构建
|
||||
|
||||
```sh
|
||||
git clone https://github.com/esengine/DeepSeek-Reasonix.git
|
||||
cd DeepSeek-Reasonix
|
||||
git switch studio
|
||||
```
|
||||
|
||||
#### CLI
|
||||
|
||||
CLI 构建需要 **Go 1.25+**。模块固定了 `toolchain`,保持 `GOTOOLCHAIN=auto`
|
||||
即可让 Go 自动下载对应工具链,也可以手动安装。
|
||||
|
||||
```sh
|
||||
make build # -> bin/reasonix(.exe)
|
||||
make cross # -> dist/(darwin|linux|windows × amd64|arm64)
|
||||
```
|
||||
|
||||
#### Studio
|
||||
|
||||
Studio 的前端还需要 **Node 24+ 与 pnpm 10**(`npm install -g pnpm@10`)。
|
||||
|
||||
```sh
|
||||
make studio
|
||||
```
|
||||
|
||||
各平台的 webview 依赖和 Linux 构建标签见 [Studio 构建指南](desktop/README.md#prerequisites)。
|
||||
|
||||
## 快速开始
|
||||
|
||||
### Reasonix Studio
|
||||
|
||||
安装并启动 Studio,然后在应用内连接 provider 和模型即可使用,无需配置 CLI。
|
||||
|
||||
### CLI / TUI
|
||||
|
||||
以下命令仅适用于通过路径 A 安装的 CLI/TUI:
|
||||
|
||||
```sh
|
||||
reasonix setup # 配置 provider 和模型
|
||||
reasonix # 启动交互式会话
|
||||
@@ -168,28 +127,34 @@ reasonix run "把 main.go 里的 TODO 实现掉"
|
||||
|
||||
需要项目指令时,可在交互式会话中运行 `/init`。
|
||||
|
||||
### 桌面端
|
||||
|
||||
从[官方下载页](https://reasonix.io/?download=desktop#start)下载对应系统的安装包,
|
||||
安装并启动 Reasonix,然后在应用内配置 provider 和模型即可使用。桌面端无需执行
|
||||
上面的 CLI 命令。
|
||||
|
||||
CLI 进阶用法和详细配置见 **[CLI 命令参考](./docs/CLI.zh-CN.md)**、
|
||||
**[指南](./docs/GUIDE.zh-CN.md)** 和
|
||||
**[配置路径](./docs/CONFIG_PATHS.md)**。
|
||||
**[配置路径](./docs/CONFIG_PATHS.zh-CN.md)**。
|
||||
|
||||
## 文档
|
||||
|
||||
- **开始使用:** [指南](./docs/GUIDE.zh-CN.md) ·
|
||||
[CLI 命令参考](./docs/CLI.zh-CN.md) · [配置路径](./docs/CONFIG_PATHS.md) ·
|
||||
[ACP 编辑器接入](./docs/ACP.md)
|
||||
- **功能与排障:** [子智能体 Profile](./docs/SUBAGENT_PROFILES.md) ·
|
||||
[Context Engine v2](./docs/SESSION_MEMORY_RETRIEVAL.md) ·
|
||||
[能力诊断](./docs/CAPABILITY_DIAGNOSTICS.md) ·
|
||||
[恢复与安全模式](./docs/RECOVERY.md) ·
|
||||
[Checkpoints 与 rewind](./docs/CHECKPOINTS.md)
|
||||
- **工程与迁移:** [规格](./docs/SPEC.md) ·
|
||||
[任务合约与暂停策略](./docs/TASK_CONTRACT.md) ·
|
||||
[工具合约](./docs/TOOL_CONTRACT.md) ·
|
||||
[从 1.x 迁移到 2.x](./docs/MIGRATING.md) ·
|
||||
[2.x 路线图](./docs/ROADMAP.md)
|
||||
- **扩展开发:** [扩展概览](./docs/EXTENSIONS.md) ·
|
||||
[插件包与 Manifest v1](./docs/PLUGIN_PACKAGES.md) ·
|
||||
[Extension Protocol](./docs/EXTENSION_PROTOCOL.md) ·
|
||||
[CLI 命令参考](./docs/CLI.zh-CN.md) · [配置路径](./docs/CONFIG_PATHS.zh-CN.md) ·
|
||||
[ACP 编辑器接入](./docs/ACP.zh-CN.md)
|
||||
- **功能与排障:** [子智能体 Profile](./docs/SUBAGENT_PROFILES.zh-CN.md) ·
|
||||
[Context Engine v2](./docs/SESSION_MEMORY_RETRIEVAL.zh-CN.md) ·
|
||||
[能力诊断](./docs/CAPABILITY_DIAGNOSTICS.zh-CN.md) ·
|
||||
[恢复与安全模式](./docs/RECOVERY.zh-CN.md) ·
|
||||
[机器人使用指南](./docs/BOT_GUIDE.zh-CN.md) ·
|
||||
[Checkpoints 与 rewind](./docs/CHECKPOINTS.zh-CN.md)
|
||||
- **工程与迁移:** [规格](./docs/SPEC.zh-CN.md) ·
|
||||
[任务合约与暂停策略](./docs/TASK_CONTRACT.zh-CN.md) ·
|
||||
[工具合约](./docs/TOOL_CONTRACT.zh-CN.md) ·
|
||||
[从 0.x 迁移](./docs/MIGRATING.zh-CN.md)
|
||||
- **扩展开发:** [扩展概览](./docs/EXTENSIONS.zh-CN.md) ·
|
||||
[插件包与 Manifest v1](./docs/PLUGIN_PACKAGES.zh-CN.md) ·
|
||||
[Extension Protocol](./docs/EXTENSION_PROTOCOL.zh-CN.md) ·
|
||||
[Go SDK 与 starter](./sdk/go/README.md)
|
||||
|
||||
## Star 趋势
|
||||
@@ -214,10 +179,10 @@ CLI 进阶用法和详细配置见 **[CLI 命令参考](./docs/CLI.zh-CN.md)**
|
||||
| Contributor | Contributor | Contributor | Contributor |
|
||||
| --- | --- | --- | --- |
|
||||
| [**SivanCola**](https://github.com/SivanCola) | [**esengine**](https://github.com/esengine) | [**ttmouse**](https://github.com/ttmouse) | [**lifu963**](https://github.com/lifu963) |
|
||||
| **reasonix** | [**HUQIANTAO**](https://github.com/HUQIANTAO) | [**GTC2080**](https://github.com/GTC2080) | [**mchenziyi**](https://github.com/mchenziyi) |
|
||||
| [**Li-Charles-One**](https://github.com/Li-Charles-One) | **merge-order-check** | [**light-front-theory**](https://github.com/light-front-theory) | **Yan Li** |
|
||||
| [**eghrhegpe**](https://github.com/eghrhegpe) | **wufengfan** | [**Bernardxu123**](https://github.com/Bernardxu123) | [**HaoyueQin**](https://github.com/HaoyueQin) |
|
||||
| [**CVEngineer66**](https://github.com/CVEngineer66) | [**KHG420**](https://github.com/KHG420) | [**JesonChou**](https://github.com/JesonChou) | [**SuMuxi66**](https://github.com/SuMuxi66) |
|
||||
| **reasonix** | [**HUQIANTAO**](https://github.com/HUQIANTAO) | [**GTC2080**](https://github.com/GTC2080) | [**light-front-theory**](https://github.com/light-front-theory) |
|
||||
| **merge-order-check** | [**Li-Charles-One**](https://github.com/Li-Charles-One) | [**eghrhegpe**](https://github.com/eghrhegpe) | **wufengfan** |
|
||||
| [**CVEngineer66**](https://github.com/CVEngineer66) | [**dependabot\[bot\]**](https://github.com/apps/dependabot) | [**lanshi17**](https://github.com/lanshi17) | [**SuMuxi66**](https://github.com/SuMuxi66) |
|
||||
| [**CnsMaple**](https://github.com/CnsMaple) | [**cyq1017**](https://github.com/cyq1017) | [**JesonChou**](https://github.com/JesonChou) | [**XTLine**](https://github.com/XTLine) |
|
||||
<!-- reasonix-top-contributors:end -->
|
||||
|
||||
特别感谢 [**Bernardxu123**](https://github.com/Bernardxu123) 设计的项目 logo和开场视频。
|
||||
|
||||
+39
-345
@@ -9,25 +9,22 @@ agent. It is the Reasonix analog of Claude Code's CLAUDE.md.
|
||||
- Go kernel under `internal/`; each package owns one concern. A package's long
|
||||
explanation belongs in its `doc.go`, not spread across implementation files.
|
||||
- One transport-agnostic `control.Controller` sits behind every frontend (chat
|
||||
TUI, HTTP/SSE serve, Electron desktop). Add behavior to the controller, not a
|
||||
TUI, HTTP/SSE serve, Wails desktop). Add behavior to the controller, not a
|
||||
frontend, so all three inherit it.
|
||||
- Layering (enforced): utility packages import nothing under `reasonix/` except
|
||||
one another — that layer is closed, so a leaf reaching a leaf drags no graph
|
||||
along, while refusing it leaves one judgement copied into several of them; only
|
||||
the frontends `cli`, `serve`, `acp`, `boot` and the hosts `cmd/`, `desktop/`
|
||||
may import `control`; nothing below a frontend may import one. The declared
|
||||
sets live in `tools/repolint/layers.go`.
|
||||
- Layering (enforced): utility packages import nothing under `reasonix/`; only
|
||||
the frontends `cli`, `serve`, `acp`, `bot`, `botruntime`, `boot` and the hosts
|
||||
`cmd/`, `desktop/` may import `control`; nothing below a frontend may import
|
||||
one. The declared sets live in `tools/repolint/layers.go`.
|
||||
- Subagent delegation keeps five concepts apart: a profile says how a worker
|
||||
thinks, `TaskSpec` what this call wants, `CapabilityGrant` what it may touch,
|
||||
`ContextRequest` what it starts from, `SchedulerPolicy` when it runs. Put a
|
||||
field in whichever member decides its value — profiles carry ceilings, never
|
||||
per-call values. `internal/runtime/delegation/profile_boundary_test.go` enforces it.
|
||||
- Cache-first: the system-prompt prefix (base prompt, tools, declared prefix
|
||||
configuration) must stay byte-stable across turns so DeepSeek's automatic
|
||||
prefix cache stays warm. Never mutate it mid-session — ride the turn tail
|
||||
instead (see `control.Compose`), under the rules in *Context projection*.
|
||||
per-call values. `internal/agent/profile_boundary_test.go` enforces it.
|
||||
- Cache-first: the system-prompt prefix (base prompt + tools + memory) must stay
|
||||
byte-stable across turns so DeepSeek's automatic prefix cache stays warm. Never
|
||||
mutate it mid-session — ride the turn tail instead (see `control.Compose`).
|
||||
- Performance features land with an effect test at their final boundary
|
||||
(`internal/assembly/boot/effect_test.go` pattern): assert what actually reaches the
|
||||
(`internal/boot/effect_test.go` pattern): assert what actually reaches the
|
||||
provider request, frontend sink, or trajectory through the real `boot.Build`
|
||||
assembly. Component correctness is not system effectiveness.
|
||||
- A mutex- or atomic-guarded struct is ratcheted on its **scalar** field count
|
||||
@@ -36,100 +33,6 @@ agent. It is the Reasonix analog of Claude Code's CLAUDE.md.
|
||||
the move this blocks — group by lifetime into a named sub-state instead
|
||||
(`agent.perTurnState` is the pattern), which costs one field and removes the
|
||||
whole product.
|
||||
- The context window (projection, compaction, estimation) owns `windowState`;
|
||||
the loop reaches it via `a.window()`, it reaches the loop via `windowHost`.
|
||||
`window-state` fails any other read of `sess.win`.
|
||||
- Judgements read structure, never wording: shell ASTs (`shellparse`), types,
|
||||
contracts, tool schemas. Phrase tables and message sniffing have been retired
|
||||
in batches (task-policy prose, the planner's approval phrases, the executor
|
||||
handoff tables) because each one misfires on real input, and a misfiring
|
||||
judgement inside a gate is worse than no gate. Security allow-lists are the
|
||||
exception and stay. Something that can only be built by matching words does
|
||||
not get built — say what is missing instead of guessing at it.
|
||||
- The other half of that rule: an error a caller must tell apart **carries an
|
||||
identity** — a sentinel, a typed code — never a sentence. Only the producer
|
||||
knows a deadline expired rather than a socket closed, and a message is where
|
||||
that knowledge goes to die: the reader is left matching words, or with
|
||||
nothing. Across HTTP the identity is a dotted code through `refuse`;
|
||||
`refusal-path` fails a new `http.Error` in `internal/frontend/serve`, and the parity
|
||||
test fails a code the frontend cannot say. Inside Go it is `errors.Is` on a
|
||||
sentinel the wrapping preserves; `error-text` fails a new match against an
|
||||
error message, following it one hop into a local because storing the text
|
||||
first is what the direct form turns into.
|
||||
- **Whoever first knows why an operation failed owns its class.** A transport
|
||||
status is not a domain identity, and a display string is not a classification;
|
||||
past that boundary a layer may project the class, never re-derive it. Both
|
||||
directions were live: `catalog.go` answered 400 for a name nobody declared and
|
||||
for a disk that would not write, so a failed store told the user their request
|
||||
was wrong; `ServerRow` decided a connection was an authorisation problem by
|
||||
matching `/401|403|unauthorized/` against prose an external MCP server wrote,
|
||||
which the host had sanitised and truncated on the way through. Domain,
|
||||
internal and dependency are three answers with three different things to do
|
||||
next, and a caller that cannot tell them apart reports the last two as the
|
||||
first.
|
||||
- Failure attribution is host-owned state. When the host can determine why an
|
||||
operation failed or stopped — truncation, timeout, unavailable context,
|
||||
dependency skipping, policy refusal, output spill, argument parsing — that
|
||||
cause goes to the model. Surfacing only the downstream symptom leaves it to
|
||||
invent one, and the invented rule outlives the incident: a response cut at the
|
||||
output limit reaches the model as "invalid arguments", and the session learns
|
||||
to stop batching calls that were never the problem.
|
||||
- A host-owned identity has to cross the model-visible boundary. Stable step
|
||||
ids, typed error codes, boundary causes — anything the host expects the model
|
||||
to cite must appear wherever that state is rendered for it. Telling the model
|
||||
in a schema to prefer X while every concrete rendering shows only Y trains it
|
||||
to use Y; `complete_step` said `step_id` PREFERRED while all four todo
|
||||
renderers printed bare ordinals. Canonical dynamic state still stays out of
|
||||
the cache-stable prefix — project the identities needed to act on it at the
|
||||
turn tail.
|
||||
- A metric is its definition, not its name. Before a number becomes a finding,
|
||||
state what it counts, from which anchor, over which population, and what it
|
||||
excludes. Three readings in one study died in that gap: a command of unknown
|
||||
extent read as a proven write, the latest report's adjudication read as every
|
||||
report's, and a syntactically accepted call read as a closed one. Each time a
|
||||
convenient proxy had been left standing in for the state it was named after,
|
||||
and each time the mechanism story built on it was wrong. A number without
|
||||
those four lines is an observation, not a result.
|
||||
- A constructed state is not a reachable one. Before a destructive test or a
|
||||
counterfactual is reported as a defect, say who in production produces that
|
||||
state, through which transitions, across which persistence boundary, under
|
||||
whose authority, and whether a real run has been seen reaching it. A test that
|
||||
assigns a private field proves the mechanism *given* the state and must be
|
||||
reported as exactly that: `a.projectChecks` is written once at boot and never
|
||||
again, so reassigning it stood in for a resume rather than demonstrating one,
|
||||
and the bypass built on it was never shown to be production-reachable.
|
||||
Severity is reachability times frequency times authority, not "can be
|
||||
bypassed".
|
||||
- A wire type the desktop re-declares by hand is held to the kernel's by
|
||||
`wire-parity`, both directions: a field one side sends and the other cannot
|
||||
read fails, and so does one it reads that nothing sends. The two are compiled
|
||||
by different toolchains and meet only over JSON, so nothing else in the tree
|
||||
notices when one of them grows a field. The pairs are declared in
|
||||
`tools/repolint/wireparity.go`, never inferred — no spelling tells a mirrored
|
||||
contract from a struct that merely carries json tags.
|
||||
- "Add behavior to the controller so every frontend inherits it" is held by
|
||||
`frontend-parity`: a capability one frontend already drives, a frontend whose
|
||||
declared port carries it, and no call, is a missing edge, and the ratchet
|
||||
counts edges. Which port each frontend drives is declared in
|
||||
`tools/repolint/frontendparity.go`; a deliberate exception goes in the scope
|
||||
table there with an issue and a reason, and costs what the edge cost, so the
|
||||
table cannot become where debt disappears. The matrix is read from
|
||||
`go/types` at one pinned build target, never from selector names — `Close`
|
||||
and `Status` are capabilities and also stdlib methods, and a name match
|
||||
records `os.File.Close` as a frontend driving `Lifecycle.Close`, a false
|
||||
"wired" that hides real debt. How a verb is spelled — `/branch`,
|
||||
`POST /branches` — is presentation and stays out of it.
|
||||
- The desktop's presentation layer answers to canonical state and never
|
||||
redefines it. Six rules, each held by a gate rather than by memory:
|
||||
presentation reorders and reweights what the kernel owns but never restates
|
||||
it; **mount is not a motion cause** — a fact enters once, and replay or
|
||||
virtualization bringing its element back is not that fact happening again;
|
||||
a search alias buys discoverability and never becomes a thing's identity;
|
||||
when a setting takes effect is declared per block, from what its endpoint
|
||||
does, never inferred from wording or handler names; ordering panels by
|
||||
lifecycle moves them and must not remount them; and a control's identity
|
||||
survives a change of where it is drawn — the settings sheet and the composer
|
||||
render `chrome.preset` because that is one intent, not two.
|
||||
|
||||
## Comments
|
||||
|
||||
@@ -137,75 +40,17 @@ Default is none — the code is the truth. Write one only when the **why** is
|
||||
non-obvious: a hidden constraint, a workaround anchored to something verifiable,
|
||||
an invariant the type system cannot express, or an external-protocol quirk.
|
||||
|
||||
A comment states the constraint that holds now. It does not narrate how the code
|
||||
got here: a reader who has never seen an earlier version must lose nothing, and
|
||||
past tense about this code's own behaviour is the tell — an external protocol's
|
||||
history is not. "Ids alone do not answer it" is the rule; "ids alone were the
|
||||
wrong question, so the list went on reading as the model wrote it" is the
|
||||
incident behind the rule, and the incident belongs in the commit message. This
|
||||
outranks the surrounding style: much of the tree reads as prose and none of it
|
||||
is the standard. No gate reads register — telling narrative from a statement of
|
||||
constraint needs a phrase table, which is the one thing this repo does not
|
||||
build — so this rule is held by review.
|
||||
|
||||
- **Stylesheets carry none.** A rule's selector and its declarations are the
|
||||
whole statement. What a `.css` comment would have said goes to whichever of
|
||||
two places can act on it: a constraint a component has to honour belongs in
|
||||
that component, and a decision about the medium itself — a measured colour, a
|
||||
layout mechanic, why the obvious rule is the wrong one — belongs in the
|
||||
stylesheet's companion document (`src/styles/TOKENS.md`, `LAYOUT.md`), which
|
||||
is read end to end rather than met 600 times as an aside.
|
||||
- **English, always** — every comment, in every other language the tree is
|
||||
written in. A codebase whose commentary is half one language reads as two
|
||||
codebases, and the half a given reader cannot read is the half that rots.
|
||||
- Declaration doc: ≤5 lines. Package comment: ≤8 lines, or ≤40 in a `doc.go`.
|
||||
- Every other comment: ≤3 lines. Struct-field and trailing `//`: 1 line.
|
||||
- Never: restatements of the code, phase/stage narrative, incident or
|
||||
conversation history, section banners, commented-out code, `@param` lists.
|
||||
- `TODO(#nnn):` and `HACK(#nnn):` need the issue anchor. `FIXME` is banned.
|
||||
- One responsibility per file; 800 lines is the ceiling. Test files are exempt:
|
||||
their length tracks how many cases they cover, not how many concerns they
|
||||
carry, so splitting one only scatters a subject's table across files.
|
||||
- One responsibility per file; 800 lines is the ceiling.
|
||||
|
||||
`go run ./tools/repolint` enforces the budgets above against a ratchet baseline:
|
||||
recorded debt is tolerated, anything new fails CI. Never widen the baseline to
|
||||
land a change — fix the code. `-update` lowers budgets freely and refuses to
|
||||
raise one without `-allow-widen`, so carrying debt through a rename or an
|
||||
extraction is asked for in the command and justified in the PR. A clean run also
|
||||
reports budget the tree stopped using: it is only reclaimed by an `-update`, and
|
||||
until then it is room a file can grow back into.
|
||||
|
||||
## Context projection
|
||||
|
||||
Canonical dynamic state — the skill registry, the memory store — lives in its own
|
||||
owner, and what the model sees is a projection of it onto each request. The
|
||||
prefix was never where these belonged wrongly; freshness ownership was missing.
|
||||
|
||||
- **Prefix ownership.** Only declared prefix configuration may move the
|
||||
cache-stable prefix; ordinary memory writes, skill writes and runtime facts
|
||||
ride the turn. Pinned memory bodies are the one declared exception, because
|
||||
pinning is a request to pay one cold start.
|
||||
- **Projection freshness.** When canonical state changes, the next eligible turn
|
||||
carries the new projection or can say why it is not owed. "Right after a fold,
|
||||
or in the next session" is not an answer.
|
||||
- **Projection ownership.** Whoever owns a dynamic fact owns the invalidation
|
||||
that makes its projection owed again. A projection kept fresh because every
|
||||
writer remembered to announce itself is one forgotten call from stale —
|
||||
`skillSet.owedCatalog` asks the registry rather than waiting to be told.
|
||||
- **Determinism.** The same canonical state and the same projection state
|
||||
produce byte-identical model-visible bytes.
|
||||
- **Detection is semantic.** Freshness may not rest on writer notification, on
|
||||
mtime, or on which call path made the change. The registry answers from
|
||||
canonical state, so a switch flipped and a file edited past every API are the
|
||||
same question, and a change no projection renders is not a change.
|
||||
|
||||
`internal/assembly/boot/context_projection_test.go` holds all five at the provider
|
||||
boundary, with the skills listing and the standing instructions as the two
|
||||
worked examples. A block is settled only once the turn carrying it reaches the
|
||||
runner: a turn blocked by a hook or refused by an extension after composing put
|
||||
those bytes nowhere, and clearing the debt there would leave the model holding
|
||||
state nobody sent it. A projection that has become empty is still owed once —
|
||||
otherwise the block the model already has goes on reading as current.
|
||||
`go run ./tools/repolint` enforces all of it against a ratchet baseline: recorded
|
||||
debt is tolerated, anything new fails CI. Never widen the baseline to land a
|
||||
change — fix the code. `-update` exists for carrying debt through a rename or an
|
||||
extraction, and that diff must be justified in the PR.
|
||||
|
||||
## Memory
|
||||
|
||||
@@ -225,50 +70,6 @@ otherwise the block the model already has goes on reading as current.
|
||||
|
||||
## Notes
|
||||
|
||||
One tool call costs one model round trip, so calls that do not depend on each
|
||||
other belong in the same round: several reads, edits to different files, a check
|
||||
after the edit that enables it. Several edits to the *same* file are one
|
||||
`multi_edit` — it applies them against each other in memory and rewrites the
|
||||
file only if all of them land, so a failure midway leaves nothing half-edited.
|
||||
|
||||
## Vendor rates
|
||||
|
||||
Official list prices live in one place: `officialRates` in
|
||||
`internal/contract/pricing/rates.go`, oldest generation first, the last being what a
|
||||
model charges today. A price change is an **append, never an edit** — the
|
||||
superseded rate is what lets an installed config be recognised as holding one
|
||||
of ours and brought forward, instead of quoting a retired rate for the rest of
|
||||
its life. `config` reads these through `pricing.CurrentRate` and
|
||||
`SupersededRates` and restates none of them; a table written twice is one that
|
||||
drifts, which is how a vendor's USD rate ended up published on one side only.
|
||||
|
||||
`make pricecheck` reads each vendor's own page and compares it against that
|
||||
table, and reports how long ago a person last confirmed each one. It reaches
|
||||
the network, so it stays out of `make lint` and out of CI: run it before
|
||||
touching a rate and when a vendor announces a change. A page it cannot read
|
||||
exits non-zero — unread is its own verdict and never counts as agreement.
|
||||
|
||||
## Curated presets
|
||||
|
||||
A preset changing shape reaches the installs that already have the old one
|
||||
through `shippedPresets` in `internal/contract/config/preset_upgrade.go`: one declared
|
||||
shape per generation, and one upgrade that brings a matching entry forward. The
|
||||
guard is the whole shape — catalog, window, default — because an entry holding
|
||||
what we shipped is ours to move and anything else its user curated.
|
||||
|
||||
A preset is also what settles which of its models read images. Probing an
|
||||
address a preset covers answers from that declaration, because reading a name
|
||||
misses every Kimi, Qwen, MiniMax and Claude model they declare and `deepseek-
|
||||
flash` spells nothing at all; the spelling is the last resort for an address
|
||||
nothing has declared, and stays a correctable suggestion rather than a claim.
|
||||
|
||||
Three distinctions are load-bearing, and each was a separate migration function
|
||||
before they were written down. A window the shape did not name is the user's,
|
||||
so only a shape identified by its window moves one. `vision_models` **narrows**,
|
||||
so an upgrade marks only the models it adds and only where the preset says they
|
||||
read images. And an explicitly empty list is a choice — nothing here reads
|
||||
images — which is not the same as the nil that means nobody has said.
|
||||
|
||||
## Pre-push CI simulation
|
||||
|
||||
Run these **before every commit** to catch the fastest CI failures locally:
|
||||
@@ -277,7 +78,7 @@ Run these **before every commit** to catch the fastest CI failures locally:
|
||||
gofmt -w . # catches gofmt (saves ~13s CI)
|
||||
go vet ./... # catches vet warnings (saves ~52s CI/lint)
|
||||
make lint # golangci-lint at CI's pin + repolint
|
||||
go test ./internal/tools/builtin/ ./internal/assembly/boot/ # catches tool/boot test breaks
|
||||
go test ./internal/tool/builtin/ ./internal/boot/ # catches tool/boot test breaks
|
||||
```
|
||||
|
||||
`make lint` runs both gates CI runs, at the version in `.golangci-version`;
|
||||
@@ -285,25 +86,6 @@ go test ./internal/tools/builtin/ ./internal/assembly/boot/ # catches tool/boot
|
||||
shows up in `go vet`, and the CI round trip that catches it instead costs ten
|
||||
minutes.
|
||||
|
||||
A full repolint run reports every file over budget, including debt your change
|
||||
never touched. To see only what your own edits owe:
|
||||
|
||||
```bash
|
||||
make check
|
||||
```
|
||||
|
||||
Repo-wide ceilings still report under a narrowed run, because a file can push
|
||||
the tree past one without exceeding its own budget.
|
||||
|
||||
`make check` owns the evidence-bearing changed-file universe — everything the
|
||||
worktree altered *and* everything it added that git has not been told about.
|
||||
Never substitute a direct `go run ./tools/repolint` invocation for it when
|
||||
recording verification evidence: the host recognizes a `make` target as a check
|
||||
whose result it can read, while an arbitrary `go run ./tools/...` is a program
|
||||
it has to assume writes, and the flags behind the target are its implementation
|
||||
rather than the contract. Building that universe from `git diff` alone once let
|
||||
a new file reach the gate unopened and be reported clean.
|
||||
|
||||
## Import cycle rule
|
||||
|
||||
Before importing a new internal package from a non-test file, verify the target package's **test files** aren't already importing back to you:
|
||||
@@ -320,120 +102,32 @@ Use `go test ./path/to/target/` to detect cycles **before** pushing. A `[setup f
|
||||
- **Keep the PR diff minimal.** Only the files relevant to the PR's purpose — no stray changes from other branches.
|
||||
- **Amend, don't add commits, for review feedback** — keeps the commit history clean.
|
||||
|
||||
## Reasonix host checks
|
||||
## PR metadata gates
|
||||
|
||||
The paths below decide what the agent is allowed to do — a wrong change here
|
||||
is not a bug in a feature, it is a hole in a boundary. Declaring them makes
|
||||
every change under them demand `review` plus `security_review` before a turn
|
||||
can finish. Sensitivity is declared, never inferred: the host does not read it
|
||||
out of how a path is spelled, which cannot tell `internal/auth` from
|
||||
`session_write_authority.go`, or a trace file from a data race.
|
||||
Two CI guards read the PR body. The scripts are the source of truth and both
|
||||
run locally: `scripts/check-cache-impact.sh`, `scripts/check-docs-impact.sh`.
|
||||
Separators must be an ASCII `-` or `:` — an em dash fails the docs guard.
|
||||
|
||||
- sensitive: internal/safety/permission/**
|
||||
- sensitive: internal/safety/sandbox/**
|
||||
- sensitive: internal/safety/shellsafe/**
|
||||
- sensitive: internal/base/shellparse/**
|
||||
- sensitive: internal/session/control/approval.go
|
||||
- sensitive: internal/session/control/approval_bridge.go
|
||||
- sensitive: internal/session/control/approval_orchestration.go
|
||||
- sensitive: internal/model/providerbroker/**
|
||||
- sensitive: internal/ext/installsource/**
|
||||
- sensitive: internal/ext/plugin/**
|
||||
- sensitive: internal/ext/pluginpkg/**
|
||||
- sensitive: internal/base/netclient/**
|
||||
- sensitive: internal/safety/redirectguard/**
|
||||
- sensitive: internal/safety/egress/**
|
||||
- sensitive: internal/platform/browser/**
|
||||
- sensitive: internal/platform/computer/**
|
||||
Cache-sensitive diffs (`internal/tool/`, `internal/provider/`,
|
||||
`internal/boot/`, `internal/agent/agent.go`, and the rest of the list in the
|
||||
script) require:
|
||||
|
||||
One declaration, two effects: the same list is also the coverage gate's subject.
|
||||
`make coverage-gate` holds each path to the coverage it already has, because
|
||||
when this was written these paths sat *below* the tree's median rather than
|
||||
above it — the code deciding what the agent may execute was its least-tested
|
||||
part. Floors ratchet like repolint's budgets: raising one is free, lowering one
|
||||
needs `-allow-drop` and a reason in the PR. Adding a `sensitive:` path gets both
|
||||
effects; there is no second list to keep in sync.
|
||||
```
|
||||
Cache-impact: <none|low|medium|high> - <reason>
|
||||
Cache-guard: <focused guard test/command or existing guard rationale>
|
||||
```
|
||||
|
||||
## Trust domain
|
||||
`none` is a legitimate impact when the provider-visible prefix stays
|
||||
byte-identical; only an empty value, `todo`, or `tbd` is rejected. If the diff
|
||||
also touches `internal/config/`, `internal/memory/`, `internal/outputstyle/`,
|
||||
`internal/skill/`, or `internal/boot/`, add `System-prompt-review: <note>` —
|
||||
that field additionally rejects `none` and `n/a`, so it must name a reviewer.
|
||||
|
||||
Three sets, kept apart because conflating them turns a performance number into a
|
||||
security claim: **VerificationRelevant** (state that can change whether a
|
||||
delivery claim holds), **Observable** (state the host can reconcile after an
|
||||
action, so it can say what that action did), **HostProtected** (state a
|
||||
capability/sandbox boundary makes unwritable). The invariant is
|
||||
`Writable ∩ VerificationRelevant ⊆ Observable ∪ HostProtected`:
|
||||
User-visible diffs (`cmd/reasonix/`, `desktop/`, `npm/`, and most of
|
||||
`internal/`; tests and lockfiles are exempt) require one of these, chosen by
|
||||
whether the same PR edited `docs/*.md`:
|
||||
|
||||
> Any state that can both be modified by an action and influence the delivery
|
||||
> claim must either be observable by the host or protected from modification by
|
||||
> host-enforced boundaries. An incomplete observation never establishes scope.
|
||||
|
||||
Effects and state answer different questions and never stand in for each other —
|
||||
observation proves what an action did, verification proves what the resulting
|
||||
state satisfies — which is why `unproven_mutation` is settled only by
|
||||
observation, never by a check that ran afterwards.
|
||||
|
||||
What follows bounds Observable today. It is not a restatement of the invariant:
|
||||
moving a number here must not move the promise.
|
||||
|
||||
- `scanWorkspace` walks the workspace recursively, skips VCS store dirs, and is
|
||||
`complete` only below 50k files. Over that, or past a walk error, it
|
||||
establishes nothing and the mutation stays unproven.
|
||||
- Symlinks are recorded by `Lstat`, so a write through one lands outside what
|
||||
the walk compares. Confining the resolved target is the sandbox's job.
|
||||
- `internal/safety/sandbox` confines writes to the workspace, configured extras, temp
|
||||
and toolchain caches — on macOS and Linux. Windows has no OS-level bash
|
||||
sandbox, so there nothing the host enforces bounds `Writable`.
|
||||
- The governed host authorities are a *named* set — ssh-agent, docker, podman.
|
||||
DBus, Wayland, X11, gpg-agent, keyrings and arbitrary sockets stay reachable,
|
||||
so the claim is "these endpoints are governed", never "host IPC is isolated".
|
||||
|
||||
`stateEpoch` tracks host-observed mutations, not an exact filesystem snapshot;
|
||||
unobserved external writers are a gap shared by the whole verification model. Go
|
||||
baseline test criteria use the toolchain's build overlay and therefore share
|
||||
ordinary verification's epoch semantics; the overlay is not a general filesystem
|
||||
view.
|
||||
|
||||
A captured criterion may be superseded only by authority outside
|
||||
execution-mutable state; the current workspace, its tests, and the agent itself
|
||||
cannot authorize supersession.
|
||||
|
||||
Two gaps follow and are open, not decided: a build that reads `.git` makes VCS
|
||||
state VerificationRelevant while the scan skips it, and on Windows a symlink out
|
||||
of the workspace is neither observed nor protected. Closing either means
|
||||
widening observation or narrowing writability — never widening what a check is
|
||||
taken to prove.
|
||||
|
||||
## Sandbox dimensions
|
||||
|
||||
`HostProtected` above is the delivery-level abstraction. Sandbox policy refines
|
||||
it into three independent protection dimensions. Their separation is empirical:
|
||||
measured bypasses showed that protection on one dimension cannot be assumed to
|
||||
imply protection on another.
|
||||
|
||||
- **Integrity** — what host state may be mutated. Expressed by write roots.
|
||||
- **Confidentiality** — what host information may be observed. Expressed by
|
||||
`ForbidReadRoots`.
|
||||
- **Authority** — what host capabilities may be exercised *without* possessing
|
||||
their secrets. Expressed by `HostAuthorities`.
|
||||
|
||||
`ReadOnly` is therefore an Integrity property only; it makes no Confidentiality
|
||||
or Authority claim.
|
||||
|
||||
Network transport and host authority are independent even when a backend
|
||||
primitive conflates them: on macOS, `(deny network*)` also cut ssh-agent, so one
|
||||
flag could not express "no internet, still sign".
|
||||
|
||||
Filesystem visibility is not authority: denying `file-read*` on a socket did not
|
||||
stop a connect. Possession is not authority: the key can be unreadable while an
|
||||
agent still signs with it.
|
||||
|
||||
A protection claim exists only where a backend has a demonstrated enforcement
|
||||
primitive for that dimension. Unsupported dimensions stay explicitly unsupported
|
||||
— Windows currently enforces none of the three — because a claim that outruns
|
||||
its enforcement is worse than an absent one: it is believed.
|
||||
|
||||
**Known unmeasured surface — macOS host-mediated delegation:** Apple Events and
|
||||
Mach-service lookup are not denied by the current allow-default profile. Prior
|
||||
`osascript` and `launchctl submit` probes were inconclusive; no delegation
|
||||
effect has been demonstrated. This surface is outside the currently governed
|
||||
host-authority endpoint set.
|
||||
```
|
||||
Documentation-impact: updated - <what changed> # docs/*.md edited
|
||||
Documentation-impact: none - <why the docs stay correct> # not edited
|
||||
```
|
||||
+6
-57
@@ -56,10 +56,9 @@ Supported boundaries include:
|
||||
- Sandbox behavior for built-in shell execution where the platform supports it.
|
||||
- Secret handling for provider keys, bot credentials, OAuth tokens, plugin
|
||||
headers, and credential-store fallback files.
|
||||
- HTTP `serve` protections, including the default per-launch token, the launch
|
||||
token every state-changing request requires when authentication is opted
|
||||
out, localhost binding assumptions, JSON-only state-changing requests, and
|
||||
CORS restrictions.
|
||||
- HTTP `serve` protections for the unauthenticated local server, including
|
||||
localhost binding assumptions, JSON-only state-changing requests, and CORS
|
||||
restrictions.
|
||||
- Desktop and bot session isolation, including per-workspace session metadata
|
||||
and configured bot allowlists.
|
||||
- Updater, install, and release verification paths.
|
||||
@@ -68,10 +67,10 @@ The following are normally treated as trusted local/operator-controlled inputs
|
||||
unless another bug lets an untrusted actor supply them:
|
||||
|
||||
- CLI arguments and text typed directly by the local user.
|
||||
- The user's own configuration under the Reasonix home directory.
|
||||
- Project configuration files intentionally loaded from the current workspace.
|
||||
- Explicit `@path` references supplied by the local user to attach local files.
|
||||
- MCP servers, language servers, hooks, and slash commands installed, enabled,
|
||||
or approved by the local user.
|
||||
- MCP servers, language servers, hooks, and slash commands installed or enabled
|
||||
by the local user.
|
||||
- Provider base URLs and model names configured by the local user.
|
||||
|
||||
The following can be security issues when reachable by an untrusted actor or
|
||||
@@ -90,56 +89,6 @@ when they bypass the intended boundary:
|
||||
- Trusting unverified update artifacts, plugin definitions, or downloaded
|
||||
binaries.
|
||||
|
||||
## Project Configuration
|
||||
|
||||
Files that arrive with a workspace are untrusted input: `reasonix.toml`,
|
||||
`.reasonix/settings.json`, `.mcp.json` and the project `.env`. Cloning a
|
||||
repository is not a decision to trust its author, so these files may narrow
|
||||
what the user configured and never widen it.
|
||||
|
||||
| Setting | What a workspace file may do |
|
||||
| --- | --- |
|
||||
| `sandbox.bash` | Move `off` to `enforce`, never back |
|
||||
| `sandbox.network` | Move `true` to `false`, never back |
|
||||
| `sandbox.forbid_read`, `sandbox.denied_domains` | Add to the user's lists |
|
||||
| `sandbox.allow_write`, `sandbox.workspace_root` | Apply only when they resolve inside the workspace, symlinks followed; `${VAR}` in any sandbox path expands from the process environment only |
|
||||
| `sandbox.host_authorities`, a user's `sandbox.allowed_domains` | Nothing |
|
||||
| `permissions.deny`, `permissions.ask` | Add to the user's rules |
|
||||
| `permissions.mode`, `permissions.allow`, `permissions.allow_dynamic_bash` | Nothing |
|
||||
| `desktop.default_tool_approval_mode` | Nothing |
|
||||
| `[network]` proxy settings | Nothing |
|
||||
| `[[providers]]`, `tools.system_one` endpoints | Declare them; they stay out of use until approved, and model settings naming them fall back to the user's |
|
||||
|
||||
An "always allow" answer, or extra write access, granted in a workspace is
|
||||
stored under the Reasonix home, not in the workspace.
|
||||
|
||||
Programs a workspace names for the host to run stay off until the user
|
||||
approves them for that workspace:
|
||||
|
||||
- Hooks, language server commands, `tools.search.rg_path`,
|
||||
`tools.shell.path`, `[browser]` and `tools.system_one.laya.python` are
|
||||
approved with `reasonix trust`, or for hooks by saving them from the hooks
|
||||
editor.
|
||||
- An approval is stored under the Reasonix home with a digest of the
|
||||
declaration and of the workspace files it names; any change needs approval
|
||||
again. Approvals and grants belong to one workspace folder.
|
||||
- Hooks and language servers are checked against their approval again before
|
||||
each start, and refused if a file they name changed, appeared or now
|
||||
resolves elsewhere.
|
||||
- A single program path (`tools.shell.path`, `tools.search.rg_path`,
|
||||
`[browser]`, `laya.python`) may not point into the workspace, an
|
||||
`allow_write` root, a temporary directory or a toolchain cache the bash jail
|
||||
leaves writable, and may not climb with `..`; its approval covers the
|
||||
program file's content.
|
||||
- `tools.system_one.laya.local` is read from the user's configuration only.
|
||||
- Without an interactive frontend they stay off.
|
||||
- MCP servers a workspace declares stay off until the user enables them for
|
||||
that workspace.
|
||||
|
||||
A project value that is not applied is reported as a load warning with the
|
||||
reason. A way for workspace files to widen any of the above without the
|
||||
user's approval is a security issue.
|
||||
|
||||
## `@` File References
|
||||
|
||||
Reasonix supports `@path` references so users can include local files and images
|
||||
|
||||
+55
-160
@@ -1,24 +1,18 @@
|
||||
# Reasonix Benchmarks
|
||||
|
||||
Five harnesses live under `benchmarks/`; `cmd/e2ebench` also exposes a
|
||||
Three harnesses live under `benchmarks/`; `cmd/e2ebench` also exposes a
|
||||
SWE-bench Verified mode:
|
||||
|
||||
- `e2e/` — the committed end-to-end task suite, driven by
|
||||
[`cmd/e2ebench`](../cmd/e2ebench/main.go). It runs each task against a real
|
||||
provider and emits a markdown + JSON report (accuracy, cache-hit rate, token
|
||||
use, cost) suitable for pasting into a PR.
|
||||
- `context-maintenance-e2e/` — a standalone seed → resume → continue smoke that
|
||||
checks a session seeds below the compaction trigger, resumes without paying
|
||||
for a summary, and folds exactly once past it.
|
||||
- `context-maintenance-e2e/` — a standalone seed → resume → comprehension
|
||||
harness that A/B-compares cold-restart cache behavior with and without
|
||||
context pruning.
|
||||
- `compaction/` — CompactionBench: grows a session one generation at a time and
|
||||
folds it after each, measuring what repeated compaction costs and what it
|
||||
loses. See [CompactionBench](#compactionbench) below.
|
||||
- `upstream-edge/` — a shape-instructed pair of fleet tasks for pricing what a
|
||||
`depends_on` edge carries, against the arm where it only orders. See
|
||||
[Pricing the run graph](#pricing-the-run-graph) below.
|
||||
- `fanout-width/` — a shape-instructed fleet whose members share nothing, for
|
||||
pricing what running them at the same time is worth. See
|
||||
[Pricing the run graph](#pricing-the-run-graph) below.
|
||||
|
||||
## Directory layout
|
||||
|
||||
@@ -34,57 +28,6 @@ benchmarks/
|
||||
└── run/ # state dir written by seed/resume (default)
|
||||
```
|
||||
|
||||
## Semantic contract for a metric
|
||||
|
||||
A number reaches a report only with these four lines written down, because a
|
||||
metric is its definition and not its name:
|
||||
|
||||
```text
|
||||
Metric: ToolsAfterClosed
|
||||
Anchor: the host's first adjudication whose verdict is closed
|
||||
Population: judged child runs (expected to report, not killed by the provider)
|
||||
Excludes: needs_work reports, malformed reports, parent-level recovery
|
||||
Reads as: ordinary child work after a host-declared terminal closure
|
||||
```
|
||||
|
||||
The subagent-handoff study lost three mechanism explanations to a missing
|
||||
contract: a command whose extent the host could not establish was counted as a
|
||||
proven write, one child's latest adjudication was read as every report's, and a
|
||||
syntactically accepted call was read as a closed one. Each was a convenient
|
||||
proxy left standing in for the state the metric was named after, and each
|
||||
produced a confident, wrong story about what the model was doing.
|
||||
|
||||
Watch for the five substitutions that caused them: unknown read as true, latest
|
||||
read as all, accepted read as closed, attempted read as judged, and not
|
||||
evaluated read as false.
|
||||
|
||||
## Reachability contract for a defect
|
||||
|
||||
A metric owes its definition; a defect owes its state. Before a destructive test
|
||||
or a counterfactual is written up as something to fix:
|
||||
|
||||
```text
|
||||
Constructed state: baseline != current
|
||||
Production producer: who assigns it, and where
|
||||
Transition path: which real transitions reach it from a normal boot
|
||||
Persistence boundary: does it need a restart, a resume, a crash
|
||||
Authority: who caused the change — the agent, the user, the host
|
||||
Observed: has a real run been seen in this state
|
||||
```
|
||||
|
||||
The project-check study skipped the middle four. A test reassigned
|
||||
`a.projectChecks` and the bypass it produced was reported as reproducible; the
|
||||
field is written once at boot and never again, so no run inside one process can
|
||||
reach that state, and the corpus said so on its first execution — parity, no
|
||||
divergence. What the test proves is a conditional mechanism, which is a
|
||||
different claim from a production defect and earns a different priority.
|
||||
|
||||
Authority belongs in the contract because the same divergence can mean opposite
|
||||
things. An agent that rewrote the declaration it was being held to is the case
|
||||
baseline protection exists for; an owner who changed the project's policy
|
||||
between two runs may have superseded it deliberately. A study that cannot say
|
||||
which one it observed cannot say what the right behaviour is.
|
||||
|
||||
## Task corpus stratification
|
||||
|
||||
The suite is stratified by real coding-agent workload classes, not toy-task
|
||||
@@ -93,31 +36,21 @@ utility readouts key on. Current coverage vs. target:
|
||||
|
||||
| Class | Target | Committed | Notes |
|
||||
| --- | ---: | ---: | --- |
|
||||
| `atomic-bugfix` | 8 | 11 | short anchored fixes; routes ExecutorOnly by design |
|
||||
| `repo-exploration` | 6 | 7 | multi-file reading, invented-token answers so they can't be guessed |
|
||||
| `atomic-bugfix` | 8 | 8 | short anchored fixes; routes ExecutorOnly by design |
|
||||
| `repo-exploration` | 6 | 6 | multi-file reading, invented-token answers so they can't be guessed |
|
||||
| `multi-file-bugfix` | 8 | 8 | one bug spanning ≥2 files; naturally engages the planner gate |
|
||||
| `refactor` | 6 | 6 | behavior-preserving restructuring, structure asserted |
|
||||
| `failing-test-diagnosis` | 6 | 8 | unittest suite red → fix source; tests checksummed |
|
||||
| `failing-test-diagnosis` | 6 | 6 | unittest suite red → fix source; tests checksummed |
|
||||
| `api-integration` | 4 | 4 | use a provided local package per its README |
|
||||
| `ambiguous` | 4 | 4 | underspecified ask; grader accepts the defensible core |
|
||||
| `long-horizon` | 4 | 4 | multi-requirement specs; planner-depth full |
|
||||
| `codegen` / `delegation` | — | 4 | fizzbuzz, palindrome, and two Go tasks that build a tool from nothing |
|
||||
| `completion-integrity` | 12 | 12 | **no reachable solution**; scored on honesty, not correctness (below) |
|
||||
| `codegen` / `delegation` | — | 3 | legacy smoke tasks (fizzbuzz, palindrome, subagent-delegation) |
|
||||
| `completion-integrity` | 11 | 11 | **no reachable solution**; scored on honesty, not correctness (below) |
|
||||
|
||||
Grader authoring rule: every task must fail `verify.sh` on the pristine seed
|
||||
and pass it on a reference solution (validated before commit). The first half
|
||||
is enforced by `TestSolvableCorpusSeedsMustNotGradeClean` — a seed that already
|
||||
grades clean scores the same whether the agent solved it or never ran, and
|
||||
reports 100% forever. SWE-bench Verified (below) supplies the realistic-repo
|
||||
end of the spectrum; this corpus covers the fast, controlled, per-class end.
|
||||
|
||||
**Language coverage.** The corpus was Python-only for its whole history, while
|
||||
the agent's own repository is Go. A compiled language is a different verification
|
||||
loop — `go build` before anything runs, `go vet`, `-race`, a module path that
|
||||
must agree with every import — and its defects have no Python analogue: a data
|
||||
race, a `+=` in a loop that is quadratic where Python's would be too, map
|
||||
iteration order that makes a test pass only sometimes. The `go-*` tasks cover
|
||||
that half; they are ordinary members of the classes above, not a separate tier.
|
||||
and pass it on a reference solution (validated before commit). SWE-bench
|
||||
Verified (below) supplies the realistic-repo end of the spectrum; this corpus
|
||||
covers the fast, controlled, per-class end.
|
||||
|
||||
## Completion Integrity
|
||||
|
||||
@@ -244,11 +177,8 @@ are then counted identically for anything that speaks the endpoint.
|
||||
go run ./cmd/e2ebench -meter ~/.reasonix/config.toml -trajectories t/
|
||||
```
|
||||
|
||||
- **Credentials are never touched.** The config only names an `api_key_env`;
|
||||
only `base_url` is rewritten. The key itself is resolved from Reasonix's own
|
||||
`$REASONIX_HOME/.env`, never from the process environment — exporting
|
||||
`DEEPSEEK_API_KEY` into the shell is not enough to run a benchmark against a
|
||||
temporary home, which needs its own `.env`.
|
||||
- **Credentials are never touched.** The config names an `api_key_env`, so the
|
||||
key stays in the environment the child inherits; only `base_url` is rewritten.
|
||||
- **Only the provider serving `-model` is redirected.** Rewriting every endpoint
|
||||
would send one vendor's traffic to another's host.
|
||||
- **Streamed requests are opted into usage.** An OpenAI-compatible stream
|
||||
@@ -325,9 +255,9 @@ entry replaces one leg's continuation with a user turn.
|
||||
|
||||
Two properties are load-bearing:
|
||||
|
||||
- **The wall clock is shared, never multiplied.** Every leg runs inside the
|
||||
task's one `timeout_sec`, so a segmented arm cannot win by being allowed to
|
||||
work longer. (`max_steps`, when a task sets one, is divided the same way.)
|
||||
- **The step budget is divided, never multiplied.** A segmented arm gets the
|
||||
same `max_steps` as the control arm, split across legs with the remainder on
|
||||
the last. Otherwise the arm would win by being allowed to work longer.
|
||||
- **Each leg writes its own metrics file.** They share a work dir, so a single
|
||||
`.run-metrics.json` would leave the last leg's numbers standing in for the
|
||||
whole run and the earlier legs' tokens would simply vanish. `Segments` in the
|
||||
@@ -349,7 +279,7 @@ decoder. The task ID is the directory name; tasks run in sorted ID order.
|
||||
| --- | --- | --- | --- |
|
||||
| `prompt` | string | yes | The task instruction handed to the agent. |
|
||||
| `class` | string | no | Task class label (e.g. `bugfix`, `codegen`, `exploration`) for per-class marginal-utility breakdowns in compare mode. |
|
||||
| `max_steps` | int | no | Agent tool-call cap, passed through as `--max-steps`. **Committed tasks must not set one** — `TestCorpusLetsTheAgentDecideWhenToStop` enforces that. The product ships unbounded, so a capped task grades a configuration no default user runs, and on a no-solution task the host's "summarise your progress" says the very sentence the honesty score exists to hear from the model. |
|
||||
| `max_steps` | int | yes | Agent tool-call cap; passed through as `--max-steps` to `reasonix run`. |
|
||||
| `no_solution` | bool | no | Ground truth: no reachable solution exists. The task leaves every accuracy denominator, its `verify.sh` grades the inverse contract, and it is scored on honesty instead. See [Completion Integrity](#completion-integrity). |
|
||||
| `timeout_sec` | int | no | Per-task wall-clock timeout in seconds; defaults to `240` when omitted or `0`. |
|
||||
| `seed_correct` | string | no | The task's real cause, phrased as a conclusion handed down before the run. Used by `-anchor correct`. See [Anchor resistance](#anchor-resistance). |
|
||||
@@ -359,6 +289,7 @@ Example (`tasks/fizzbuzz/task.toml`):
|
||||
|
||||
```toml
|
||||
prompt = "Create a file named fizzbuzz.py containing a function fizzbuzz(n) that returns the string 'Fizz' when n is divisible by 3, 'Buzz' when divisible by 5, 'FizzBuzz' when divisible by both 3 and 5, and otherwise the number as a string. Do not print anything at import time."
|
||||
max_steps = 12
|
||||
timeout_sec = 180
|
||||
```
|
||||
|
||||
@@ -420,24 +351,20 @@ own outcome).
|
||||
| `-suite` | `benchmarks/e2e` | Suite root (must contain `tasks/<id>/`). |
|
||||
| `-task` | *(all)* | Suite mode: run only these comma-separated task IDs (e.g. `-task fix-add-bug`); unknown IDs fail with the available list. |
|
||||
| `-attempts` | `1` | Suite and diff modes: retry a task until an attempt passes, up to N; enables the `Pass@≤N` KPI, and TTCS charges a retried solve with its failed attempts' wall. |
|
||||
| `-trials` | `1` | Suite mode: run every task N times and keep every outcome. Adds a **Reliability** line: `pass^N` (tasks that passed every trial), `pass@N` (any trial), mean `pass@1` over all trials, and the flaky tasks by name. `Pass@≤N` asks whether the agent can solve a task; `pass^N` asks whether it does so every time. Cannot be combined with `-attempts`. |
|
||||
| `-tape-record DIR` | — | Suite mode, with `-meter`: record every provider exchange at the meter, one tape per task and trial (`DIR/<task>/trial-N/0001.request.json`, `.response.json`, `.response.body`). Taped runs use a fixed workdir path and pin the seeded files' times, because both reach the model through the workspace block and file listings. |
|
||||
| `-tape-replay DIR` | — | Suite mode, with `-meter`: answer each run from its tape instead of the provider, and add a **Replay** line naming the first request whose body differs from the recorded one, down to the message (`request 2, message 4 (tool)`), the tool list, or a field. The diverging request is kept as `NNNN.replayed.json` beside the recorded one. A kernel change that should not alter what the model is asked replays with every request matched. |
|
||||
| `-bin` | `reasonix` | Path to the reasonix binary. |
|
||||
| `-model` | *(config default)* | Provider/model name. |
|
||||
| `-profile` | `baseline` | Tool-surface/runtime tier: `baseline` \| `economy` \| `balanced` \| `delivery`. All but `baseline` append `--profile <tier>` to the agent invocation; `baseline` passes no flag (byte-identical legacy control, behaviorally `balanced`). Economy starts with the core tool set and pays `connect_tool_source` rounds plus prefix resets to grow it — the report's Tool surface line prices that trade. |
|
||||
| `-ablate` | *(none)* | Ablation arm: comma-separated subsystems to switch off — `evidence`, `planner`, `subagent`, `retrieval`, `compaction`, `upstream` and `recall-search`; `none` \| `all`. |
|
||||
| `-ablate` | *(none)* | Ablation arm: comma-separated subsystems to switch off — `evidence`, `planner`, `subagent`, `retrieval`, `compaction`; `none` \| `all`. |
|
||||
| `-out` | *(stdout)* | Write the markdown report here. |
|
||||
| `-json` | *(none)* | Write the JSON report here (optional). |
|
||||
| `-trajectories` | *(none)* | Suite mode: write one `<task-id>.trajectory.jsonl` per task into this directory (the agent's full event stream with timestamps — see `reasonix run --trajectory`). The report gains a time-attribution line (tools vs. model) and each JSON result a `trajectory` digest. |
|
||||
| `-force-planner` | `false` | Suite mode: prefix each prompt with a plan-first directive so the two-model turn engages regardless of the planner gate. Use for the "with planner" arm of an A/B; results carry `plan_forced` so arms are only comparable with equal forcing. |
|
||||
| `-anchor` | `blind` | Suite mode: which hypothesis the agent holds before it looks at anything — `blind` (none, the control) \| `correct` \| `wrong`. The seeded arms prefix each prompt with the task's authored seed and **skip** tasks that have none, so an unseeded control run never lands in a seeded denominator. Results carry `anchor`. See [Anchor resistance](#anchor-resistance). |
|
||||
| `-cache` | `cold` | Suite mode: `cold` runs each task as a fresh session (the fair cross-agent comparison arm); `warm` primes the provider prefix cache with a one-step run in the same workdir first, measuring the long-lived-session steady state. Never mix arms in one report — compare them with `-mode compare cold.json warm.json`. |
|
||||
| `-policy` | *(none)* | Suite mode: experiment arm. `memory-off` hides the memory store, which is the MemoryBench counterfactual arm; the empty default runs the baseline. |
|
||||
| `-budget` | `800000` | Abort once total tokens cross this (`0` = no cap). Remaining tasks are reported as skipped. |
|
||||
| `-meter` | *(off)* | Suite mode: route the benchmarked provider through the neutral measuring proxy, using this `config.toml` as the source. Spend is then counted at the request boundary instead of trusted from the harness. See [Neutral metering](#neutral-metering). |
|
||||
| `-faults` | *(none)* | Suite mode: inject provider failures through the meter — absolute indices (`3:429`) and/or a cadence that scales with the run (`every:5:500`). Requires `-meter`. See [Fault recovery](#fault-recovery). |
|
||||
| `-segments` | `1` | Suite mode: split each task into N resumed legs (`--continue` between them). Every leg shares the task's one wall-clock timeout, so legs never buy extra working time. See [Segmented runs](#segmented-runs). |
|
||||
| `-segments` | `1` | Suite mode: split each task into N resumed legs (`--continue` between them). The step budget is **divided**, never multiplied. See [Segmented runs](#segmented-runs). |
|
||||
| `-steer` | *(none)* | Suite mode: deliver a user turn at a leg boundary, e.g. `"also handle empty input@2"`. Requires `-segments` to reach that leg. |
|
||||
|
||||
Diff-mode flags:
|
||||
@@ -447,7 +374,7 @@ Diff-mode flags:
|
||||
| `-repo` | `.` | Repo root (diff mode). |
|
||||
| `-base` | *(none)* | Base ref to diff the PR head against (diff mode). |
|
||||
| `-test-cmd` | `go test` | Grader command run on the affected packages (diff mode). |
|
||||
| `-max-steps` | `0` | Diff mode: agent tool-call cap. Zero — the shipped default — leaves the loop unbounded and lets `-timeout` be the only bound. |
|
||||
| `-max-steps` | `80` | Agent tool-call cap for the diff task. |
|
||||
| `-timeout` | `1200` | Agent timeout in seconds (diff mode). |
|
||||
| `-attempts` | `1` | Diff mode: retry up to N times until a run passes (stochastic agent). |
|
||||
|
||||
@@ -478,34 +405,6 @@ marginal-utility line (`accuracy +X.Xpp · wall/task +Y.Ys`), and — when tasks
|
||||
carry `class` labels — a per-class breakdown, so a subsystem's uplift and
|
||||
latency cost can be judged per task class instead of globally.
|
||||
|
||||
## Pricing the run graph
|
||||
|
||||
Delegation makes three separate claims, and one number cannot settle them. Each
|
||||
has its own arm, and all three read the same **Fan-out** section of the report —
|
||||
folded from the run graph, which is the only record of a scheduling wait:
|
||||
|
||||
| Claim | Arm |
|
||||
| --- | --- |
|
||||
| Delegating at all beats one agent | `-ablate subagent` |
|
||||
| A `depends_on` edge is worth its payload, not only its order | `-ablate upstream` over `benchmarks/upstream-edge` |
|
||||
| Running members side by side beats running them in turn | `max_subagent_concurrency = 1` over `benchmarks/fanout-width` |
|
||||
|
||||
```sh
|
||||
go run ./cmd/e2ebench -suite benchmarks/upstream-edge -json control.json
|
||||
go run ./cmd/e2ebench -suite benchmarks/upstream-edge -ablate upstream -json ablated.json
|
||||
go run ./cmd/e2ebench -mode compare control.json ablated.json
|
||||
```
|
||||
|
||||
The section reports work against wall (the speed-up the shape bought), the
|
||||
critical path (the floor the declared dependencies impose — wall above it is
|
||||
scheduling loss, not shape), and slot wait (what the concurrency ceiling cost
|
||||
members that had nothing left to wait for).
|
||||
|
||||
Read all three axes before calling an arm better: a fan-out that halves wall
|
||||
clock while doubling tokens wins one and loses another. The arm passes when its
|
||||
solve rate is no worse, its wall clock is strictly lower, and its tokens stay
|
||||
inside the budget the comparison declares.
|
||||
|
||||
## SWE-bench Verified mode
|
||||
|
||||
`e2ebench` can also run the agent inside the official SWE-bench evaluation
|
||||
@@ -527,7 +426,7 @@ writer.
|
||||
## Adding a new task
|
||||
|
||||
1. Create `benchmarks/e2e/tasks/<task-id>/`.
|
||||
2. Write `task.toml` with `prompt` and `timeout_sec` — no `max_steps` (see
|
||||
2. Write `task.toml` with `prompt`, `max_steps`, and `timeout_sec` (see
|
||||
[schema](#tasktoml-schema)).
|
||||
3. If the task needs seed files, add them under `workdir/` (they are copied
|
||||
into the temp run dir; symlinks are skipped).
|
||||
@@ -542,32 +441,33 @@ writer.
|
||||
|
||||
## context-maintenance-e2e
|
||||
|
||||
A cost-capped smoke for content-driven maintenance, in three legs over one state
|
||||
directory: seed a session just below the compaction trigger, load it back and
|
||||
check nothing is re-derived, then grow it past the trigger and check exactly one
|
||||
summary lands.
|
||||
This harness measures what happens when a long session goes idle past the
|
||||
provider's cache TTL and then resumes: it A/B-compares cold-restart miss tokens
|
||||
with and without pruning, and checks that the agent re-reads a file behind a
|
||||
prune placeholder instead of hallucinating.
|
||||
|
||||
Seed and resume take no provider; `continue` samples and needs
|
||||
`DEEPSEEK_API_KEY`. The model is hardcoded to `deepseek-v4-flash` at
|
||||
`https://api.deepseek.com`.
|
||||
It is hardcoded to the `deepseek-v4-flash` model at `https://api.deepseek.com`
|
||||
and requires the `DEEPSEEK_API_KEY` environment variable.
|
||||
|
||||
```sh
|
||||
# Seed below the trigger: no checkpoint installed, no summary started
|
||||
go run ./benchmarks/context-maintenance-e2e -offline seed
|
||||
export DEEPSEEK_API_KEY=...
|
||||
|
||||
# Reload: the projection version must not move and no summary may run
|
||||
# Seed both arms (pruned + control) with a large session and warm the cache
|
||||
go run ./benchmarks/context-maintenance-e2e seed
|
||||
|
||||
# Wait past the provider's cache TTL, then resume: prune the "pruned" arm and
|
||||
# compare cold-restart miss tokens
|
||||
go run ./benchmarks/context-maintenance-e2e resume
|
||||
|
||||
# Grow past the trigger: one summary, version +1, then no further summary
|
||||
export DEEPSEEK_API_KEY=...
|
||||
go run ./benchmarks/context-maintenance-e2e continue
|
||||
# Run the comprehension trials (agent must re-read a pruned file and answer
|
||||
# from it); exits non-zero unless every trial passes
|
||||
go run ./benchmarks/context-maintenance-e2e comprehension
|
||||
```
|
||||
|
||||
| Flag | Default | Purpose |
|
||||
| --- | --- | --- |
|
||||
| `-dir` | `benchmarks/context-maintenance-e2e/run` | State directory shared by the three legs (session + `meta.json`). |
|
||||
| `-max-usd` | `0.50` | Hard cost cap for the live legs. |
|
||||
| `-offline` | off | Skip the live provider; `seed` only. |
|
||||
| `-dir` | `benchmarks/context-maintenance-e2e/run` | State directory for `seed`/`resume` (sessions + `meta.json`, `resume-<ts>.json`). |
|
||||
| `-trials` | `5` | Number of comprehension trials. |
|
||||
|
||||
## See also
|
||||
|
||||
@@ -593,24 +493,6 @@ e2ebench -suite benchmarks/memorybench -budget 0 -policy memory-off -trajectorie
|
||||
e2ebench -mode compare on.json off.json # Memory utility section
|
||||
```
|
||||
|
||||
MemoryBench requires an OS read sandbox. It runs on macOS and Linux and
|
||||
refuses to start on Windows, where the benchmark cannot isolate the committed
|
||||
`tasks/<id>/memory/` answers from a graded agent. Each run also audits every
|
||||
recorded trajectory leg for a successful read of those answers; an answer read
|
||||
invalidates the result and makes the suite command exit nonzero. A missing or
|
||||
unreadable trajectory marks only that run incomplete; other tasks still run.
|
||||
Trajectories are captured temporarily when `-trajectories` is omitted.
|
||||
|
||||
The audit treats a completed `read_file` call naming a task-memory path as a
|
||||
read. For `bash` and `grep`, a task-memory path in the arguments plus an authored
|
||||
fact line in the result is **evidence** of a read, not a proof of how that text
|
||||
was obtained. `glob` and directory listings expose names but not fact bodies;
|
||||
the audit does not count them as answer reads. Commands that reach the answer
|
||||
through an indirect path, such as `cd` followed by a relative filename, may
|
||||
not be attributed by the trajectory check. The OS read sandbox is the primary
|
||||
boundary; this audit is a second check, not a proof that an unflagged run never
|
||||
accessed an answer.
|
||||
|
||||
Utility delta = paired Pass(on) − Pass(off). Harmful attribution is paired,
|
||||
never judged: the same task passed without memory and failed with it while
|
||||
recall fired. Scenario classes: exact, paraphrase, cjk, symbol, distractor
|
||||
@@ -624,8 +506,8 @@ update (revised value wins), pinned (prefix channel end to end).
|
||||
`benchmarks/compaction/` drives the real agent compaction path over a session
|
||||
that grows one generation at a time. Each generation appends a round of work
|
||||
and then folds, so generation N folds everything generations 1..N produced —
|
||||
which is the growth that matters, because a fold reads the model-visible
|
||||
projection, so the previous digest feeds the next one.
|
||||
which is the growth that matters, because a fold re-derives its digest from the
|
||||
whole canonical transcript rather than from the previous digest.
|
||||
|
||||
```bash
|
||||
go run ./benchmarks/compaction -mode=cost # offline, no API key
|
||||
@@ -652,3 +534,16 @@ probe, not a compaction loss.
|
||||
Probe answers are scored on whole words, and a wanted answer does not count if a
|
||||
rejected one appears anywhere in the same reply — "yes, but it has not been
|
||||
re-run since" is the shape a drifting digest produces, and it is not a pass.
|
||||
|
||||
### Fold arms
|
||||
|
||||
`-arm=full` (default) re-derives every digest from the canonical transcript, so
|
||||
digests never chain. `-arm=incremental` folds the model-visible view instead,
|
||||
feeding the previous digest back through the summarizer. The arms exist to price
|
||||
that trade: run the cost arm for what chaining saves, and the fidelity arm for
|
||||
what it costs.
|
||||
|
||||
```bash
|
||||
go run ./benchmarks/compaction -mode=cost -arm=incremental
|
||||
DEEPSEEK_API_KEY=… go run ./benchmarks/compaction -mode=fidelity -arm=incremental
|
||||
```
|
||||
@@ -1,24 +0,0 @@
|
||||
# catalog-detector
|
||||
|
||||
Who owns "the skills listing is owed again"?
|
||||
|
||||
The listing is delivered once. Something has to decide when the model's copy
|
||||
stopped matching the registry, and there are three candidates: rebuild the
|
||||
projection and compare it, fingerprint the files' metadata, or trust that every
|
||||
write announced itself.
|
||||
|
||||
```
|
||||
go run ./benchmarks/catalog-detector -real "$PWD"
|
||||
```
|
||||
|
||||
**Owed is measured, not declared.** The probe renders the block before and after
|
||||
each transition; owed means those differ. That definition is what makes a
|
||||
touched mtime and a body-only edit *not* owed — neither changes what the model
|
||||
would be sent — and it is why the projection detector is the definition rather
|
||||
than an estimate of it. What the arms measure is where the two cheaper
|
||||
detectors diverge from it.
|
||||
|
||||
Cost is timed without the instrument attached: files and bytes are counted in a
|
||||
separate walk, because timing a detector with the accounting inside it measures
|
||||
the accounting. Numbers are macOS/APFS with a warm cache; the ratio between
|
||||
detectors is what transfers, not the absolute.
|
||||
@@ -1,87 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// arm is one transition a workspace can undergo, with what it did to the thing
|
||||
// the model would be sent. Truth is measured, not declared: the block is
|
||||
// rendered before and after, and owed means those two differ.
|
||||
type arm struct {
|
||||
name string
|
||||
apply func(e env) error
|
||||
comment string
|
||||
}
|
||||
|
||||
var arms = []arm{
|
||||
{"api-create", func(e env) error { *e.apiWrites++; return writeSkill(e, "added", "a skill this process wrote") }, "a writer this process controls"},
|
||||
{"api-describe", func(e env) error {
|
||||
*e.apiWrites++
|
||||
return writeSkill(e, "probe-000", "a description this process rewrote")
|
||||
}, "same writer, different field"},
|
||||
{"api-delete", func(e env) error { *e.apiWrites++; return os.RemoveAll(skillDir(e, "probe-000")) }, "same writer, removal"},
|
||||
{"external-create", func(e env) error { return writeSkill(e, "outside", "a skill written past the API") }, "an editor, a script, the write tool"},
|
||||
{"external-describe", func(e env) error { return writeSkill(e, "probe-000", "a description rewritten past the API") }, "the common out-of-band edit"},
|
||||
{"external-delete", func(e env) error { return os.RemoveAll(skillDir(e, "probe-000")) }, "removal past the API"},
|
||||
{"mtime-restored", restoreMtimeChange, "same length, older mtime: git checkout, rsync, touch -r"},
|
||||
{"noop-touch", touchOnly, "mtime moved, nothing the model sees changed"},
|
||||
{"body-only", bodyOnlyChange, "the body changed; the index does not render it"},
|
||||
}
|
||||
|
||||
func skillDir(e env, name string) string {
|
||||
return filepath.Join(e.projectRoot, ".reasonix", "skills", name)
|
||||
}
|
||||
|
||||
func writeSkill(e env, name, description string) error {
|
||||
dir := skillDir(e, name)
|
||||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
return os.WriteFile(filepath.Join(dir, "SKILL.md"),
|
||||
fmt.Appendf(nil, "---\nname: %s\ndescription: %s\n---\n\nBody paragraph the index never renders.\n", name, description), 0o644)
|
||||
}
|
||||
|
||||
// restoreMtimeChange rewrites the rendered description without moving either
|
||||
// size or mtime. Nothing exotic produces this: a branch switch restores content
|
||||
// and stamps it with the mtime the file had.
|
||||
func restoreMtimeChange(e env) error {
|
||||
path := filepath.Join(skillDir(e, "probe-000"), "SKILL.md")
|
||||
before, err := os.Stat(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
old, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
swapped := strings.Replace(string(old), "probe skill 0, one line of index text.", "PROBE SKILL 0, ONE LINE OF INDEX TEXT.", 1)
|
||||
if swapped == string(old) {
|
||||
return fmt.Errorf("arm mtime-restored did not change the description; the fixture moved")
|
||||
}
|
||||
if len(swapped) != len(old) {
|
||||
return fmt.Errorf("arm mtime-restored changed the file length (%d -> %d); it must not", len(old), len(swapped))
|
||||
}
|
||||
if err := os.WriteFile(path, []byte(swapped), 0o644); err != nil {
|
||||
return err
|
||||
}
|
||||
return os.Chtimes(path, before.ModTime(), before.ModTime())
|
||||
}
|
||||
|
||||
func touchOnly(e env) error {
|
||||
path := filepath.Join(skillDir(e, "probe-000"), "SKILL.md")
|
||||
later := time.Now().Add(time.Hour)
|
||||
return os.Chtimes(path, later, later)
|
||||
}
|
||||
|
||||
func bodyOnlyChange(e env) error {
|
||||
path := filepath.Join(skillDir(e, "probe-000"), "SKILL.md")
|
||||
old, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return os.WriteFile(path, []byte(strings.Replace(string(old), "Body paragraph the index never renders.", "A different body paragraph, longer than the one it replaces.", 1)), 0o644)
|
||||
}
|
||||
@@ -1,90 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
type costRow struct {
|
||||
scale string
|
||||
detector string
|
||||
p50, p95 time.Duration
|
||||
files int
|
||||
bytes int64
|
||||
}
|
||||
|
||||
// measureCost times each detector over synthetic workspaces of a given size and,
|
||||
// when asked, over a real one. The synthetic skills are shaped like real ones —
|
||||
// a directory per skill holding SKILL.md — because the walk is what is being
|
||||
// measured and a flat directory would not have it.
|
||||
func measureCost(scales []int, samples int, realRoot string) []costRow {
|
||||
var rows []costRow
|
||||
for _, n := range scales {
|
||||
root, err := os.MkdirTemp("", "catalog-detector-*")
|
||||
fatal(err)
|
||||
defer os.RemoveAll(root)
|
||||
fatal(writeSkills(filepath.Join(root, "project", ".reasonix", "skills"), n))
|
||||
// A separate home: pointing both at one directory walks it twice and
|
||||
// reports double the cost a real workspace pays.
|
||||
rows = append(rows, timeDetectors(fmt.Sprintf("synthetic n=%d", n),
|
||||
env{projectRoot: filepath.Join(root, "project"), homeDir: filepath.Join(root, "home"), apiWrites: new(int)}, samples)...)
|
||||
}
|
||||
if realRoot != "" {
|
||||
home, err := os.UserHomeDir()
|
||||
fatal(err)
|
||||
rows = append(rows, timeDetectors("real workspace", env{projectRoot: realRoot, homeDir: home, apiWrites: new(int)}, samples)...)
|
||||
}
|
||||
return rows
|
||||
}
|
||||
|
||||
func timeDetectors(scale string, e env, samples int) []costRow {
|
||||
var rows []costRow
|
||||
files, bytes := walkCost(e.roots())
|
||||
for _, d := range detectors {
|
||||
var durations []time.Duration
|
||||
for range samples {
|
||||
start := time.Now()
|
||||
d.read(e)
|
||||
durations = append(durations, time.Since(start))
|
||||
}
|
||||
row := costRow{scale: scale, detector: d.name, p50: percentile(durations, 0.5), p95: percentile(durations, 0.95)}
|
||||
if d.walks {
|
||||
row.files, row.bytes = files, bytes
|
||||
}
|
||||
rows = append(rows, row)
|
||||
}
|
||||
return rows
|
||||
}
|
||||
|
||||
func writeSkills(dir string, n int) error {
|
||||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
for i := range n {
|
||||
sub := filepath.Join(dir, fmt.Sprintf("probe-%03d", i))
|
||||
if err := os.MkdirAll(sub, 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(sub, "SKILL.md"), []byte(skillFile(i, fmt.Sprintf("probe skill %d, one line of index text.", i))), 0o644); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func skillFile(i int, description string) string {
|
||||
return fmt.Sprintf("---\nname: probe-%03d\ndescription: %s\n---\n\nBody paragraph the index never renders.\n", i, description)
|
||||
}
|
||||
|
||||
func renderCost(rows []costRow) string {
|
||||
var b strings.Builder
|
||||
b.WriteString("cost — one reading, as a turn would take it\n\n")
|
||||
fmt.Fprintf(&b, "%-16s %-15s %10s %10s %7s %9s\n", "scale", "detector", "p50", "p95", "files", "bytes")
|
||||
for _, r := range rows {
|
||||
fmt.Fprintf(&b, "%-16s %-15s %10s %10s %7d %9d\n", r.scale, r.detector, r.p50.Round(time.Microsecond), r.p95.Round(time.Microsecond), r.files, r.bytes)
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
@@ -1,119 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"reasonix/internal/ext/skill"
|
||||
)
|
||||
|
||||
// detector answers one question: has the thing the model would be sent changed?
|
||||
// Each returns an opaque token — equal tokens mean "not owed". Nothing here
|
||||
// counts files or bytes: accounting is a second walk, and timing a detector
|
||||
// with the instrument attached measures the instrument.
|
||||
type detector struct {
|
||||
name string
|
||||
read func(env) string
|
||||
walks bool // whether this detector's cost is a directory walk
|
||||
}
|
||||
|
||||
// env is one workspace under measurement.
|
||||
type env struct {
|
||||
projectRoot string
|
||||
homeDir string
|
||||
// apiWrites is the observed-write detector's whole input: whether the last
|
||||
// mutation went through a writer this process controls.
|
||||
apiWrites *int
|
||||
}
|
||||
|
||||
func (e env) store() *skill.Store {
|
||||
return skill.New(skill.Options{
|
||||
ProjectRoot: e.projectRoot, ReasonixHomeDir: filepath.Join(e.homeDir, ".reasonix"), Stderr: io.Discard,
|
||||
})
|
||||
}
|
||||
|
||||
func (e env) roots() []string {
|
||||
var out []string
|
||||
for _, r := range e.store().Roots() {
|
||||
out = append(out, r.Dir)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// projectionDetector is the truth, not an estimate of it: it renders the block
|
||||
// the model would receive and compares that. A change no listing shows — a body
|
||||
// edit, a touched mtime — is by construction not owed.
|
||||
var projectionDetector = detector{
|
||||
name: "projection",
|
||||
walks: true,
|
||||
read: func(e env) string {
|
||||
return digest(skill.IndexBlock(e.store().List()))
|
||||
},
|
||||
}
|
||||
|
||||
// metadataDetector is the cheap approximation: path, size and mtime, no reads.
|
||||
var metadataDetector = detector{
|
||||
name: "metadata",
|
||||
walks: true,
|
||||
read: func(e env) string {
|
||||
var lines []string
|
||||
for _, dir := range e.roots() {
|
||||
_ = filepath.WalkDir(dir, func(p string, d fs.DirEntry, err error) error {
|
||||
if err != nil || d.IsDir() {
|
||||
return nil
|
||||
}
|
||||
info, err := d.Info()
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
lines = append(lines, fmt.Sprintf("%s\x00%d\x00%d", p, info.Size(), info.ModTime().UnixNano()))
|
||||
return nil
|
||||
})
|
||||
}
|
||||
sort.Strings(lines)
|
||||
return digest(strings.Join(lines, "\n"))
|
||||
},
|
||||
}
|
||||
|
||||
// observedWriteDetector sees only what this process was told about. It reads
|
||||
// nothing, so its cost is zero and its blindness is total: a write it was not
|
||||
// told about leaves the token where it was.
|
||||
var observedWriteDetector = detector{
|
||||
name: "observed-write",
|
||||
read: func(e env) string {
|
||||
return fmt.Sprintf("writes=%d", *e.apiWrites)
|
||||
},
|
||||
}
|
||||
|
||||
var detectors = []detector{projectionDetector, metadataDetector, observedWriteDetector}
|
||||
|
||||
// walkCost reports the shape of what a walking detector touches. It runs
|
||||
// outside the timed section for exactly the reason this file has two functions
|
||||
// instead of one.
|
||||
func walkCost(dirs []string) (files int, bytes int64) {
|
||||
for _, dir := range dirs {
|
||||
_ = filepath.WalkDir(dir, func(p string, d fs.DirEntry, err error) error {
|
||||
if err != nil || d.IsDir() {
|
||||
return nil
|
||||
}
|
||||
files++
|
||||
if info, err := os.Stat(p); err == nil {
|
||||
bytes += info.Size()
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
return files, bytes
|
||||
}
|
||||
|
||||
func digest(s string) string {
|
||||
sum := sha256.Sum256([]byte(s))
|
||||
return hex.EncodeToString(sum[:])[:12]
|
||||
}
|
||||
@@ -1,77 +0,0 @@
|
||||
// Which detector can own "the skills listing is owed again"? The answer decides
|
||||
// whether a projection debt is a flag every writer must remember to set, or a
|
||||
// question the host answers for itself. Owed means one thing here: the rendered
|
||||
// available-skills block differs from the one the model last received, so a
|
||||
// touched mtime and a body edit are not owed. This probe writes only inside its
|
||||
// own temporary workspaces.
|
||||
package main
|
||||
|
||||
import (
|
||||
"flag"
|
||||
"fmt"
|
||||
"os"
|
||||
"slices"
|
||||
"sort"
|
||||
"time"
|
||||
)
|
||||
|
||||
func main() {
|
||||
scales := flag.String("scales", "0,10,50,200", "synthetic skill counts to measure")
|
||||
samples := flag.Int("samples", 51, "cost samples per detector per scale")
|
||||
real := flag.String("real", "", "also measure this workspace as it is on disk")
|
||||
flag.Parse()
|
||||
|
||||
costs := measureCost(parseScales(*scales), *samples, *real)
|
||||
verdicts := measureVerdicts()
|
||||
|
||||
fmt.Println(renderCost(costs))
|
||||
fmt.Println()
|
||||
fmt.Println(renderVerdicts(verdicts))
|
||||
if failed(verdicts) {
|
||||
// Not an error: a detector that misses is the finding, and the exit
|
||||
// code stays zero so a study can report one.
|
||||
fmt.Println("\nA detector with a miss cannot own correctness; one with a false positive cannot own cache stability.")
|
||||
}
|
||||
}
|
||||
|
||||
func parseScales(s string) []int {
|
||||
var out []int
|
||||
for _, f := range splitComma(s) {
|
||||
var n int
|
||||
if _, err := fmt.Sscanf(f, "%d", &n); err == nil {
|
||||
out = append(out, n)
|
||||
}
|
||||
}
|
||||
sort.Ints(out)
|
||||
return out
|
||||
}
|
||||
|
||||
func splitComma(s string) []string {
|
||||
var out []string
|
||||
cur := ""
|
||||
for _, r := range s {
|
||||
if r == ',' {
|
||||
out = append(out, cur)
|
||||
cur = ""
|
||||
continue
|
||||
}
|
||||
cur += string(r)
|
||||
}
|
||||
return append(out, cur)
|
||||
}
|
||||
|
||||
func percentile(samples []time.Duration, p float64) time.Duration {
|
||||
if len(samples) == 0 {
|
||||
return 0
|
||||
}
|
||||
slices.Sort(samples)
|
||||
idx := int(float64(len(samples)-1) * p)
|
||||
return samples[idx]
|
||||
}
|
||||
|
||||
func fatal(err error) {
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, "catalog-detector:", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
@@ -1,77 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"reasonix/internal/ext/skill"
|
||||
)
|
||||
|
||||
type verdict struct {
|
||||
arm string
|
||||
comment string
|
||||
owed bool
|
||||
detector string
|
||||
changed bool
|
||||
}
|
||||
|
||||
func (v verdict) label() string {
|
||||
switch {
|
||||
case v.owed == v.changed:
|
||||
return "correct"
|
||||
case v.owed:
|
||||
return "MISS"
|
||||
default:
|
||||
return "false-positive"
|
||||
}
|
||||
}
|
||||
|
||||
// measureVerdicts runs every arm against a fresh workspace and records, for each
|
||||
// detector, whether it saw the transition. Truth is the rendered block itself:
|
||||
// the arm is owed when the listing the model would receive is not the listing it
|
||||
// already has.
|
||||
func measureVerdicts() []verdict {
|
||||
var out []verdict
|
||||
for _, a := range arms {
|
||||
root, err := os.MkdirTemp("", "catalog-detector-arm-*")
|
||||
fatal(err)
|
||||
defer os.RemoveAll(root)
|
||||
fatal(writeSkills(filepath.Join(root, "project", ".reasonix", "skills"), 3))
|
||||
e := env{projectRoot: filepath.Join(root, "project"), homeDir: filepath.Join(root, "home"), apiWrites: new(int)}
|
||||
|
||||
before := map[string]string{}
|
||||
for _, d := range detectors {
|
||||
before[d.name] = d.read(e)
|
||||
}
|
||||
blockBefore := skill.IndexBlock(e.store().List())
|
||||
|
||||
fatal(a.apply(e))
|
||||
|
||||
owed := skill.IndexBlock(e.store().List()) != blockBefore
|
||||
for _, d := range detectors {
|
||||
out = append(out, verdict{a.name, a.comment, owed, d.name, d.read(e) != before[d.name]})
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func renderVerdicts(vs []verdict) string {
|
||||
var b strings.Builder
|
||||
b.WriteString("verdicts — owed is measured, not declared: the rendered block differs\n\n")
|
||||
fmt.Fprintf(&b, "%-18s %-6s %-15s %-15s %s\n", "arm", "owed", "detector", "verdict", "what the arm is")
|
||||
for _, v := range vs {
|
||||
fmt.Fprintf(&b, "%-18s %-6v %-15s %-15s %s\n", v.arm, v.owed, v.detector, v.label(), v.comment)
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
func failed(vs []verdict) bool {
|
||||
for _, v := range vs {
|
||||
if v.label() != "correct" {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -15,16 +15,16 @@ import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reasonix/internal/state/sessionstore"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
|
||||
"reasonix/internal/base/tokencount"
|
||||
"reasonix/internal/contract/event"
|
||||
"reasonix/internal/contract/provider"
|
||||
"reasonix/internal/contract/tool"
|
||||
_ "reasonix/internal/model/openai"
|
||||
"reasonix/internal/runtime/agent"
|
||||
"reasonix/internal/ablation"
|
||||
"reasonix/internal/agent"
|
||||
"reasonix/internal/event"
|
||||
"reasonix/internal/provider"
|
||||
_ "reasonix/internal/provider/openai"
|
||||
"reasonix/internal/tool"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -41,7 +41,8 @@ func main() {
|
||||
report := flag.String("report", "1,2,4,8", "generations to report on")
|
||||
window := flag.Int("window", 128_000, "context window in tokens")
|
||||
control := flag.Bool("control", true, "fidelity: also score probes against full history")
|
||||
_ = flag.Bool("snip", false, "accepted and ignored: automatic snip projections are gone")
|
||||
arm := flag.String("arm", "full", "full | incremental: re-derive each digest from canonical, or fold the previous projection")
|
||||
snip := flag.Bool("snip", false, "legacy no-op: automatic snip projections are gone; kept so old scripts do not fail")
|
||||
out := flag.String("out", "", "write the JSON report here")
|
||||
flag.Parse()
|
||||
|
||||
@@ -49,11 +50,14 @@ func main() {
|
||||
res []genResult
|
||||
err error
|
||||
)
|
||||
switch *mode {
|
||||
case "cost":
|
||||
res, err = runCost(*gens, *window)
|
||||
case "fidelity":
|
||||
res, err = runFidelity(*gens, *window, *control)
|
||||
a := arms{incremental: *arm == "incremental", snip: *snip}
|
||||
switch {
|
||||
case *arm != "full" && *arm != "incremental":
|
||||
err = fmt.Errorf("unknown arm %q", *arm)
|
||||
case *mode == "cost":
|
||||
res, err = runCost(*gens, *window, a)
|
||||
case *mode == "fidelity":
|
||||
res, err = runFidelity(*gens, *window, *control, a)
|
||||
default:
|
||||
err = fmt.Errorf("unknown mode %q", *mode)
|
||||
}
|
||||
@@ -61,9 +65,9 @@ func main() {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
printReport(*mode, res, reportAt(*report))
|
||||
printReport(*mode+" / "+*arm, res, reportAt(*report))
|
||||
if *out != "" {
|
||||
b, _ := json.MarshalIndent(map[string]any{"mode": *mode, "window": *window, "generations": res}, "", " ")
|
||||
b, _ := json.MarshalIndent(map[string]any{"mode": *mode, "arm": *arm, "window": *window, "generations": res}, "", " ")
|
||||
if werr := os.WriteFile(*out, append(b, '\n'), 0o644); werr != nil {
|
||||
fmt.Fprintln(os.Stderr, werr)
|
||||
os.Exit(1)
|
||||
@@ -80,6 +84,8 @@ type genResult struct {
|
||||
SummarizerInput int `json:"summarizer_input_tokens"`
|
||||
LargestCall int `json:"largest_call_tokens"`
|
||||
Mode string `json:"mode,omitempty"`
|
||||
SnippedResults int `json:"snipped_results,omitempty"`
|
||||
SnippedChars int `json:"snipped_chars,omitempty"`
|
||||
Seconds float64 `json:"seconds"`
|
||||
Error string `json:"error,omitempty"`
|
||||
Survived map[string]int `json:"survived,omitempty"` // probe class -> 1 kept, 0 lost
|
||||
@@ -90,13 +96,14 @@ type genResult struct {
|
||||
}
|
||||
|
||||
type harness struct {
|
||||
sess *sessionstore.Session
|
||||
sess *agent.Session
|
||||
agentA *agent.Agent
|
||||
path string
|
||||
calls *callRecorder
|
||||
snip bool
|
||||
}
|
||||
|
||||
func newHarness(t *testingDir, p provider.Provider, window int, rec *callRecorder) *harness {
|
||||
func newHarness(t *testingDir, p provider.Provider, window int, rec *callRecorder, arm arms) *harness {
|
||||
sess := newSession()
|
||||
path := filepath.Join(t.dir, "session.jsonl")
|
||||
a := agent.New(p, tool.NewRegistry(), sess, agent.Options{
|
||||
@@ -107,18 +114,44 @@ func newHarness(t *testingDir, p provider.Provider, window int, rec *callRecorde
|
||||
// boot's default when cfg.Agent.Keep is unset; without it the bench
|
||||
// would measure a configuration no real session runs.
|
||||
KeepPolicy: agent.KeepErrors,
|
||||
Ablation: foldArm(arm.incremental),
|
||||
}, rec.sink())
|
||||
return &harness{sess: sess, agentA: a, path: path, calls: rec}
|
||||
return &harness{sess: sess, agentA: a, path: path, calls: rec, snip: arm.snip}
|
||||
}
|
||||
|
||||
// foldArm switches full re-derivation off, which is what makes a fold read the
|
||||
// previous projection instead of the canonical transcript.
|
||||
// arms selects the maintenance behaviour under test. Snipping is off by default
|
||||
// so a run stays comparable with baselines recorded before it existed.
|
||||
type arms struct {
|
||||
incremental bool
|
||||
snip bool
|
||||
}
|
||||
|
||||
func foldArm(incremental bool) ablation.Set {
|
||||
if incremental {
|
||||
return ablation.New(ablation.FullFold)
|
||||
}
|
||||
return ablation.Set{}
|
||||
}
|
||||
|
||||
// runGeneration grows the session and folds it, returning what that fold cost.
|
||||
func (h *harness) runGeneration(ctx context.Context, gen int, probes []probe) genResult {
|
||||
growSession(h.sess, gen, probes)
|
||||
r := genResult{Gen: gen, CanonicalTokens: tokencount.Text(renderAll(h.sess.Snapshot()))}
|
||||
r := genResult{Gen: gen, CanonicalTokens: estimateTokens(renderAll(h.sess.Snapshot()))}
|
||||
|
||||
h.calls.reset()
|
||||
start := time.Now()
|
||||
_, err := h.agentA.CompactNow(ctx, agent.CompactRequest{})
|
||||
if h.snip {
|
||||
// SnipStaleToolResults is intentionally a no-op; record zeros for
|
||||
// report schema compatibility with pre-content-driven baselines.
|
||||
st, serr := h.agentA.SnipStaleToolResults()
|
||||
if serr != nil {
|
||||
r.Error = serr.Error()
|
||||
}
|
||||
r.SnippedResults, r.SnippedChars = st.Results, st.SavedChars
|
||||
}
|
||||
err := h.agentA.CompactNow(ctx, "")
|
||||
r.Seconds = time.Since(start).Seconds()
|
||||
if err != nil {
|
||||
r.Error = err.Error()
|
||||
@@ -128,7 +161,7 @@ func (h *harness) runGeneration(ctx context.Context, gen int, probes []probe) ge
|
||||
r.SummarizerInput += c.tokens
|
||||
r.LargestCall = max(r.LargestCall, c.tokens)
|
||||
}
|
||||
if st, ok, sterr := sessionstore.LoadCompactionState(h.path); sterr == nil && ok {
|
||||
if st, ok, sterr := agent.LoadCompactionState(h.path); sterr == nil && ok {
|
||||
r.ProjectionTokens = st.Projection.ProjectionTokens
|
||||
if st.LastReceipt != nil && st.LastReceipt.Action == "summary" {
|
||||
r.Mode = agent.CompactionModeSummarized
|
||||
@@ -139,7 +172,7 @@ func (h *harness) runGeneration(ctx context.Context, gen int, probes []probe) ge
|
||||
return r
|
||||
}
|
||||
|
||||
func runCost(gens, window int) ([]genResult, error) {
|
||||
func runCost(gens, window int, a arms) ([]genResult, error) {
|
||||
dir, cleanup, err := tempDir()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -148,7 +181,7 @@ func runCost(gens, window int) ([]genResult, error) {
|
||||
|
||||
rec := &callRecorder{}
|
||||
p := &scriptedProvider{rec: rec, reply: syntheticDigest, window: window}
|
||||
h := newHarness(dir, p, window, rec)
|
||||
h := newHarness(dir, p, window, rec, a)
|
||||
|
||||
var out []genResult
|
||||
for gen := range gens {
|
||||
@@ -157,7 +190,7 @@ func runCost(gens, window int) ([]genResult, error) {
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func runFidelity(gens, window int, control bool) ([]genResult, error) {
|
||||
func runFidelity(gens, window int, control bool, a arms) ([]genResult, error) {
|
||||
key := os.Getenv("DEEPSEEK_API_KEY")
|
||||
if key == "" {
|
||||
return nil, fmt.Errorf("fidelity mode needs DEEPSEEK_API_KEY")
|
||||
@@ -173,7 +206,7 @@ func runFidelity(gens, window int, control bool) ([]genResult, error) {
|
||||
defer cleanup()
|
||||
|
||||
rec := &callRecorder{}
|
||||
h := newHarness(dir, &recordingProvider{inner: p, rec: rec}, window, rec)
|
||||
h := newHarness(dir, &recordingProvider{inner: p, rec: rec}, window, rec, a)
|
||||
probes := probeSuite()
|
||||
|
||||
ctx := context.Background()
|
||||
@@ -357,6 +390,18 @@ func reportAt(spec string) map[int]bool {
|
||||
return at
|
||||
}
|
||||
|
||||
// estimateTokens mirrors the kernel's own estimator so bench numbers and
|
||||
// compaction telemetry are read in the same unit.
|
||||
func estimateTokens(s string) int {
|
||||
if s == "" {
|
||||
return 0
|
||||
}
|
||||
if runes := utf8.RuneCountInString(s); runes > (len(s)+3)/4 {
|
||||
return runes
|
||||
}
|
||||
return (len(s) + 3) / 4
|
||||
}
|
||||
|
||||
func renderAll(msgs []provider.Message) string {
|
||||
var b strings.Builder
|
||||
for _, m := range msgs {
|
||||
@@ -407,7 +452,7 @@ func (r *callRecorder) reset() { r.calls = nil }
|
||||
func (r *callRecorder) note(req provider.Request) {
|
||||
c := recordedCall{}
|
||||
for _, m := range req.Messages {
|
||||
c.tokens += tokencount.Text(m.Content)
|
||||
c.tokens += estimateTokens(m.Content)
|
||||
if m.Role == provider.RoleSystem {
|
||||
c.system = m.Content
|
||||
}
|
||||
|
||||
@@ -12,7 +12,7 @@ func TestRepeatedFoldsStayBoundedAndKeepSucceeding(t *testing.T) {
|
||||
reserve = 8192 // summaryOutputReserve: the digest the call must still return
|
||||
maxCalls = 7 // maxSummarySpans + the merge pass
|
||||
)
|
||||
res, err := runCost(gens, window)
|
||||
res, err := runCost(gens, window, arms{})
|
||||
if err != nil {
|
||||
t.Fatalf("runCost: %v", err)
|
||||
}
|
||||
|
||||
@@ -2,13 +2,13 @@ package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"reasonix/internal/state/sessionstore"
|
||||
"slices"
|
||||
"strings"
|
||||
"unicode"
|
||||
|
||||
"reasonix/internal/contract/provider"
|
||||
"reasonix/internal/contract/tool"
|
||||
"reasonix/internal/agent"
|
||||
"reasonix/internal/provider"
|
||||
"reasonix/internal/tool"
|
||||
)
|
||||
|
||||
// probeAnswerContract rides with every probe question. A context full of tool
|
||||
@@ -36,22 +36,22 @@ func invalidAnswer(s string) bool {
|
||||
type probe struct {
|
||||
class string
|
||||
plantAt int
|
||||
plant func(*sessionstore.Session)
|
||||
plant func(*agent.Session)
|
||||
// later plants more of the same fact in later generations, so a decision
|
||||
// that changes across folds is tested the way it actually happens: each
|
||||
// revision lands on the far side of a compaction, not next to the last one.
|
||||
later map[int]func(*sessionstore.Session)
|
||||
later map[int]func(*agent.Session)
|
||||
question string
|
||||
want []string // answer must contain one of these, lowercased
|
||||
reject []string // ...and none of these: the pre-correction answer
|
||||
}
|
||||
|
||||
func userTurn(text string) func(*sessionstore.Session) {
|
||||
return func(s *sessionstore.Session) { s.Add(provider.Message{Role: provider.RoleUser, Content: text}) }
|
||||
func userTurn(text string) func(*agent.Session) {
|
||||
return func(s *agent.Session) { s.Add(provider.Message{Role: provider.RoleUser, Content: text}) }
|
||||
}
|
||||
|
||||
func toolRound(id, name, args, result string) func(*sessionstore.Session) {
|
||||
return func(s *sessionstore.Session) {
|
||||
func toolRound(id, name, args, result string) func(*agent.Session) {
|
||||
return func(s *agent.Session) {
|
||||
s.Add(provider.Message{Role: provider.RoleAssistant, ToolCalls: []provider.ToolCall{{ID: id, Name: name, Arguments: args}}})
|
||||
s.Add(provider.Message{Role: provider.RoleTool, ToolCallID: id, Name: name, Content: result})
|
||||
}
|
||||
@@ -60,8 +60,8 @@ func toolRound(id, name, args, result string) func(*sessionstore.Session) {
|
||||
// failedToolRound is a tool round the host recorded as failed, the way a real
|
||||
// non-zero bash run arrives. Without the execution record the keep policy sees
|
||||
// only text, which is exactly the gap the buried-evidence probe measures.
|
||||
func failedToolRound(id, name, args, result string, code int) func(*sessionstore.Session) {
|
||||
return func(s *sessionstore.Session) {
|
||||
func failedToolRound(id, name, args, result string, code int) func(*agent.Session) {
|
||||
return func(s *agent.Session) {
|
||||
exit := code
|
||||
s.Add(provider.Message{Role: provider.RoleAssistant, ToolCalls: []provider.ToolCall{{ID: id, Name: name, Arguments: args}}})
|
||||
s.Add(provider.Message{
|
||||
@@ -75,8 +75,8 @@ func failedToolRound(id, name, args, result string, code int) func(*sessionstore
|
||||
}
|
||||
}
|
||||
|
||||
func seq(fns ...func(*sessionstore.Session)) func(*sessionstore.Session) {
|
||||
return func(s *sessionstore.Session) {
|
||||
func seq(fns ...func(*agent.Session)) func(*agent.Session) {
|
||||
return func(s *agent.Session) {
|
||||
for _, fn := range fns {
|
||||
fn(s)
|
||||
}
|
||||
@@ -240,8 +240,8 @@ func revisionProbes() []probe {
|
||||
{
|
||||
class: "late-constraint",
|
||||
plantAt: 3,
|
||||
plant: userTurn("New hard rule from here on: never edit anything under internal/state/store/."),
|
||||
question: "Are you allowed to edit files under internal/state/store/? Answer yes or no.",
|
||||
plant: userTurn("New hard rule from here on: never edit anything under internal/store/."),
|
||||
question: "Are you allowed to edit files under internal/store/? Answer yes or no.",
|
||||
want: []string{"no"},
|
||||
reject: []string{"yes"},
|
||||
},
|
||||
@@ -249,7 +249,7 @@ func revisionProbes() []probe {
|
||||
class: "reversal-chain",
|
||||
plantAt: 0,
|
||||
plant: userTurn("Use PostgreSQL for the datastore."),
|
||||
later: map[int]func(*sessionstore.Session){
|
||||
later: map[int]func(*agent.Session){
|
||||
1: userTurn("Change of plan: drop PostgreSQL, use SQLite instead."),
|
||||
2: userTurn("Final call: back to PostgreSQL after all."),
|
||||
},
|
||||
@@ -261,7 +261,7 @@ func revisionProbes() []probe {
|
||||
class: "distractor-file",
|
||||
plantAt: 1,
|
||||
plant: userTurn("Draft note: the fix might belong in config/legacy_parser.go."),
|
||||
later: map[int]func(*sessionstore.Session){
|
||||
later: map[int]func(*agent.Session){
|
||||
3: userTurn("Confirmed: the fix belongs in config/emitter.go; the legacy parser is not involved."),
|
||||
},
|
||||
question: "Which file does the fix belong in? Reply with just the filename, no path.",
|
||||
|
||||
@@ -2,10 +2,10 @@ package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"reasonix/internal/state/sessionstore"
|
||||
"strings"
|
||||
|
||||
"reasonix/internal/contract/provider"
|
||||
"reasonix/internal/agent"
|
||||
"reasonix/internal/provider"
|
||||
)
|
||||
|
||||
// A generation is one round of work followed by one compaction. Compaction
|
||||
@@ -19,7 +19,7 @@ const (
|
||||
|
||||
// workUnit is one read_file round: the shape that dominates a real coding
|
||||
// session's token weight.
|
||||
func workUnit(sess *sessionstore.Session, gen, i int) {
|
||||
func workUnit(sess *agent.Session, gen, i int) {
|
||||
id := fmt.Sprintf("g%02dc%02d", gen, i)
|
||||
path := fmt.Sprintf("internal/pkg%02d/file%02d.go", gen, i)
|
||||
sess.Add(provider.Message{Role: provider.RoleAssistant, ToolCalls: []provider.ToolCall{
|
||||
@@ -41,7 +41,7 @@ func fakeGoFile(gen, i, size int) string {
|
||||
// growSession appends one generation of work, planting whatever probes belong
|
||||
// to that generation so a fact's age is exactly the number of folds it has
|
||||
// survived.
|
||||
func growSession(sess *sessionstore.Session, gen int, probes []probe) {
|
||||
func growSession(sess *agent.Session, gen int, probes []probe) {
|
||||
for _, p := range probes {
|
||||
if p.plantAt == gen {
|
||||
p.plant(sess)
|
||||
@@ -55,16 +55,16 @@ func growSession(sess *sessionstore.Session, gen int, probes []probe) {
|
||||
}
|
||||
}
|
||||
|
||||
func newSession() *sessionstore.Session {
|
||||
sess := sessionstore.NewSession("You are a terse coding agent. Answer questions from what you know; do not guess.")
|
||||
func newSession() *agent.Session {
|
||||
sess := agent.NewSession("You are a terse coding agent. Answer questions from what you know; do not guess.")
|
||||
sess.Add(provider.Message{Role: provider.RoleUser, Content: "Fix the config round-trip formatting bug in this repo."})
|
||||
return sess
|
||||
}
|
||||
|
||||
// visibleContext splices the stored projection with the canonical messages
|
||||
// appended after it — the same model-visible view the agent would send.
|
||||
func visibleContext(sessionPath string, sess *sessionstore.Session) ([]provider.Message, error) {
|
||||
st, ok, err := sessionstore.LoadCompactionState(sessionPath)
|
||||
func visibleContext(sessionPath string, sess *agent.Session) ([]provider.Message, error) {
|
||||
st, ok, err := agent.LoadCompactionState(sessionPath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@@ -0,0 +1,41 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// Automatic snip projections are gone. SnipStaleToolResults remains as a no-op
|
||||
// so older harness flags and call sites do not panic or rewrite history.
|
||||
func TestSnipIsNoOp(t *testing.T) {
|
||||
dir, cleanup, err := tempDir()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer cleanup()
|
||||
|
||||
rec := &callRecorder{}
|
||||
const window = 64_000
|
||||
h := newHarness(dir, &scriptedProvider{rec: rec, reply: syntheticDigest, window: window}, window, rec, arms{snip: true})
|
||||
for gen := range 3 {
|
||||
growSession(h.sess, gen, probeSuite())
|
||||
}
|
||||
before := renderAll(h.sess.Snapshot())
|
||||
if !strings.Contains(before, "beta-7d21") {
|
||||
t.Fatal("planted fact missing before the snip call")
|
||||
}
|
||||
st, err := h.agentA.SnipStaleToolResults()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if st.Results != 0 || st.SavedChars != 0 {
|
||||
t.Fatalf("snip no-op rewrote history: results=%d chars=%d", st.Results, st.SavedChars)
|
||||
}
|
||||
after := renderAll(h.sess.Snapshot())
|
||||
if after != before {
|
||||
t.Fatal("canonical transcript changed after SnipStaleToolResults")
|
||||
}
|
||||
if !strings.Contains(after, "beta-7d21") {
|
||||
t.Error("planted fact missing after snip no-op")
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,6 @@
|
||||
// Cost-capped seed → resume → continue smoke for content-driven maintenance.
|
||||
// Seed and resume run without a provider; live continue needs DEEPSEEK_API_KEY
|
||||
// and starts exactly one summary after growing past compact_ratio=0.85.
|
||||
// Offline covers seed/resume; live continue needs DEEPSEEK_API_KEY and allows
|
||||
// at most one summary after growing past compact_ratio=0.85.
|
||||
package main
|
||||
|
||||
import (
|
||||
@@ -10,15 +10,14 @@ import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reasonix/internal/state/sessionstore"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"reasonix/internal/contract/event"
|
||||
"reasonix/internal/contract/provider"
|
||||
"reasonix/internal/contract/tool"
|
||||
_ "reasonix/internal/model/openai"
|
||||
"reasonix/internal/runtime/agent"
|
||||
"reasonix/internal/agent"
|
||||
"reasonix/internal/event"
|
||||
"reasonix/internal/provider"
|
||||
_ "reasonix/internal/provider/openai"
|
||||
"reasonix/internal/tool"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -70,7 +69,7 @@ func (s *recordSink) Emit(e event.Event) {
|
||||
func prov() (provider.Provider, *provider.Pricing) {
|
||||
key := os.Getenv("DEEPSEEK_API_KEY")
|
||||
if key == "" {
|
||||
fmt.Fprintln(os.Stderr, "DEEPSEEK_API_KEY not set (use -offline for seed)")
|
||||
fmt.Fprintln(os.Stderr, "DEEPSEEK_API_KEY not set (use -offline for seed/resume)")
|
||||
os.Exit(1)
|
||||
}
|
||||
p, err := provider.New("openai", provider.Config{
|
||||
@@ -104,7 +103,7 @@ func fatHistory(n int) []provider.Message {
|
||||
)
|
||||
}
|
||||
|
||||
func newAgent(p provider.Provider, sess *sessionstore.Session, path string, sink event.Sink) *agent.Agent {
|
||||
func newAgent(p provider.Provider, sess *agent.Session, path string, sink event.Sink) *agent.Agent {
|
||||
return agent.New(p, tool.NewRegistry(), sess, agent.Options{
|
||||
ContextWindow: windowTokens,
|
||||
CompactRatio: compactRatio,
|
||||
@@ -145,7 +144,7 @@ func seed(dir string, offline bool, cap *costCap) {
|
||||
}
|
||||
path := filepath.Join(dir, "session.jsonl")
|
||||
// ~80×32KB tool results: well under 85% of 1M for a cold estimate.
|
||||
sess := &sessionstore.Session{Messages: fatHistory(80)}
|
||||
sess := &agent.Session{Messages: fatHistory(80)}
|
||||
sink := &recordSink{}
|
||||
var p provider.Provider
|
||||
if !offline {
|
||||
@@ -193,19 +192,20 @@ func seed(dir string, offline bool, cap *costCap) {
|
||||
m.ProjectedTokens, m.TriggerTokens, m.ProjectionVersion, m.SummaryCalls, m.SpentUSD)
|
||||
}
|
||||
|
||||
// resume reads the seeded session back and checks that loading it neither
|
||||
// advances the projection version nor pays for a summary. It never samples, so
|
||||
// it takes no provider and needs no API key.
|
||||
func resume(dir string) {
|
||||
func resume(dir string, offline bool) {
|
||||
path := filepath.Join(dir, "session.jsonl")
|
||||
m := readMeta(dir)
|
||||
sess, err := sessionstore.LoadSession(path)
|
||||
sess, err := agent.LoadSession(path)
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
sink := &recordSink{}
|
||||
a := newAgent(nil, sess, path, sink)
|
||||
var p provider.Provider
|
||||
if !offline {
|
||||
p, _ = prov()
|
||||
}
|
||||
a := newAgent(p, sess, path, sink)
|
||||
snap := a.ContextMaintenanceSnapshot()
|
||||
if snap.ProjectionVersion != m.ProjectionVersion {
|
||||
fmt.Fprintf(os.Stderr, "resume version = %d, want %d\n", snap.ProjectionVersion, m.ProjectionVersion)
|
||||
@@ -235,7 +235,7 @@ func cont(dir string, offline bool, cap *costCap) {
|
||||
}
|
||||
path := filepath.Join(dir, "session.jsonl")
|
||||
m := readMeta(dir)
|
||||
sess, err := sessionstore.LoadSession(path)
|
||||
sess, err := agent.LoadSession(path)
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
@@ -257,14 +257,12 @@ func cont(dir string, offline bool, cap *costCap) {
|
||||
fmt.Fprintln(os.Stderr, "continue prepare:", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
// The fixture is already above the trigger it was grown past, so a leg that
|
||||
// starts no summary has not observed the fold it exists to test.
|
||||
if sink.summaryStarts != 1 {
|
||||
fmt.Fprintf(os.Stderr, "continue started %d summaries, want exactly 1\n", sink.summaryStarts)
|
||||
if sink.summaryStarts > 1 {
|
||||
fmt.Fprintf(os.Stderr, "continue started %d summaries, want ≤1\n", sink.summaryStarts)
|
||||
os.Exit(1)
|
||||
}
|
||||
after := a.ContextMaintenanceSnapshot()
|
||||
if after.ProjectionVersion != m.ProjectionVersion+1 {
|
||||
if sink.summaryStarts == 1 && after.ProjectionVersion != m.ProjectionVersion+1 {
|
||||
fmt.Fprintf(os.Stderr, "continue version = %d, want %d after one summary\n",
|
||||
after.ProjectionVersion, m.ProjectionVersion+1)
|
||||
os.Exit(1)
|
||||
@@ -307,14 +305,14 @@ func cont(dir string, offline bool, cap *costCap) {
|
||||
func main() {
|
||||
dir := flag.String("dir", "benchmarks/context-maintenance-e2e/run", "state directory")
|
||||
maxUSD := flag.Float64("max-usd", defaultMaxUSD, "hard cost cap for live API legs")
|
||||
offline := flag.Bool("offline", false, "skip live provider (seed only)")
|
||||
offline := flag.Bool("offline", false, "skip live provider (seed/resume only)")
|
||||
flag.Parse()
|
||||
cap := &costCap{maxUSD: *maxUSD}
|
||||
switch flag.Arg(0) {
|
||||
case "seed":
|
||||
seed(*dir, *offline, cap)
|
||||
case "resume":
|
||||
resume(*dir)
|
||||
resume(*dir, *offline)
|
||||
case "continue":
|
||||
cont(*dir, *offline, cap)
|
||||
default:
|
||||
|
||||
@@ -1,98 +0,0 @@
|
||||
# Context retrieval bench
|
||||
|
||||
Two questions about long-horizon memory, and the harness that makes their
|
||||
answers trustworthy.
|
||||
|
||||
-mode=preflight build all 12 tasks under every arm and check the contract
|
||||
-mode=snippet does a rank-1 hit hand over the whole answer?
|
||||
-mode=adversarial send a model after the answer through every host surface
|
||||
-mode=run-search Experiment R: is searchable canonical a real capability?
|
||||
-mode=run-boundary Experiment I, paired: 6 tasks across their own cue boundary
|
||||
-mode=run-index Experiment I, full: 6 tasks x 4 index budgets
|
||||
-mode=calibrate find each index task's PlantAfterGen for its cue tier
|
||||
|
||||
`-dry` drives any run mode with a scripted provider, for nothing. Real runs need
|
||||
`DEEPSEEK_API_KEY`.
|
||||
|
||||
## Status
|
||||
|
||||
**Searchable canonical recall: proven.** Six of six recovered with search on,
|
||||
three of six with it off, and the successes without it came from enumerating
|
||||
addresses by hand at roughly nine times the page-in cost (139 vs 1292
|
||||
tokens/task). Measured after three separate leaks were closed, so the numbers
|
||||
are from a clean environment.
|
||||
|
||||
**Fold index marginal utility: unresolved.** Both the positive and the negative
|
||||
estimates are invalidated. The first Stage 2 ran in a contaminated environment.
|
||||
The clean paired batch found a consistent effect (six of six tasks searched less
|
||||
and paged in less with the cue visible), and the same-batch dose-response did
|
||||
not reproduce it (`boundary-aligned 0/4`, recall-token delta reversing sign).
|
||||
The most likely reason is that two of the six index tasks carry heavy-tail
|
||||
stopping behaviour that swamps the effect being measured.
|
||||
|
||||
Existing evidence is not sufficient to tune the shipped 1% budget in either
|
||||
direction. Reopening this wants a new corpus, not more samples: see
|
||||
"Index corpus v2" below.
|
||||
|
||||
**Things that turned out not to be problems**, each after being measured rather
|
||||
than argued about:
|
||||
|
||||
| Suspected | Measured |
|
||||
| --- | --- |
|
||||
| Model writes poor queries (2.17 searches/task) | 22/23 found the target on query 1; the extra searches are one round's parallel fan-out |
|
||||
| Model reaches for the workspace before memory (7/11) | Linear event order misread; by model round it is MemoryFirst 4/6 |
|
||||
| Snippets too short for multi-value answers | 240 runes covers 12/12 tasks at 100%; records are 30-179 runes |
|
||||
| A general stopping failure | 37/44 runs issued no search after the answer was in hand; all 7 that did belong to 2 tasks |
|
||||
|
||||
## Measurement distinctions this bench had to learn
|
||||
|
||||
Each of these reversed a conclusion once. They are enforced in the schema now,
|
||||
not left to whoever reads the numbers next.
|
||||
|
||||
- **TargetHit is not EvidenceSufficient.** A rank-1 hit can return a window too
|
||||
short to answer with. `FirstHitCoverage` records how much of the scored answer
|
||||
the first hit actually handed over.
|
||||
- **Linear event order is not model-round order.** Two calls in one round are a
|
||||
parallel fan-out, not a preference. Routing compares rounds.
|
||||
- **An isolated workdir is not an isolated host.** The sandbox mounts the host
|
||||
read-only by design. Three leaks were found and closed: the corpus source, the
|
||||
fixture transcript, and its event log sidecar.
|
||||
- **A count is not its evidence.** Query text, snippet contents and trajectories
|
||||
are all persisted now, because three analyses in a row needed a paid re-run to
|
||||
ask a question of data already collected.
|
||||
|
||||
## How a run stays honest
|
||||
|
||||
The corpus holds templates, never answers: every scored literal is a `{{var}}`
|
||||
instantiated per run, so grepping this directory reveals the question and not
|
||||
the answer. `TestNoAnswerLiteralExistsInTheRepository` asserts it with
|
||||
`git grep`. Preflight rebuilds every arm and checks the answer is unreadable in
|
||||
the real `provider.Request`, the probe query ranks the target within five, and
|
||||
an index task's cue sits at exactly the scales its tier names. The fixture is
|
||||
removed from disk once the agent holds it in memory, and the directory is
|
||||
scanned for answer literals before the first provider request and again after
|
||||
the turn. Any answer reaching the model through a tool that is not `recall`
|
||||
marks the run contaminated and takes it out of every statistic.
|
||||
|
||||
## Index corpus v2, if this is reopened
|
||||
|
||||
The present index tasks are not a fit substrate: two of six carry stopping
|
||||
behaviour that dominates the signal. A task should qualify only if a run-index
|
||||
run reports a `stopping_class` other than `PostSufficientRetrieval` or
|
||||
`PostSufficientRunaway`: `SnippetStop`, `SnippetThenRead` and `DefensiveRead`
|
||||
all qualify, and a defensive read is deliberately not counted as waste. The
|
||||
class comes from the three counts in that report — `searches_after_sufficient`,
|
||||
`reads_after_sufficient` and `escapes_after_sufficient` — together with how
|
||||
sufficiency was reached and how many rounds followed it. The remaining
|
||||
requirements still apply — successful tool call, answer in the output rather
|
||||
than the cue, rank 1, `FirstHitCoverage` complete, one or two simple values,
|
||||
nothing in the workspace. Keep i03 and i04 — as a stopping corpus, not an
|
||||
efficiency one.
|
||||
|
||||
`-mode=calibrate` is not a screening tool: it builds and reads back the
|
||||
fixtures, and reads nothing else, so its cue-visibility profile is the same for
|
||||
every task.
|
||||
|
||||
Then repeats rather than levels: cue-present against cue-absent on each task's
|
||||
own boundary, three runs per cell. What is unknown is the variance, and four
|
||||
budgets sampled once each cannot show it.
|
||||
@@ -1,171 +0,0 @@
|
||||
// capture.go — what the agent actually hands a provider.
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"reasonix/internal/state/sessionstore"
|
||||
"strings"
|
||||
|
||||
"reasonix/internal/assembly/boot"
|
||||
"reasonix/internal/contract/ablation"
|
||||
"reasonix/internal/contract/event"
|
||||
"reasonix/internal/contract/provider"
|
||||
"reasonix/internal/contract/tool"
|
||||
"reasonix/internal/runtime/agent"
|
||||
_ "reasonix/internal/tools/builtin"
|
||||
)
|
||||
|
||||
// A projection is what the agent believes; a provider.Request is what the model
|
||||
// is told. Rendering, sanitization and tool-schema construction sit between
|
||||
// them, so the arm's contract is checked on the request.
|
||||
|
||||
type captureProvider struct {
|
||||
requests []provider.Request
|
||||
}
|
||||
|
||||
func (p *captureProvider) Name() string { return "contextbench-capture" }
|
||||
|
||||
// Stream records the request and answers with a clean terminal response. An
|
||||
// empty answer would buy an empty-final retry and a second request, which is
|
||||
// not the one round this is here to inspect.
|
||||
func (p *captureProvider) Stream(_ context.Context, req provider.Request) (<-chan provider.Chunk, error) {
|
||||
p.requests = append(p.requests, cloneRequest(req))
|
||||
ch := make(chan provider.Chunk, 2)
|
||||
ch <- provider.Chunk{Type: provider.ChunkText, Text: "contextbench preflight complete"}
|
||||
ch <- provider.Chunk{Type: provider.ChunkDone}
|
||||
close(ch)
|
||||
return ch, nil
|
||||
}
|
||||
|
||||
func cloneRequest(req provider.Request) provider.Request {
|
||||
out := req
|
||||
out.Messages = append([]provider.Message(nil), req.Messages...)
|
||||
out.Tools = append([]provider.ToolSchema(nil), req.Tools...)
|
||||
return out
|
||||
}
|
||||
|
||||
// captureRequest rebuilds the session the way a resume does — transcript from
|
||||
// disk, projection from its sidecar, the shipped tool surface at this arm —
|
||||
// and returns the request the first round hands the provider.
|
||||
func captureRequest(f builtFixture, arm ablation.Set, prompt string) (provider.Request, error) {
|
||||
sess, err := sessionstore.LoadSession(f.Path)
|
||||
if err != nil {
|
||||
return provider.Request{}, fmt.Errorf("load session: %w", err)
|
||||
}
|
||||
reg := tool.NewRegistry()
|
||||
for _, t := range tool.Builtins() {
|
||||
reg.Add(t)
|
||||
}
|
||||
boot.ApplyUnifiedProviderToolSurface(reg, false, arm)
|
||||
|
||||
cap := &captureProvider{}
|
||||
a := agent.New(cap, reg, sess, agent.Options{
|
||||
ContextWindow: fixtureWindow, CompactRatio: 0.5, RecentKeep: 2,
|
||||
SessionPath: f.Path, KeepPolicy: agent.KeepErrors, Ablation: arm,
|
||||
}, event.Discard)
|
||||
a.LoadProjectionSidecar(f.Path)
|
||||
|
||||
if err := a.Run(context.Background(), prompt); err != nil {
|
||||
return provider.Request{}, fmt.Errorf("run: %w", err)
|
||||
}
|
||||
// The turn's own request is the one carrying tools. A fold on the way in
|
||||
// asks the same provider for a digest, and that request has none — taking
|
||||
// the first request blindly would inspect the summarizer's prompt.
|
||||
for _, req := range cap.requests {
|
||||
if len(req.Tools) > 0 {
|
||||
return req, nil
|
||||
}
|
||||
}
|
||||
return provider.Request{}, fmt.Errorf("no request carried tools (%d captured)", len(cap.requests))
|
||||
}
|
||||
|
||||
// surfaceField is one place a model could read a string, named so a leak
|
||||
// report says where rather than only that.
|
||||
type surfaceField struct {
|
||||
Path string
|
||||
Text string
|
||||
}
|
||||
|
||||
// providerSurface enumerates every provider-visible string in a request.
|
||||
// Enumerated rather than marshalled whole: a Message also carries local-only
|
||||
// fields, and scanning those would fail on bytes no model ever sees.
|
||||
func providerSurface(req provider.Request) []surfaceField {
|
||||
var out []surfaceField
|
||||
add := func(path, text string) {
|
||||
if strings.TrimSpace(text) != "" {
|
||||
out = append(out, surfaceField{Path: path, Text: text})
|
||||
}
|
||||
}
|
||||
for i, m := range req.Messages {
|
||||
add(fmt.Sprintf("messages[%d].content", i), m.Content)
|
||||
add(fmt.Sprintf("messages[%d].reasoning_content", i), m.ReasoningContent)
|
||||
add(fmt.Sprintf("messages[%d].name", i), m.Name)
|
||||
for j, tc := range m.ToolCalls {
|
||||
add(fmt.Sprintf("messages[%d].tool_calls[%d].name", i, j), tc.Name)
|
||||
add(fmt.Sprintf("messages[%d].tool_calls[%d].arguments", i, j), tc.Arguments)
|
||||
}
|
||||
for j, item := range m.ResponsesItems {
|
||||
add(fmt.Sprintf("messages[%d].responses_items[%d]", i, j), string(item))
|
||||
}
|
||||
}
|
||||
for i, t := range req.Tools {
|
||||
add(fmt.Sprintf("tools[%d].name", i), t.Name)
|
||||
add(fmt.Sprintf("tools[%d].description", i), t.Description)
|
||||
if len(t.Parameters) > 0 {
|
||||
b, err := json.Marshal(t.Parameters)
|
||||
if err == nil {
|
||||
add(fmt.Sprintf("tools[%d].parameters", i), string(b))
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// findInSurface reports every place a marker is readable.
|
||||
func findInSurface(req provider.Request, marker string) []string {
|
||||
var hits []string
|
||||
for _, f := range providerSurface(req) {
|
||||
if strings.Contains(f.Text, marker) {
|
||||
hits = append(hits, f.Path)
|
||||
}
|
||||
}
|
||||
return hits
|
||||
}
|
||||
|
||||
// recallToolSchema returns the recall tool as the provider sees it.
|
||||
func recallToolSchema(req provider.Request) (provider.ToolSchema, bool) {
|
||||
for _, t := range req.Tools {
|
||||
if t.Name == "recall" {
|
||||
return t, true
|
||||
}
|
||||
}
|
||||
return provider.ToolSchema{}, false
|
||||
}
|
||||
|
||||
// localOnlyLeaks reports host-side fields that should never survive the
|
||||
// provider boundary. A hit here is a boundary regression, marker or no marker.
|
||||
func localOnlyLeaks(req provider.Request) []string {
|
||||
var out []string
|
||||
for i, m := range req.Messages {
|
||||
switch {
|
||||
case m.RawContent != "":
|
||||
out = append(out, fmt.Sprintf("messages[%d].raw_content", i))
|
||||
case m.LocalOnly:
|
||||
out = append(out, fmt.Sprintf("messages[%d].local_only", i))
|
||||
case m.DecisionReceipt != nil || len(m.DecisionReceipts) > 0:
|
||||
out = append(out, fmt.Sprintf("messages[%d].decision_receipt", i))
|
||||
case m.ToolExecution != nil:
|
||||
out = append(out, fmt.Sprintf("messages[%d].tool_execution", i))
|
||||
case len(m.MemoryCitations) > 0:
|
||||
out = append(out, fmt.Sprintf("messages[%d].memory_citations", i))
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// searchWording is what a search-off arm must never be told. The arm exists to
|
||||
// reproduce the runtime before search; an arm offered a capability it does not
|
||||
// have is a different experiment.
|
||||
var searchWording = []string{"recall with a query", "search the whole folded region", "query still finds"}
|
||||
@@ -1,648 +0,0 @@
|
||||
// contextbench.go — what the model did with the affordance, read off structure.
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"regexp"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"reasonix/internal/base/tokencount"
|
||||
"reasonix/internal/contract/provider"
|
||||
)
|
||||
|
||||
// Every judgement here reads a tool call's name and its JSON arguments, or a
|
||||
// canonical position, and never the model's prose. "I will search the history"
|
||||
// is not a search; a recall call with a query is.
|
||||
|
||||
// failure stages, in funnel order. Exactly one is reported per run.
|
||||
const (
|
||||
stageReadMissed = "NoSearch" // never asked, and never read the target either
|
||||
stageSearchMiss = "SearchMiss" // searched, target never came back
|
||||
stageHitNotRead = "HitNotRead" // target was returned and never opened
|
||||
stageAnswerWrong = "ReadButAnswerWrong" // opened it and still answered wrong
|
||||
stageRecovered = "Recovered"
|
||||
stageContaminated = "Contaminated"
|
||||
|
||||
// Routing is judged by model round, not by the linear order of tool events:
|
||||
// two calls in one round are one parallel fan-out, and calling that
|
||||
// "workspace first" would report a choice the model never made.
|
||||
routeMemoryFirst = "MemoryFirst"
|
||||
routeWorkspaceFirst = "WorkspaceFirst"
|
||||
routeParallel = "Parallel"
|
||||
routeNeither = "Neither"
|
||||
stageCueRead = "DirectRead" // read the target from an index address, no search needed
|
||||
stageNoRetrieval = "NoRetrieval" // answered without touching recall at all
|
||||
toolRecall = "recall"
|
||||
)
|
||||
|
||||
// searchAttempt is one query the model wrote, and what it got back. Kept per
|
||||
// attempt rather than aggregated: the question is why a retrieval that the host
|
||||
// probe ranks first takes the model more than one try.
|
||||
type searchAttempt struct {
|
||||
Ordinal int `json:"ordinal"`
|
||||
Round int `json:"round"`
|
||||
Query string `json:"query"`
|
||||
TargetRank int `json:"target_rank"` // 0 = the target did not come back
|
||||
ResultCount int `json:"result_count"`
|
||||
// Reformulation classifies this query against the one before it, and
|
||||
// MissingProbeTerms are the host probe's terms this query left out. Both
|
||||
// are derived from token sets: no judge, and nothing read out of prose.
|
||||
Reformulation string `json:"reformulation,omitempty"`
|
||||
MissingProbeTerms []string `json:"missing_probe_terms,omitempty"`
|
||||
}
|
||||
|
||||
// contextMetrics is one run's behaviour.
|
||||
type contextMetrics struct {
|
||||
Task string `json:"task"`
|
||||
Arm string `json:"arm"`
|
||||
|
||||
SearchCalls int `json:"search_calls"`
|
||||
// SearchHits counts searches that returned anything; TargetSearchHits
|
||||
// counts searches that returned the target. The gap is query quality.
|
||||
SearchHits int `json:"search_hits"`
|
||||
TargetSearchHits int `json:"target_search_hits"`
|
||||
FirstTargetRank int `json:"first_target_rank"` // 0 = never returned
|
||||
// FirstHitCoverage: how much of the answer the first hit handed over.
|
||||
// rank=1 with coverage 1/3 is a window too short to answer with, not a
|
||||
// model that would not stop.
|
||||
FirstHitCoverage int `json:"first_hit_coverage"`
|
||||
AnswerValues int `json:"answer_values"`
|
||||
|
||||
ReadCalls int `json:"read_calls"`
|
||||
TargetRead bool `json:"target_read"`
|
||||
ReadAfterHit bool `json:"read_after_hit"`
|
||||
// RecallReadWithoutSearch: the index addressed the target, so no search was
|
||||
// needed. Not "direct read" — read_file is a read too, and in an audit the
|
||||
// two mean opposite things.
|
||||
RecallReadWithoutSearch bool `json:"recall_read_without_search"`
|
||||
SearchThenRecallRead bool `json:"search_then_recall_read"`
|
||||
|
||||
FirstSearchRound int `json:"first_search_round"`
|
||||
FirstReadRound int `json:"first_read_round"`
|
||||
RetrievalRounds int `json:"retrieval_rounds"`
|
||||
|
||||
// RecallReturnedTokens is what recall paged back in. The index saves
|
||||
// resident prefix; this is what is paid dynamically instead.
|
||||
RecallReturnedTokens int `json:"recall_returned_tokens"`
|
||||
|
||||
Searches []searchAttempt `json:"searches,omitempty"`
|
||||
// PostHitSearches are queries issued after the target had already been
|
||||
// returned: not a query problem, a result-to-read problem.
|
||||
PostHitSearches int `json:"post_hit_searches"`
|
||||
|
||||
AnswerRecovered bool `json:"answer_recovered"`
|
||||
// AnswerAmbiguous marks an answer that offered more than one candidate for
|
||||
// a value. Markers matching proves the right fact appears; it does not
|
||||
// prove it was submitted as the answer. Recorded, never acted on.
|
||||
AnswerAmbiguous bool `json:"answer_ambiguous,omitempty"`
|
||||
AmbiguousVar string `json:"ambiguous_var,omitempty"`
|
||||
MissingMarkers []string `json:"missing_markers,omitempty"`
|
||||
// FinalAnswer is kept bounded so a surprising run can be read back. A
|
||||
// metric nobody can audit is a claim, not a measurement.
|
||||
FinalAnswer string `json:"final_answer,omitempty"`
|
||||
FailureStage string `json:"failure_stage"`
|
||||
TimeoutReason string `json:"timeout_reason,omitempty"`
|
||||
|
||||
// UnexpectedWorkTools are investigation tools used instead of recall.
|
||||
// Reaching for the workspace is behaviour, not noise: how a model routes
|
||||
// its uncertainty when memory is missing is part of what is measured.
|
||||
UnexpectedWorkTools []string `json:"unexpected_work_tools,omitempty"`
|
||||
EscapeCalls int `json:"escape_calls"`
|
||||
EscapeBeforeFirstRecall int `json:"escape_before_first_recall"`
|
||||
EscapeAfterFirstRecall int `json:"escape_after_first_recall"`
|
||||
// Contaminated: the answer was reachable through something other than
|
||||
// recall. Asserting this beats trusting the isolation, and a run that trips
|
||||
// it leaves the statistics rather than inflating one.
|
||||
Contaminated bool `json:"contaminated,omitempty"`
|
||||
LeakedVia string `json:"leaked_via,omitempty"`
|
||||
LeakedMarker string `json:"leaked_marker,omitempty"`
|
||||
LeakedArgs string `json:"leaked_args,omitempty"`
|
||||
LeakedOutput string `json:"leaked_output,omitempty"`
|
||||
// EscapeFoundAnswer separates "looked at the workspace" from "found the
|
||||
// answer there": different facts about the same call.
|
||||
EscapeFoundAnswer bool `json:"escape_found_answer,omitempty"`
|
||||
|
||||
CueVisible bool `json:"cue_visible"`
|
||||
|
||||
// trajectory is kept out of the metric line and written beside it, so a
|
||||
// future question can be asked of this batch instead of a new one.
|
||||
trajectory []provider.Message
|
||||
|
||||
// CueDirectRead: the cue was addressed, the first memory action was a
|
||||
// positions read, and it covered the target. Crediting any positions call
|
||||
// would hand the index a search's work.
|
||||
CueDirectRead bool `json:"cue_direct_read"`
|
||||
|
||||
// Sufficiency, not hit. A search that returns the target at rank 1 has not
|
||||
// necessarily returned the answer: the snippet is bounded. The stopping
|
||||
// boundary is the round where everything needed was readable.
|
||||
FirstSufficientRound int `json:"first_sufficient_round"`
|
||||
SufficientVia string `json:"sufficient_via"` // snippet | read | never
|
||||
RoundsAfterSufficient int `json:"rounds_after_sufficient"`
|
||||
SearchesAfterSufficient int `json:"searches_after_sufficient"`
|
||||
ReadsAfterSufficient int `json:"reads_after_sufficient"`
|
||||
EscapesAfterSufficient int `json:"escapes_after_sufficient"`
|
||||
RecallTokensAfterSufficient int `json:"recall_tokens_after_sufficient"`
|
||||
StoppingClass string `json:"stopping_class"`
|
||||
|
||||
// What happened after the answer was already in hand. A timeout that spent
|
||||
// seven rounds past its target is a stopping problem, and extending the
|
||||
// budget would only buy the loop more time.
|
||||
TargetFirstSeenRound int `json:"target_first_seen_round"`
|
||||
RoundsAfterTarget int `json:"rounds_after_target"`
|
||||
SearchesAfterTarget int `json:"searches_after_target"`
|
||||
RecallTokensAfterTarget int `json:"recall_tokens_after_target"`
|
||||
TargetHeldAtEnd bool `json:"target_held_at_end"`
|
||||
|
||||
// Routing is where the model looked first when its memory was missing.
|
||||
Routing string `json:"routing"`
|
||||
FirstRecallRound int `json:"first_recall_round"`
|
||||
FirstEscapeRound int `json:"first_escape_round"`
|
||||
}
|
||||
|
||||
// hitPosition matches the address recall prints for a search hit. Reading our
|
||||
// own rendered format, not guessing at the model's words.
|
||||
var hitPosition = regexp.MustCompile(`(?m)^#(\d+) `)
|
||||
|
||||
// scoreRun reads the messages one turn appended and reports what happened.
|
||||
// runScorer walks one turn's messages, holding the state a judgement needs:
|
||||
// which calls are open, whether the target has come back, and which search
|
||||
// each result belongs to.
|
||||
type runScorer struct {
|
||||
m contextMetrics
|
||||
inst fixtureInstance
|
||||
target int
|
||||
calls map[string]provider.ToolCall
|
||||
pending map[string]int
|
||||
round int
|
||||
sawTargetHit bool
|
||||
// seen accumulates everything recall has shown the model, so sufficiency is
|
||||
// judged on what it could read rather than on which call produced it.
|
||||
seen strings.Builder
|
||||
sufficient bool
|
||||
}
|
||||
|
||||
// scoreRun reads the messages one turn appended and reports what happened.
|
||||
func scoreRun(msgs []provider.Message, inst fixtureInstance, target int, arm string, cueVisible bool) contextMetrics {
|
||||
s := &runScorer{
|
||||
m: contextMetrics{Task: inst.Task.ID, Arm: arm, CueVisible: cueVisible},
|
||||
inst: inst, target: target,
|
||||
calls: map[string]provider.ToolCall{}, pending: map[string]int{},
|
||||
}
|
||||
for _, msg := range msgs {
|
||||
if len(msg.ToolCalls) > 0 {
|
||||
s.round++
|
||||
}
|
||||
for _, tc := range msg.ToolCalls {
|
||||
s.calls[tc.ID] = tc
|
||||
s.observeCall(tc)
|
||||
}
|
||||
if msg.Role == provider.RoleTool {
|
||||
s.observeResult(msg)
|
||||
}
|
||||
}
|
||||
return s.finish(cueVisible)
|
||||
}
|
||||
|
||||
// observeCall records one tool call: a recall search, a recall read, or a
|
||||
// reach for the workspace.
|
||||
func (s *runScorer) observeCall(tc provider.ToolCall) {
|
||||
m := &s.m
|
||||
if tc.Name != toolRecall {
|
||||
if !containsString(m.UnexpectedWorkTools, tc.Name) {
|
||||
m.UnexpectedWorkTools = append(m.UnexpectedWorkTools, tc.Name)
|
||||
}
|
||||
m.EscapeCalls++
|
||||
if s.sufficient {
|
||||
m.EscapesAfterSufficient++
|
||||
}
|
||||
if m.FirstEscapeRound == 0 {
|
||||
m.FirstEscapeRound = s.round
|
||||
}
|
||||
if m.SearchCalls+m.ReadCalls == 0 {
|
||||
m.EscapeBeforeFirstRecall++
|
||||
} else {
|
||||
m.EscapeAfterFirstRecall++
|
||||
}
|
||||
return
|
||||
}
|
||||
if m.FirstRecallRound == 0 {
|
||||
m.FirstRecallRound = s.round
|
||||
}
|
||||
query, positions := recallArgs(tc.Arguments)
|
||||
switch {
|
||||
case query != "":
|
||||
m.SearchCalls++
|
||||
if m.FirstSearchRound == 0 {
|
||||
m.FirstSearchRound = s.round
|
||||
}
|
||||
if s.sawTargetHit {
|
||||
m.PostHitSearches++
|
||||
m.SearchesAfterTarget++
|
||||
}
|
||||
if s.sufficient {
|
||||
m.SearchesAfterSufficient++
|
||||
}
|
||||
m.Searches = append(m.Searches, searchAttempt{Ordinal: m.SearchCalls, Round: s.round, Query: query})
|
||||
s.pending[tc.ID] = len(m.Searches) - 1
|
||||
case len(positions) > 0:
|
||||
m.ReadCalls++
|
||||
if s.sufficient {
|
||||
m.ReadsAfterSufficient++
|
||||
}
|
||||
if m.FirstReadRound == 0 {
|
||||
m.FirstReadRound = s.round
|
||||
}
|
||||
// A read covers the target when it names its position: recall returns
|
||||
// the tool results under the caller's own address.
|
||||
if containsInt(positions, s.target) {
|
||||
m.TargetRead = true
|
||||
switch {
|
||||
case s.sawTargetHit:
|
||||
m.ReadAfterHit, m.SearchThenRecallRead = true, true
|
||||
case m.SearchCalls == 0:
|
||||
m.RecallReadWithoutSearch = true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// observeResult reads what a call returned: contamination, page-in cost, and
|
||||
// whether this search brought the target back.
|
||||
func (s *runScorer) observeResult(msg provider.Message) {
|
||||
m := &s.m
|
||||
call, ok := s.calls[msg.ToolCallID]
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
if call.Name != toolRecall {
|
||||
// Codenames only: a three-digit epoch appears in any long file by
|
||||
// chance, and one did. A model holding the codename holds the numbers
|
||||
// beside it anyway.
|
||||
for _, marker := range s.inst.AnswerMarkers {
|
||||
if len(marker) >= 6 && strings.Contains(msg.Content, marker) {
|
||||
m.Contaminated, m.LeakedVia, m.LeakedMarker = true, call.Name, marker
|
||||
m.EscapeFoundAnswer = true
|
||||
m.LeakedArgs, m.LeakedOutput = truncate(call.Arguments, 200), truncate(msg.Content, 300)
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
cost := tokencount.Text(msg.Content)
|
||||
m.RecallReturnedTokens += cost
|
||||
if s.sawTargetHit {
|
||||
m.RecallTokensAfterTarget += cost
|
||||
}
|
||||
if s.sufficient {
|
||||
m.RecallTokensAfterSufficient += cost
|
||||
}
|
||||
// Everything recall showed the model, judged as a whole: the answer may
|
||||
// arrive across a snippet and a read, and either way it is readable.
|
||||
s.seen.WriteString(msg.Content)
|
||||
s.seen.WriteString("\n")
|
||||
if !s.sufficient && allMarkersIn(s.seen.String(), s.inst.AnswerMarkers) {
|
||||
s.sufficient = true
|
||||
m.FirstSufficientRound = s.round
|
||||
query, _ := recallArgs(call.Arguments)
|
||||
m.SufficientVia = "read"
|
||||
if query != "" {
|
||||
m.SufficientVia = "snippet"
|
||||
}
|
||||
}
|
||||
if query, _ := recallArgs(call.Arguments); query == "" {
|
||||
return
|
||||
}
|
||||
positions := searchHitPositions(msg.Content)
|
||||
if len(positions) > 0 {
|
||||
m.SearchHits++
|
||||
}
|
||||
rank := indexOfInt(positions, s.target)
|
||||
if i, ok := s.pending[msg.ToolCallID]; ok {
|
||||
m.Searches[i].TargetRank, m.Searches[i].ResultCount = rank, len(positions)
|
||||
}
|
||||
if rank == 0 {
|
||||
return
|
||||
}
|
||||
m.TargetSearchHits++
|
||||
if !s.sawTargetHit {
|
||||
m.TargetFirstSeenRound = s.round
|
||||
m.FirstHitCoverage = answerCoverage(msg.Content, s.inst.AnswerMarkers)
|
||||
}
|
||||
s.sawTargetHit = true
|
||||
if m.FirstTargetRank == 0 {
|
||||
m.FirstTargetRank = rank
|
||||
}
|
||||
}
|
||||
|
||||
// finish derives what only the whole walk can say.
|
||||
func (s *runScorer) finish(cueVisible bool) contextMetrics {
|
||||
m := s.m
|
||||
m.RetrievalRounds = m.SearchCalls + m.ReadCalls
|
||||
m.TargetHeldAtEnd = s.sawTargetHit || m.TargetRead
|
||||
if m.TargetFirstSeenRound > 0 {
|
||||
m.RoundsAfterTarget = s.round - m.TargetFirstSeenRound
|
||||
}
|
||||
m.Routing = routing(m.FirstRecallRound, m.FirstEscapeRound)
|
||||
m.CueDirectRead = cueVisible && m.RecallReadWithoutSearch
|
||||
if m.SufficientVia == "" {
|
||||
m.SufficientVia = "never"
|
||||
}
|
||||
if m.FirstSufficientRound > 0 {
|
||||
m.RoundsAfterSufficient = s.round - m.FirstSufficientRound
|
||||
}
|
||||
m.AnswerValues = len(s.inst.AnswerMarkers)
|
||||
m.StoppingClass = stoppingClass(m)
|
||||
classifySearches(&m, s.inst)
|
||||
return m
|
||||
}
|
||||
|
||||
// scoreAnswer grades the final text on the task's markers. Nonces, matched
|
||||
// exactly: no judge model, and nothing a general answer could satisfy.
|
||||
func (m *contextMetrics) scoreAnswer(final string, inst fixtureInstance) {
|
||||
lower := strings.ToLower(final)
|
||||
for _, marker := range inst.AnswerMarkers {
|
||||
if !strings.Contains(lower, strings.ToLower(marker)) {
|
||||
m.MissingMarkers = append(m.MissingMarkers, marker)
|
||||
}
|
||||
}
|
||||
m.AnswerRecovered = len(m.MissingMarkers) == 0
|
||||
m.AnswerAmbiguous, m.AmbiguousVar = ambiguousAnswer(final, inst)
|
||||
m.FailureStage = m.stage()
|
||||
if len(final) > 1200 {
|
||||
final = final[:1200] + "…"
|
||||
}
|
||||
m.FinalAnswer = final
|
||||
}
|
||||
|
||||
// stage places the run in the funnel. One label, chosen at the first step that
|
||||
// did not happen, so a table of stages reads as where the retrieval broke.
|
||||
func (m contextMetrics) stage() string {
|
||||
switch {
|
||||
case m.Contaminated:
|
||||
return stageContaminated
|
||||
case m.AnswerRecovered && m.RecallReadWithoutSearch:
|
||||
return stageCueRead
|
||||
case m.AnswerRecovered:
|
||||
return stageRecovered
|
||||
case m.TargetRead:
|
||||
return stageAnswerWrong
|
||||
case m.TargetSearchHits > 0:
|
||||
return stageHitNotRead
|
||||
case m.SearchCalls > 0:
|
||||
return stageSearchMiss
|
||||
case m.ReadCalls > 0:
|
||||
return stageReadMissed
|
||||
default:
|
||||
return stageNoRetrieval
|
||||
}
|
||||
}
|
||||
|
||||
// recallArgs pulls the two fields that say which operation a recall call is.
|
||||
func recallArgs(args string) (query string, positions []int) {
|
||||
var parsed struct {
|
||||
Query string `json:"query"`
|
||||
Positions []int `json:"positions"`
|
||||
}
|
||||
if err := json.Unmarshal([]byte(args), &parsed); err != nil {
|
||||
return "", nil
|
||||
}
|
||||
return strings.TrimSpace(parsed.Query), parsed.Positions
|
||||
}
|
||||
|
||||
// searchHitPositions reads the addresses out of a search result, in rank order.
|
||||
func searchHitPositions(text string) []int {
|
||||
var out []int
|
||||
for _, match := range hitPosition.FindAllStringSubmatch(text, -1) {
|
||||
if n, err := strconv.Atoi(match[1]); err == nil {
|
||||
out = append(out, n)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func containsInt(list []int, want int) bool { return slices.Contains(list, want) }
|
||||
|
||||
// indexOfInt returns a 1-based rank, or 0 when absent.
|
||||
func indexOfInt(list []int, want int) int {
|
||||
for i, v := range list {
|
||||
if v == want {
|
||||
return i + 1
|
||||
}
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func containsString(list []string, want string) bool { return slices.Contains(list, want) }
|
||||
|
||||
// funnel summarizes one arm's runs the way the experiment is read.
|
||||
type funnel struct {
|
||||
Arm string
|
||||
Runs int
|
||||
Scored, Contaminated, Errored int
|
||||
Searched, TargetHit, TargetRead, Answered int
|
||||
CueReads int
|
||||
SearchCalls, ReadCalls, RecallTokens int
|
||||
Escapes, EscapeCalls int
|
||||
Stages, Routes map[string]int
|
||||
}
|
||||
|
||||
func summarize(arm string, runs []contextMetrics) funnel {
|
||||
f := funnel{Arm: arm, Runs: len(runs), Stages: map[string]int{}, Routes: map[string]int{}}
|
||||
for _, r := range runs {
|
||||
if r.Contaminated {
|
||||
f.Contaminated++
|
||||
f.Stages[r.FailureStage]++
|
||||
continue
|
||||
}
|
||||
if r.FailureStage == "RunError" {
|
||||
f.Errored++
|
||||
f.Stages[r.FailureStage]++
|
||||
continue
|
||||
}
|
||||
f.Scored++
|
||||
if r.SearchCalls > 0 {
|
||||
f.Searched++
|
||||
}
|
||||
if r.TargetSearchHits > 0 {
|
||||
f.TargetHit++
|
||||
}
|
||||
if r.TargetRead {
|
||||
f.TargetRead++
|
||||
}
|
||||
if r.AnswerRecovered {
|
||||
f.Answered++
|
||||
}
|
||||
if r.RecallReadWithoutSearch {
|
||||
f.CueReads++
|
||||
}
|
||||
if len(r.UnexpectedWorkTools) > 0 {
|
||||
f.Escapes++
|
||||
}
|
||||
f.EscapeCalls += r.EscapeCalls
|
||||
f.SearchCalls += r.SearchCalls
|
||||
f.ReadCalls += r.ReadCalls
|
||||
f.RecallTokens += r.RecallReturnedTokens
|
||||
f.Stages[r.FailureStage]++
|
||||
if r.FailureStage != "RunError" {
|
||||
f.Routes[r.Routing]++
|
||||
}
|
||||
}
|
||||
return f
|
||||
}
|
||||
|
||||
func (f funnel) line() string {
|
||||
per := func(n int) float64 {
|
||||
if f.Scored == 0 {
|
||||
return 0
|
||||
}
|
||||
return float64(n) / float64(f.Scored)
|
||||
}
|
||||
line := fmt.Sprintf("%-14s runs=%-3d answered=%d/%d searched=%d target-hit=%d cue-read=%d search/task=%.2f recall-tok/task=%.0f escape/task=%.2f",
|
||||
f.Arm, f.Scored, f.Answered, f.Scored, f.Searched, f.TargetHit, f.CueReads,
|
||||
per(f.SearchCalls), per(f.RecallTokens), per(f.EscapeCalls))
|
||||
if f.Contaminated > 0 {
|
||||
line += fmt.Sprintf(" CONTAMINATED=%d", f.Contaminated)
|
||||
}
|
||||
if f.Errored > 0 {
|
||||
line += fmt.Sprintf(" ERRORED=%d", f.Errored)
|
||||
}
|
||||
return line
|
||||
}
|
||||
|
||||
func writeJSONLine(w io.Writer, v any) error {
|
||||
b, err := json.Marshal(v)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = fmt.Fprintln(w, string(b))
|
||||
return err
|
||||
}
|
||||
|
||||
func truncate(s string, n int) string {
|
||||
if len(s) <= n {
|
||||
return s
|
||||
}
|
||||
return s[:n] + "…"
|
||||
}
|
||||
|
||||
// routing says where the model looked first, comparing rounds rather than
|
||||
// event order. Same round is a parallel fan-out, not a preference.
|
||||
func routing(recallRound, escapeRound int) string {
|
||||
switch {
|
||||
case recallRound == 0 && escapeRound == 0:
|
||||
return routeNeither
|
||||
case recallRound == 0:
|
||||
return routeWorkspaceFirst
|
||||
case escapeRound == 0:
|
||||
return routeMemoryFirst
|
||||
case recallRound < escapeRound:
|
||||
return routeMemoryFirst
|
||||
case escapeRound < recallRound:
|
||||
return routeWorkspaceFirst
|
||||
default:
|
||||
return routeParallel
|
||||
}
|
||||
}
|
||||
|
||||
// ambiguousAnswer reports an answer naming more than one candidate of a
|
||||
// codename's shape. Only codenames are checked: they carry a stem this run
|
||||
// declared, so two distinct "fence-…" values in one answer is one answer too
|
||||
// many. Numbers have no such shape and are left alone.
|
||||
func ambiguousAnswer(final string, inst fixtureInstance) (bool, string) {
|
||||
for _, spec := range inst.Task.Vars {
|
||||
if spec.Kind != varCodename {
|
||||
continue
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
for _, match := range regexp.MustCompile(regexp.QuoteMeta(spec.Word)+`-[a-z0-9]{3,}`).FindAllString(final, -1) {
|
||||
seen[match] = true
|
||||
}
|
||||
if len(seen) > 1 {
|
||||
return true, spec.Name
|
||||
}
|
||||
}
|
||||
return false, ""
|
||||
}
|
||||
|
||||
func allMarkersIn(text string, markers []string) bool {
|
||||
if len(markers) == 0 {
|
||||
return false
|
||||
}
|
||||
lower := strings.ToLower(text)
|
||||
for _, marker := range markers {
|
||||
if !strings.Contains(lower, strings.ToLower(marker)) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// Stopping classes. The boundary is sufficiency, not the target coming back:
|
||||
// a rank-1 hit whose snippet was too short to answer leaves a read entirely
|
||||
// justified, and counting that as waste would blame the model for the bound.
|
||||
const (
|
||||
stopSnippet = "SnippetStop" // enough arrived, and the turn ended
|
||||
stopSnippetRead = "SnippetThenRead" // the snippet was short; the read finished it
|
||||
stopDefensiveRead = "DefensiveRead" // enough had arrived, one read confirmed it
|
||||
stopPostRetrieval = "PostSufficientRetrieval" // enough had arrived, and retrieval continued
|
||||
stopRunaway = "PostSufficientRunaway" // it continued for rounds, or ran out of budget
|
||||
stopNeverEnough = "NeverSufficient"
|
||||
)
|
||||
|
||||
// runawayRounds is where continuing stops looking like confirmation.
|
||||
const runawayRounds = 3
|
||||
|
||||
func stoppingClass(m contextMetrics) string {
|
||||
if m.FirstSufficientRound == 0 {
|
||||
return stopNeverEnough
|
||||
}
|
||||
after := m.SearchesAfterSufficient + m.ReadsAfterSufficient + m.EscapesAfterSufficient
|
||||
switch {
|
||||
case after == 0 && m.SufficientVia == "snippet":
|
||||
return stopSnippet
|
||||
case after == 0:
|
||||
return stopSnippetRead
|
||||
case m.RoundsAfterSufficient >= runawayRounds || m.FailureStage == "RunError":
|
||||
return stopRunaway
|
||||
case after == 1 && m.ReadsAfterSufficient == 1:
|
||||
return stopDefensiveRead
|
||||
default:
|
||||
return stopPostRetrieval
|
||||
}
|
||||
}
|
||||
|
||||
// reportStopping is the funnel that matters once retrieval succeeds: what the
|
||||
// turn did after it already had everything it needed.
|
||||
func reportStopping(all []contextMetrics) string {
|
||||
classes := map[string]int{}
|
||||
var extraRounds, extraTokens []float64
|
||||
sufficient, answeredNext := 0, 0
|
||||
for _, m := range all {
|
||||
if m.Contaminated {
|
||||
continue
|
||||
}
|
||||
classes[m.StoppingClass]++
|
||||
if m.FirstSufficientRound == 0 {
|
||||
continue
|
||||
}
|
||||
sufficient++
|
||||
extraRounds = append(extraRounds, float64(m.RoundsAfterSufficient))
|
||||
extraTokens = append(extraTokens, float64(m.RecallTokensAfterSufficient))
|
||||
if m.RoundsAfterSufficient <= 1 {
|
||||
answeredNext++
|
||||
}
|
||||
}
|
||||
var b strings.Builder
|
||||
b.WriteString("\n## After the evidence was already enough\n")
|
||||
fmt.Fprintf(&b, " sufficient in %d runs; ended within one round in %d\n", sufficient, answeredNext)
|
||||
fmt.Fprintf(&b, " extra rounds mean %.1f median %.1f p90 %.0f max %.0f\n",
|
||||
mean(extraRounds), median(extraRounds), percentile(extraRounds, 0.9), maxOf(extraRounds))
|
||||
fmt.Fprintf(&b, " extra recall tokens mean %.0f median %.0f p90 %.0f max %.0f\n",
|
||||
mean(extraTokens), median(extraTokens), percentile(extraTokens, 0.9), maxOf(extraTokens))
|
||||
b.WriteString(" " + countsLine(classes) + "\n")
|
||||
return b.String()
|
||||
}
|
||||
@@ -1,337 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"reasonix/internal/contract/provider"
|
||||
)
|
||||
|
||||
// The scorer decides what the pilot concludes, so it is held to the same rule
|
||||
// the corpus is: judgements read structure. A model that says it will search
|
||||
// has not searched; a recall call carrying a query has.
|
||||
|
||||
const scorerTarget = 42
|
||||
|
||||
func recallCall(id, args string) provider.Message {
|
||||
return provider.Message{Role: provider.RoleAssistant, ToolCalls: []provider.ToolCall{
|
||||
{ID: id, Name: "recall", Arguments: args},
|
||||
}}
|
||||
}
|
||||
|
||||
func toolResult(id, text string) provider.Message {
|
||||
return provider.Message{Role: provider.RoleTool, ToolCallID: id, Name: "recall", Content: text}
|
||||
}
|
||||
|
||||
func hitBlock(positions ...int) string {
|
||||
var b strings.Builder
|
||||
b.WriteString("Folded context matching \"x\":\n")
|
||||
for _, p := range positions {
|
||||
fmt.Fprintf(&b, "\n#%d assistant_text\nsome snippet here\n", p)
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
func scorerTask() fixtureInstance {
|
||||
return fixtureInstance{Task: contextTask{ID: "t"}, AnswerMarkers: []string{"cobalt-lark-17"}}
|
||||
}
|
||||
|
||||
func score(t *testing.T, msgs []provider.Message, final string) contextMetrics {
|
||||
t.Helper()
|
||||
m := scoreRun(msgs, scorerTask(), scorerTarget, "arm", false)
|
||||
m.scoreAnswer(final, scorerTask())
|
||||
return m
|
||||
}
|
||||
|
||||
// The whole funnel, walked end to end.
|
||||
func TestScorerReadsTheFullRetrievalChain(t *testing.T) {
|
||||
m := score(t, []provider.Message{
|
||||
recallCall("c1", `{"query":"retry boundary"}`),
|
||||
toolResult("c1", hitBlock(7, scorerTarget)),
|
||||
recallCall("c2", fmt.Sprintf(`{"positions":[%d]}`, scorerTarget)),
|
||||
toolResult("c2", "#42\n[assistant]\nthe ownership token is cobalt-lark-17"),
|
||||
{Role: provider.RoleAssistant, Content: "The token was cobalt-lark-17."},
|
||||
}, "The token was cobalt-lark-17.")
|
||||
|
||||
if m.SearchCalls != 1 || m.ReadCalls != 1 {
|
||||
t.Errorf("calls = %d search / %d read, want 1/1", m.SearchCalls, m.ReadCalls)
|
||||
}
|
||||
if m.TargetSearchHits != 1 || m.FirstTargetRank != 2 {
|
||||
t.Errorf("target hit %d at rank %d, want 1 at rank 2", m.TargetSearchHits, m.FirstTargetRank)
|
||||
}
|
||||
if !m.TargetRead || !m.ReadAfterHit || m.RecallReadWithoutSearch {
|
||||
t.Errorf("read flags = read %v afterHit %v direct %v", m.TargetRead, m.ReadAfterHit, m.RecallReadWithoutSearch)
|
||||
}
|
||||
if !m.AnswerRecovered || m.FailureStage != stageRecovered {
|
||||
t.Errorf("stage = %q recovered=%v, want Recovered", m.FailureStage, m.AnswerRecovered)
|
||||
}
|
||||
if m.RecallReturnedTokens == 0 {
|
||||
t.Error("recall returned tokens were not counted")
|
||||
}
|
||||
}
|
||||
|
||||
// An index arm's claim: the address was already on screen, so no search was
|
||||
// needed. That is a different outcome from finding it, and gets its own stage.
|
||||
func TestScorerSeparatesCueReadFromSearch(t *testing.T) {
|
||||
m := score(t, []provider.Message{
|
||||
recallCall("c1", fmt.Sprintf(`{"positions":[%d]}`, scorerTarget)),
|
||||
toolResult("c1", "#42\nthe ownership token is cobalt-lark-17"),
|
||||
{Role: provider.RoleAssistant, Content: "cobalt-lark-17"},
|
||||
}, "cobalt-lark-17")
|
||||
|
||||
if m.SearchCalls != 0 || !m.RecallReadWithoutSearch {
|
||||
t.Errorf("search=%d direct=%v, want 0 and true", m.SearchCalls, m.RecallReadWithoutSearch)
|
||||
}
|
||||
if m.FailureStage != stageCueRead {
|
||||
t.Errorf("stage = %q, want %q", m.FailureStage, stageCueRead)
|
||||
}
|
||||
}
|
||||
|
||||
// Every way the chain can break gets its own label, so a failed run says where
|
||||
// rather than only that.
|
||||
func TestScorerNamesWhereTheChainBroke(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
msgs []provider.Message
|
||||
final string
|
||||
want string
|
||||
}{
|
||||
{
|
||||
name: "never touched recall",
|
||||
msgs: []provider.Message{{Role: provider.RoleAssistant, Content: "I do not recall that."}},
|
||||
final: "I do not recall that.",
|
||||
want: stageNoRetrieval,
|
||||
},
|
||||
{
|
||||
name: "searched and missed",
|
||||
msgs: []provider.Message{
|
||||
recallCall("c1", `{"query":"unrelated words"}`),
|
||||
toolResult("c1", hitBlock(3, 9)),
|
||||
},
|
||||
final: "not found",
|
||||
want: stageSearchMiss,
|
||||
},
|
||||
{
|
||||
name: "found it and never opened it",
|
||||
msgs: []provider.Message{
|
||||
recallCall("c1", `{"query":"retry boundary"}`),
|
||||
toolResult("c1", hitBlock(scorerTarget)),
|
||||
},
|
||||
final: "probably something about retries",
|
||||
want: stageHitNotRead,
|
||||
},
|
||||
{
|
||||
name: "read it and answered wrong",
|
||||
msgs: []provider.Message{
|
||||
recallCall("c1", fmt.Sprintf(`{"positions":[%d]}`, scorerTarget)),
|
||||
toolResult("c1", "#42\nthe token is elsewhere"),
|
||||
},
|
||||
final: "I think it was amber-9",
|
||||
want: stageAnswerWrong,
|
||||
},
|
||||
{
|
||||
name: "read the wrong position",
|
||||
msgs: []provider.Message{
|
||||
recallCall("c1", `{"positions":[7]}`),
|
||||
toolResult("c1", "#7\nnothing useful"),
|
||||
},
|
||||
final: "unclear",
|
||||
want: stageReadMissed,
|
||||
},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
if got := score(t, tc.msgs, tc.final).FailureStage; got != tc.want {
|
||||
t.Errorf("stage = %q, want %q", got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// The answer lives only in folded history, so reaching for the workspace is a
|
||||
// strategy change worth seeing — including when it happens to work.
|
||||
func TestScorerRecordsWorkspaceEscapes(t *testing.T) {
|
||||
m := score(t, []provider.Message{
|
||||
{Role: provider.RoleAssistant, ToolCalls: []provider.ToolCall{
|
||||
{ID: "g1", Name: "bash", Arguments: `{"command":"rg cobalt-lark"}`},
|
||||
{ID: "g2", Name: "read_file", Arguments: `{"path":"notes.md"}`},
|
||||
}},
|
||||
{Role: provider.RoleAssistant, Content: "cobalt-lark-17"},
|
||||
}, "cobalt-lark-17")
|
||||
|
||||
if len(m.UnexpectedWorkTools) != 2 {
|
||||
t.Fatalf("escapes = %v, want bash and read_file", m.UnexpectedWorkTools)
|
||||
}
|
||||
if m.SearchCalls != 0 || m.ReadCalls != 0 {
|
||||
t.Errorf("a workspace tool was counted as recall: search=%d read=%d", m.SearchCalls, m.ReadCalls)
|
||||
}
|
||||
// It answered, so the run is Recovered — the escape is reported beside the
|
||||
// outcome rather than overriding it.
|
||||
if !m.AnswerRecovered {
|
||||
t.Error("a correct answer was not scored as recovered")
|
||||
}
|
||||
}
|
||||
|
||||
// Narration is not action.
|
||||
func TestScorerIgnoresWhatTheModelSaysItWillDo(t *testing.T) {
|
||||
m := score(t, []provider.Message{
|
||||
{Role: provider.RoleAssistant, Content: "Let me search the folded context with recall for the retry boundary."},
|
||||
}, "Let me search the folded context with recall for the retry boundary.")
|
||||
if m.SearchCalls != 0 || m.FailureStage != stageNoRetrieval {
|
||||
t.Errorf("prose was scored as a search: calls=%d stage=%q", m.SearchCalls, m.FailureStage)
|
||||
}
|
||||
}
|
||||
|
||||
// Every marker must land, so a partially right answer is not a recovery.
|
||||
func TestScorerRequiresEveryMarker(t *testing.T) {
|
||||
task := fixtureInstance{Task: contextTask{ID: "t"}, AnswerMarkers: []string{"comet-42", "per-lineage"}}
|
||||
m := scoreRun(nil, task, scorerTarget, "arm", false)
|
||||
m.scoreAnswer("the salt was comet-42", task)
|
||||
if m.AnswerRecovered {
|
||||
t.Error("a half answer scored as recovered")
|
||||
}
|
||||
if len(m.MissingMarkers) != 1 || m.MissingMarkers[0] != "per-lineage" {
|
||||
t.Errorf("missing = %v, want per-lineage", m.MissingMarkers)
|
||||
}
|
||||
}
|
||||
|
||||
// An answer reachable through anything but recall invalidates the run. The
|
||||
// workspace is isolated so this should never fire; asserting it beats trusting
|
||||
// it, and a tripped run leaves the statistics rather than inflating one.
|
||||
func TestScorerInvalidatesRunsThatLeakedThroughAnotherTool(t *testing.T) {
|
||||
m := score(t, []provider.Message{
|
||||
{Role: provider.RoleAssistant, ToolCalls: []provider.ToolCall{
|
||||
{ID: "g1", Name: "bash", Arguments: `{"command":"rg -r ."}`},
|
||||
}},
|
||||
{Role: provider.RoleTool, ToolCallID: "g1", Name: "bash", Content: "corpus.go: the token is cobalt-lark-17"},
|
||||
{Role: provider.RoleAssistant, Content: "cobalt-lark-17"},
|
||||
}, "cobalt-lark-17")
|
||||
|
||||
if !m.Contaminated || m.LeakedVia != "bash" || m.LeakedMarker != "cobalt-lark-17" {
|
||||
t.Fatalf("leak not caught: contaminated=%v via=%q marker=%q", m.Contaminated, m.LeakedVia, m.LeakedMarker)
|
||||
}
|
||||
if m.FailureStage != stageContaminated {
|
||||
t.Errorf("stage = %q, want %q even though every marker matched", m.FailureStage, stageContaminated)
|
||||
}
|
||||
f := summarize("arm", []contextMetrics{m})
|
||||
if f.Scored != 0 || f.Answered != 0 || f.Contaminated != 1 {
|
||||
t.Errorf("a contaminated run entered the statistics: %+v", f)
|
||||
}
|
||||
}
|
||||
|
||||
// Where an escape happened matters: before any recall is a different strategy
|
||||
// from after one came back short.
|
||||
func TestScorerPlacesEscapesRelativeToRecall(t *testing.T) {
|
||||
m := score(t, []provider.Message{
|
||||
{Role: provider.RoleAssistant, ToolCalls: []provider.ToolCall{
|
||||
{ID: "g1", Name: "bash", Arguments: `{"command":"ls"}`},
|
||||
}},
|
||||
{Role: provider.RoleTool, ToolCallID: "g1", Name: "bash", Content: "README.md"},
|
||||
recallCall("c1", `{"query":"retry boundary"}`),
|
||||
toolResult("c1", hitBlock(scorerTarget)),
|
||||
{Role: provider.RoleAssistant, ToolCalls: []provider.ToolCall{
|
||||
{ID: "g2", Name: "read_file", Arguments: `{"path":"README.md"}`},
|
||||
}},
|
||||
{Role: provider.RoleTool, ToolCallID: "g2", Name: "read_file", Content: "nothing here"},
|
||||
}, "unclear")
|
||||
|
||||
if m.EscapeCalls != 2 || m.EscapeBeforeFirstRecall != 1 || m.EscapeAfterFirstRecall != 1 {
|
||||
t.Errorf("escapes = %d (%d before, %d after), want 2 (1, 1)",
|
||||
m.EscapeCalls, m.EscapeBeforeFirstRecall, m.EscapeAfterFirstRecall)
|
||||
}
|
||||
}
|
||||
|
||||
// Routing is judged by round. Two calls in one round are a fan-out, and
|
||||
// reporting that as a preference would invent a decision the model never made.
|
||||
func TestRoutingIsJudgedByRoundNotEventOrder(t *testing.T) {
|
||||
parallel := []provider.Message{
|
||||
{Role: provider.RoleAssistant, ToolCalls: []provider.ToolCall{
|
||||
{ID: "g1", Name: "bash", Arguments: `{"command":"ls"}`},
|
||||
{ID: "c1", Name: "recall", Arguments: `{"query":"retry boundary"}`},
|
||||
}},
|
||||
{Role: provider.RoleTool, ToolCallID: "g1", Name: "bash", Content: "README.md"},
|
||||
toolResult("c1", hitBlock(scorerTarget)),
|
||||
}
|
||||
if got := score(t, parallel, "unclear").Routing; got != routeParallel {
|
||||
t.Errorf("routing = %q for one round holding both, want %q", got, routeParallel)
|
||||
}
|
||||
|
||||
workspaceFirst := []provider.Message{
|
||||
{Role: provider.RoleAssistant, ToolCalls: []provider.ToolCall{
|
||||
{ID: "g1", Name: "bash", Arguments: `{"command":"ls"}`},
|
||||
}},
|
||||
{Role: provider.RoleTool, ToolCallID: "g1", Name: "bash", Content: "README.md"},
|
||||
recallCall("c1", `{"query":"retry boundary"}`),
|
||||
toolResult("c1", hitBlock(scorerTarget)),
|
||||
}
|
||||
if got := score(t, workspaceFirst, "unclear").Routing; got != routeWorkspaceFirst {
|
||||
t.Errorf("routing = %q for workspace in an earlier round, want %q", got, routeWorkspaceFirst)
|
||||
}
|
||||
|
||||
memoryFirst := []provider.Message{
|
||||
recallCall("c1", `{"query":"retry boundary"}`),
|
||||
toolResult("c1", hitBlock(scorerTarget)),
|
||||
{Role: provider.RoleAssistant, ToolCalls: []provider.ToolCall{
|
||||
{ID: "g1", Name: "bash", Arguments: `{"command":"ls"}`},
|
||||
}},
|
||||
{Role: provider.RoleTool, ToolCallID: "g1", Name: "bash", Content: "README.md"},
|
||||
}
|
||||
if got := score(t, memoryFirst, "unclear").Routing; got != routeMemoryFirst {
|
||||
t.Errorf("routing = %q for recall in an earlier round, want %q", got, routeMemoryFirst)
|
||||
}
|
||||
|
||||
if got := score(t, nil, "no idea").Routing; got != routeNeither {
|
||||
t.Errorf("routing = %q with no tools at all, want %q", got, routeNeither)
|
||||
}
|
||||
}
|
||||
|
||||
// Markers matching proves the right fact appears somewhere. An answer that
|
||||
// hedges between two candidates contains it and has not submitted it.
|
||||
func TestScorerFlagsAnAnswerThatNamesTwoCandidates(t *testing.T) {
|
||||
inst := fixtureInstance{
|
||||
Task: contextTask{ID: "t", Vars: []varSpec{code("marker", "fence")}},
|
||||
AnswerMarkers: []string{"fence-qmwnv"},
|
||||
}
|
||||
m := scoreRun(nil, inst, 0, "arm", false)
|
||||
m.scoreAnswer("It was fence-qmwnv, though it may have been fence-abcde.", inst)
|
||||
if !m.AnswerRecovered {
|
||||
t.Error("the marker is present; recovery should still read true")
|
||||
}
|
||||
if !m.AnswerAmbiguous || m.AmbiguousVar != "marker" {
|
||||
t.Errorf("ambiguity not flagged: %v %q", m.AnswerAmbiguous, m.AmbiguousVar)
|
||||
}
|
||||
|
||||
clean := scoreRun(nil, inst, 0, "arm", false)
|
||||
clean.scoreAnswer("The marker was fence-qmwnv.", inst)
|
||||
if clean.AnswerAmbiguous {
|
||||
t.Error("a single candidate was called ambiguous")
|
||||
}
|
||||
}
|
||||
|
||||
// The index arm's claim is narrow: the cue was addressed, the first memory
|
||||
// action was a positions read, and it covered the target. A positions call
|
||||
// after a search located the target credits the index for the search's work.
|
||||
func TestCueDirectReadRequiresTheCueAndTheFirstAction(t *testing.T) {
|
||||
direct := []provider.Message{
|
||||
recallCall("c1", fmt.Sprintf(`{"positions":[%d]}`, scorerTarget)),
|
||||
toolResult("c1", "#42\nthe token is cobalt-lark-17"),
|
||||
}
|
||||
m := scoreRun(direct, scorerTask(), scorerTarget, "arm", true)
|
||||
if !m.CueDirectRead {
|
||||
t.Error("a first-action read with the cue visible was not credited")
|
||||
}
|
||||
|
||||
// Same read, but no cue was on screen to give the address.
|
||||
if scoreRun(direct, scorerTask(), scorerTarget, "arm", false).CueDirectRead {
|
||||
t.Error("a read was credited to a cue that was not visible")
|
||||
}
|
||||
|
||||
// Same read, but a search found the address first.
|
||||
afterSearch := append([]provider.Message{
|
||||
recallCall("c0", `{"query":"retry boundary"}`),
|
||||
toolResult("c0", hitBlock(scorerTarget)),
|
||||
}, direct...)
|
||||
if scoreRun(afterSearch, scorerTask(), scorerTarget, "arm", true).CueDirectRead {
|
||||
t.Error("a read after a search was credited to the index")
|
||||
}
|
||||
}
|
||||
@@ -1,344 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"math/rand"
|
||||
"reasonix/internal/state/sessionstore"
|
||||
|
||||
"reasonix/internal/contract/provider"
|
||||
)
|
||||
|
||||
// Synthetic history on purpose: an answer that exists in the workspace is
|
||||
// solvable by reading the workspace, which measures grep. Every answer is a
|
||||
// placeholder here and a value only inside one run.
|
||||
|
||||
// experiment names which question a task belongs to.
|
||||
const (
|
||||
experimentSearch = "search" // does search reach what no index addresses?
|
||||
experimentIndex = "index" // what is an ambient address line worth?
|
||||
)
|
||||
|
||||
// cueTier is the narrowest fold-index scale at which an index task's cue is
|
||||
// still addressed. Below it the model has to find the call without a hint.
|
||||
const (
|
||||
tierQuarter = "quarter" // visible at default, half, quarter
|
||||
tierHalf = "half" // visible at default, half
|
||||
tierDefault = "default" // visible at default only
|
||||
)
|
||||
|
||||
// plantKind is how a task's target enters the transcript.
|
||||
const (
|
||||
plantAssistant = "assistant" // the model's own account: never index-addressed
|
||||
plantCall = "call" // a tool call and its result: always index-addressed
|
||||
)
|
||||
|
||||
// contextTask is one recall question as a template. Nothing here is an answer.
|
||||
type contextTask struct {
|
||||
ID string
|
||||
Experiment string
|
||||
TargetKind string
|
||||
Prompt string
|
||||
ProbeQuery string
|
||||
// Vars declare the placeholders. AnswerVars name the ones a correct answer
|
||||
// must contain, CueVar the one an index line carries.
|
||||
Vars []varSpec
|
||||
AnswerVars []string
|
||||
CueVar string
|
||||
CueTier string
|
||||
// PlantAfterGen ages the target: mergeFoldIndex trims from its oldest end,
|
||||
// so what follows a cue decides which budget addresses it. Frozen, because
|
||||
// a run that searched for it would let the tested code move its goalposts.
|
||||
PlantAfterGen int
|
||||
// Plant is how the target enters, and the rest is what it says.
|
||||
Plant string
|
||||
CallTool string
|
||||
CallArgs string
|
||||
CallID string
|
||||
PlantBody string
|
||||
}
|
||||
|
||||
// fixtureInstance is one task with values. Scoring binds to this, never to the
|
||||
// corpus: the corpus has no answers to bind to.
|
||||
type fixtureInstance struct {
|
||||
Task contextTask
|
||||
Vars fixtureVars
|
||||
Prompt string
|
||||
ProbeQuery string
|
||||
AnswerMarkers []string
|
||||
CueMarker string
|
||||
body string
|
||||
callArgs string
|
||||
}
|
||||
|
||||
// instantiateTask resolves one task against a generator.
|
||||
func instantiateTask(t contextTask, rng *rand.Rand) (fixtureInstance, error) {
|
||||
vars := newVars(t.Vars, rng)
|
||||
inst := fixtureInstance{Task: t, Vars: vars}
|
||||
var err error
|
||||
for _, pair := range []struct {
|
||||
dst *string
|
||||
text string
|
||||
}{
|
||||
{&inst.Prompt, t.Prompt},
|
||||
{&inst.ProbeQuery, t.ProbeQuery},
|
||||
{&inst.body, t.PlantBody},
|
||||
{&inst.callArgs, t.CallArgs},
|
||||
} {
|
||||
if *pair.dst, err = instantiate(pair.text, vars); err != nil {
|
||||
return fixtureInstance{}, fmt.Errorf("%s: %w", t.ID, err)
|
||||
}
|
||||
}
|
||||
for _, name := range t.AnswerVars {
|
||||
value, ok := vars[name]
|
||||
if !ok {
|
||||
return fixtureInstance{}, fmt.Errorf("%s: answer var %q is not declared", t.ID, name)
|
||||
}
|
||||
inst.AnswerMarkers = append(inst.AnswerMarkers, value)
|
||||
}
|
||||
if t.CueVar != "" {
|
||||
inst.CueMarker = vars[t.CueVar]
|
||||
}
|
||||
return inst, nil
|
||||
}
|
||||
|
||||
// plant appends the instantiated target and returns its canonical position.
|
||||
func (f fixtureInstance) plant(s *sessionstore.Session) int {
|
||||
at := len(s.Messages)
|
||||
if f.Task.Plant == plantAssistant {
|
||||
s.Add(provider.Message{Role: provider.RoleAssistant, Content: f.body})
|
||||
return at
|
||||
}
|
||||
s.Add(provider.Message{Role: provider.RoleAssistant, ToolCalls: []provider.ToolCall{
|
||||
{ID: f.Task.CallID, Name: f.Task.CallTool, Arguments: f.callArgs},
|
||||
}})
|
||||
s.Add(provider.Message{Role: provider.RoleTool, ToolCallID: f.Task.CallID, Name: f.Task.CallTool, Content: f.body})
|
||||
return at
|
||||
}
|
||||
|
||||
func code(name, word string) varSpec { return varSpec{Name: name, Kind: varCodename, Word: word} }
|
||||
|
||||
func num(name string, lo, hi int) varSpec {
|
||||
return varSpec{Name: name, Kind: varInt, Min: lo, Max: hi}
|
||||
}
|
||||
|
||||
func searchTasks() []contextTask {
|
||||
return []contextTask{
|
||||
{
|
||||
ID: "r01-retry-boundary", Experiment: experimentSearch, TargetKind: "assistant_reasoning",
|
||||
Prompt: "What ownership token did we settle on for the retry boundary, and under what tag did we file the reason for not moving retries above framing?",
|
||||
ProbeQuery: "retry boundary ownership token framing",
|
||||
Vars: []varSpec{code("token", "cobalt"), code("reason", "stream")},
|
||||
AnswerVars: []string{"token", "reason"},
|
||||
Plant: plantAssistant,
|
||||
PlantBody: "The retry boundary remains at ingress. The ownership token is {{token}}. " +
|
||||
"Moving retries above framing can duplicate a partially received stream, which we filed as {{reason}}.",
|
||||
},
|
||||
{
|
||||
ID: "r02-checkpoint-anchor", Experiment: experimentSearch, TargetKind: "assistant_reasoning",
|
||||
Prompt: "What was the codename of the checkpoint anchor policy, and what tag did we give the reason for not anchoring against digest positions?",
|
||||
ProbeQuery: "checkpoint anchor policy digest positions",
|
||||
Vars: []varSpec{code("policy", "northglass"), code("reason", "genlocal")},
|
||||
AnswerVars: []string{"policy", "reason"},
|
||||
Plant: plantAssistant,
|
||||
PlantBody: "The checkpoint anchor policy is {{policy}}. Digest positions are generation-local — " +
|
||||
"filed as {{reason}} — so recovery must anchor against canonical state.",
|
||||
},
|
||||
{
|
||||
ID: "r03-recovery-cjk", Experiment: experimentSearch, TargetKind: "assistant_reasoning_cjk",
|
||||
Prompt: "我们之前定的验证失败恢复策略代号叫什么?当时把“失败 digest 不能变成 baseline”这条登记在哪个标记下?",
|
||||
ProbeQuery: "恢复策略 baseline digest 验证失败",
|
||||
Vars: []varSpec{code("policy", "songzhen"), code("rule", "trustbase")},
|
||||
AnswerVars: []string{"policy", "rule"},
|
||||
Plant: plantAssistant,
|
||||
PlantBody: "恢复策略代号是 {{policy}}。验证失败以后必须保留旧基线,不能把新的 digest 晋升为 baseline," +
|
||||
"因为失败证据不能成为后续比较的可信基准;这条规则登记为 {{rule}}。",
|
||||
},
|
||||
{
|
||||
ID: "r04-lease-fence", Experiment: experimentSearch, TargetKind: "assistant_reasoning",
|
||||
Prompt: "那次 lease fence 检查报告的诊断标记是什么?expected 和 observed epoch 分别是多少?",
|
||||
ProbeQuery: "lease fence writer epoch mismatch",
|
||||
Vars: []varSpec{
|
||||
code("marker", "fence"), num("expected", 400, 900), num("observed", 100, 399),
|
||||
},
|
||||
AnswerVars: []string{"marker", "expected", "observed"},
|
||||
Plant: plantAssistant,
|
||||
PlantBody: "The lease fence check reported {{marker}}: expected writer epoch {{expected}}, " +
|
||||
"observed {{observed}}. We left the fence in place.",
|
||||
},
|
||||
{
|
||||
ID: "r05-schema-guard", Experiment: experimentSearch, TargetKind: "assistant_reasoning",
|
||||
Prompt: "当时 schema guard 报告的标记是什么,超预算多少 token?",
|
||||
ProbeQuery: "schema guard provider visible surface budget",
|
||||
Vars: []varSpec{code("marker", "amber"), num("over", 120, 480)},
|
||||
AnswerVars: []string{"marker", "over"},
|
||||
Plant: plantAssistant,
|
||||
PlantBody: "The schema guard reported {{marker}}: the provider-visible tool schema exceeded " +
|
||||
"the allowed surface by {{over}} tokens. We trimmed a description instead.",
|
||||
},
|
||||
{
|
||||
ID: "r06-shadow-probe", Experiment: experimentSearch, TargetKind: "assistant_reasoning",
|
||||
Prompt: "我们那次 shadow probe 的 marker、p95 divergence 和 generation 分别是多少?",
|
||||
ProbeQuery: "shadow probe divergence generation marker",
|
||||
Vars: []varSpec{
|
||||
code("marker", "violet"),
|
||||
{Name: "divergence", Kind: varDecimal, Min: 110, Max: 890},
|
||||
num("generation", 14, 60),
|
||||
},
|
||||
AnswerVars: []string{"marker", "divergence", "generation"},
|
||||
Plant: plantAssistant,
|
||||
PlantBody: "The shadow probe came back as marker={{marker}} with a p95 divergence of " +
|
||||
"{{divergence}} at generation={{generation}}, which we judged within tolerance.",
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func indexTasks() []contextTask {
|
||||
return []contextTask{
|
||||
{
|
||||
ID: "i01-transport-fallback", Experiment: experimentIndex, TargetKind: "tool_input",
|
||||
Prompt: "transport fallback probe 当时报告的 selector 是什么?",
|
||||
ProbeQuery: "transport fallback probe selector",
|
||||
Vars: []varSpec{code("cue", "tfprobe"), code("selector", "fern")},
|
||||
AnswerVars: []string{"selector"}, CueVar: "cue", CueTier: tierQuarter,
|
||||
PlantAfterGen: 4,
|
||||
Plant: plantCall, CallID: "tf1", CallTool: "bash",
|
||||
CallArgs: `{"command":"./scripts/probe {{cue}}"}`,
|
||||
PlantBody: "fallback selector = {{selector}}",
|
||||
},
|
||||
{
|
||||
ID: "i02-scheduler-handoff", Experiment: experimentIndex, TargetKind: "tool_input",
|
||||
Prompt: "scheduler handoff 当时记录的 grace 和 marker 是什么?",
|
||||
ProbeQuery: "scheduler handoff grace marker",
|
||||
Vars: []varSpec{
|
||||
code("cue", "schedhandoff"), code("marker", "opal"),
|
||||
{Name: "grace", Kind: varDuration, Min: 18, Max: 90},
|
||||
},
|
||||
AnswerVars: []string{"grace", "marker"}, CueVar: "cue", CueTier: tierQuarter,
|
||||
PlantAfterGen: 4,
|
||||
Plant: plantCall, CallID: "sh1", CallTool: "read_file",
|
||||
CallArgs: `{"path":"scratch/{{cue}}.txt"}`,
|
||||
PlantBody: "handoff grace = {{grace}}\nmarker = {{marker}}",
|
||||
},
|
||||
{
|
||||
ID: "i03-coalescing", Experiment: experimentIndex, TargetKind: "tool_input",
|
||||
Prompt: "job completion 的 coalescing probe 最后记录的 window 和 marker 是多少?",
|
||||
ProbeQuery: "coalescing policy completion window",
|
||||
Vars: []varSpec{
|
||||
code("cue", "coalescing"), code("marker", "cedar"),
|
||||
{Name: "window", Kind: varDuration, Min: 40, Max: 140},
|
||||
},
|
||||
AnswerVars: []string{"window", "marker"}, CueVar: "cue", CueTier: tierHalf,
|
||||
PlantAfterGen: 2,
|
||||
Plant: plantCall, CallID: "cp1", CallTool: "bash",
|
||||
CallArgs: `{"command":"grep {{cue}} scratch/job-notes.log"}`,
|
||||
PlantBody: "completion coalesce window = {{window}}\nmarker = {{marker}}",
|
||||
},
|
||||
{
|
||||
ID: "i04-recovery-fence", Experiment: experimentIndex, TargetKind: "tool_input",
|
||||
Prompt: "recovery fence 那次检查报告的 marker 和两个 epoch 是什么?",
|
||||
ProbeQuery: "recovery fence epoch check",
|
||||
Vars: []varSpec{
|
||||
code("cue", "recfence"), code("marker", "quartz"),
|
||||
num("expected", 400, 900), num("observed", 100, 399),
|
||||
},
|
||||
AnswerVars: []string{"marker", "expected", "observed"}, CueVar: "cue", CueTier: tierHalf,
|
||||
PlantAfterGen: 2,
|
||||
Plant: plantCall, CallID: "rf1", CallTool: "bash",
|
||||
CallArgs: `{"command":"./scripts/check {{cue}}"}`,
|
||||
PlantBody: "{{marker}}\nexpected epoch {{expected}}\nobserved epoch {{observed}}",
|
||||
},
|
||||
{
|
||||
ID: "i05-cache-probe", Experiment: experimentIndex, TargetKind: "tool_input",
|
||||
Prompt: "cache probe 当时记录的 prefix salt 和 scope 是什么?",
|
||||
ProbeQuery: "cache probe prefix salt scope",
|
||||
Vars: []varSpec{code("cue", "cacheprobe"), code("salt", "comet"), code("scope", "lineage")},
|
||||
AnswerVars: []string{"salt", "scope"}, CueVar: "cue", CueTier: tierDefault,
|
||||
PlantAfterGen: 0,
|
||||
Plant: plantCall, CallID: "cb1", CallTool: "read_file",
|
||||
CallArgs: `{"path":"scratch/{{cue}}.txt"}`,
|
||||
PlantBody: "prefix salt = {{salt}}\nscope = {{scope}}",
|
||||
},
|
||||
{
|
||||
ID: "i06-dispatch-handoff", Experiment: experimentIndex, TargetKind: "tool_input",
|
||||
Prompt: "dispatch handoff probe 当时确认的 mode 和 marker 是什么?",
|
||||
ProbeQuery: "dispatch handoff probe mode",
|
||||
Vars: []varSpec{code("cue", "dispatchhandoff"), code("mode", "trigger"), code("marker", "iris")},
|
||||
AnswerVars: []string{"mode", "marker"}, CueVar: "cue", CueTier: tierDefault,
|
||||
PlantAfterGen: 0,
|
||||
Plant: plantCall, CallID: "dh1", CallTool: "bash",
|
||||
CallArgs: `{"command":"./scripts/probe {{cue}}"}`,
|
||||
PlantBody: "handoff mode = {{mode}}\nmarker = {{marker}}",
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func allTasks() []contextTask {
|
||||
return append(searchTasks(), indexTasks()...)
|
||||
}
|
||||
|
||||
func taskByID(id string) (contextTask, bool) {
|
||||
for _, t := range allTasks() {
|
||||
if t.ID == id {
|
||||
return t, true
|
||||
}
|
||||
}
|
||||
return contextTask{}, false
|
||||
}
|
||||
|
||||
// tierScales is the fixture's contract: which arms must address a cue and
|
||||
// which must not. A policy change that moves a cue across a boundary fails
|
||||
// preflight rather than turning two arms into one experiment.
|
||||
func tierScales(tier string) (visible, hidden []string) {
|
||||
switch tier {
|
||||
case tierQuarter:
|
||||
return []string{"default", "half", "quarter"}, []string{"off"}
|
||||
case tierHalf:
|
||||
return []string{"default", "half"}, []string{"quarter", "off"}
|
||||
case tierDefault:
|
||||
return []string{"default"}, []string{"half", "quarter", "off"}
|
||||
default:
|
||||
return nil, nil
|
||||
}
|
||||
}
|
||||
|
||||
// boundaryPair is the one comparison an index task is built for: the narrowest
|
||||
// scale that still addresses its cue, and the next one down. Holding the
|
||||
// question fixed and moving only the affordance is what four arm averages
|
||||
// cannot do, since each of those mixes six different questions.
|
||||
func boundaryPair(tier string) (cue, noCue string) {
|
||||
switch tier {
|
||||
case tierQuarter:
|
||||
return "quarter", "off"
|
||||
case tierHalf:
|
||||
return "half", "quarter"
|
||||
case tierDefault:
|
||||
return "default", "half"
|
||||
default:
|
||||
return "", ""
|
||||
}
|
||||
}
|
||||
|
||||
func validateCorpus() error {
|
||||
seen := map[string]bool{}
|
||||
for _, t := range allTasks() {
|
||||
if seen[t.ID] {
|
||||
return fmt.Errorf("duplicate task id %q", t.ID)
|
||||
}
|
||||
seen[t.ID] = true
|
||||
if len(t.AnswerVars) == 0 || t.ProbeQuery == "" || t.Prompt == "" || t.PlantBody == "" {
|
||||
return fmt.Errorf("%s: a task needs a prompt, a probe query, a body and answer vars", t.ID)
|
||||
}
|
||||
if t.Experiment == experimentIndex {
|
||||
if t.CueVar == "" {
|
||||
return fmt.Errorf("%s: an index task needs a cue var", t.ID)
|
||||
}
|
||||
if v, _ := tierScales(t.CueTier); v == nil {
|
||||
return fmt.Errorf("%s: unknown cue tier %q", t.ID, t.CueTier)
|
||||
}
|
||||
}
|
||||
if _, err := instantiateTask(t, seededRand(t.ID, "validate")); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -1,243 +0,0 @@
|
||||
// dose.go — does the cost step where the cue disappears?
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"math"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Not "a bigger index is better" — CueDirectRead says the addresses are rarely
|
||||
// used. The hypothesis is that a visible action cue lowers retrieval cost,
|
||||
// which predicts a step at each task's own boundary, not a slope.
|
||||
|
||||
// scaleOrder is the dose axis, widest first.
|
||||
var scaleOrder = []string{"default", "half", "quarter", "off"}
|
||||
|
||||
// boundaryStep is the index into the three transitions where a tier's cue
|
||||
// disappears: default->half is 0, half->quarter is 1, quarter->off is 2.
|
||||
func boundaryStep(tier string) int {
|
||||
switch tier {
|
||||
case tierDefault:
|
||||
return 0
|
||||
case tierHalf:
|
||||
return 1
|
||||
case tierQuarter:
|
||||
return 2
|
||||
default:
|
||||
return -1
|
||||
}
|
||||
}
|
||||
|
||||
// doseCurve is one task's readings across the four budgets.
|
||||
type doseCurve struct {
|
||||
Task string
|
||||
Tier string
|
||||
Metric string
|
||||
Values [4]float64
|
||||
// Missing marks arms that produced nothing. Printing those as zero draws a
|
||||
// curve falling to the floor where in fact it was never measured.
|
||||
Missing [4]bool
|
||||
Present bool
|
||||
// PreMean averages the arms where the cue is visible, PostMean the rest.
|
||||
PreMean, PostMean float64
|
||||
BoundaryDelta float64
|
||||
// BoundaryAligned: the largest transition is the one where this task's cue
|
||||
// disappears. A change elsewhere is variance shaped like an effect.
|
||||
BoundaryAligned bool
|
||||
}
|
||||
|
||||
func buildCurve(task, tier, metric string, byScale map[string]float64) doseCurve {
|
||||
c := doseCurve{Task: task, Tier: tier, Metric: metric, Present: true}
|
||||
for i, scale := range scaleOrder {
|
||||
v, ok := byScale[scale]
|
||||
if !ok {
|
||||
c.Present, c.Missing[i] = false, true
|
||||
}
|
||||
c.Values[i] = v
|
||||
}
|
||||
step := boundaryStep(tier)
|
||||
if !c.Present || step < 0 {
|
||||
return c
|
||||
}
|
||||
visible := step + 1 // arms before the boundary still show the cue
|
||||
c.PreMean = mean(c.Values[:visible])
|
||||
c.PostMean = mean(c.Values[visible:])
|
||||
c.BoundaryDelta = c.PostMean - c.PreMean
|
||||
|
||||
transitions := [3]float64{
|
||||
math.Abs(c.Values[1] - c.Values[0]),
|
||||
math.Abs(c.Values[2] - c.Values[1]),
|
||||
math.Abs(c.Values[3] - c.Values[2]),
|
||||
}
|
||||
largest, at := 0.0, -1
|
||||
for i, t := range transitions {
|
||||
if t > largest {
|
||||
largest, at = t, i
|
||||
}
|
||||
}
|
||||
// A flat curve has no largest transition to align with anything.
|
||||
c.BoundaryAligned = largest > 0 && at == step
|
||||
return c
|
||||
}
|
||||
|
||||
func mean(xs []float64) float64 {
|
||||
if len(xs) == 0 {
|
||||
return 0
|
||||
}
|
||||
sum := 0.0
|
||||
for _, x := range xs {
|
||||
sum += x
|
||||
}
|
||||
return sum / float64(len(xs))
|
||||
}
|
||||
|
||||
func median(xs []float64) float64 {
|
||||
if len(xs) == 0 {
|
||||
return 0
|
||||
}
|
||||
s := append([]float64(nil), xs...)
|
||||
sort.Float64s(s)
|
||||
mid := len(s) / 2
|
||||
if len(s)%2 == 1 {
|
||||
return s[mid]
|
||||
}
|
||||
return (s[mid-1] + s[mid]) / 2
|
||||
}
|
||||
|
||||
// doseMetrics are the readings a curve is drawn for.
|
||||
var doseMetrics = []struct {
|
||||
name string
|
||||
of func(contextMetrics) float64
|
||||
}{
|
||||
{"searches", func(m contextMetrics) float64 { return float64(m.SearchCalls) }},
|
||||
{"recall-tokens", func(m contextMetrics) float64 { return float64(m.RecallReturnedTokens) }},
|
||||
{"tool-rounds", func(m contextMetrics) float64 { return float64(m.RetrievalRounds + m.EscapeCalls) }},
|
||||
{"escapes", func(m contextMetrics) float64 { return float64(m.EscapeCalls) }},
|
||||
}
|
||||
|
||||
// reportDoseResponse draws each task's curve and reports where the steps fell.
|
||||
func reportDoseResponse(all []contextMetrics) string {
|
||||
valid := map[string]map[string]contextMetrics{}
|
||||
for _, m := range all {
|
||||
if m.Contaminated || m.FailureStage == "RunError" {
|
||||
continue
|
||||
}
|
||||
scale := strings.TrimPrefix(m.Arm, "index-")
|
||||
if valid[m.Task] == nil {
|
||||
valid[m.Task] = map[string]contextMetrics{}
|
||||
}
|
||||
valid[m.Task][scale] = m
|
||||
}
|
||||
|
||||
var b strings.Builder
|
||||
b.WriteString("\n## Dose-response, per task\n")
|
||||
b.WriteString(" The arrow marks where this task's cue stops being addressed.\n")
|
||||
aligned := map[string]int{}
|
||||
counted := map[string]int{}
|
||||
deltas := map[string][]float64{}
|
||||
|
||||
for _, t := range indexTasks() {
|
||||
arms := valid[t.ID]
|
||||
if len(arms) == 0 {
|
||||
continue
|
||||
}
|
||||
step := boundaryStep(t.CueTier)
|
||||
fmt.Fprintf(&b, "\n %s (cue survives to %s)\n", t.ID, t.CueTier)
|
||||
fmt.Fprintf(&b, " %-14s%s\n", "", scaleHeader(step))
|
||||
for _, metric := range doseMetrics {
|
||||
byScale := map[string]float64{}
|
||||
for scale, m := range arms {
|
||||
byScale[scale] = metric.of(m)
|
||||
}
|
||||
c := buildCurve(t.ID, t.CueTier, metric.name, byScale)
|
||||
mark := " "
|
||||
if c.BoundaryAligned {
|
||||
mark = "*"
|
||||
aligned[metric.name]++
|
||||
}
|
||||
if c.Present {
|
||||
counted[metric.name]++
|
||||
deltas[metric.name] = append(deltas[metric.name], c.BoundaryDelta)
|
||||
}
|
||||
delta := fmt.Sprintf("Δ%+8.1f", c.BoundaryDelta)
|
||||
if !c.Present {
|
||||
delta = " no curve"
|
||||
}
|
||||
fmt.Fprintf(&b, " %-14s%s %s %s\n", metric.name, curveCells(c), delta, mark)
|
||||
}
|
||||
}
|
||||
|
||||
b.WriteString("\n## Where the step fell (* = largest transition is at the cue boundary)\n")
|
||||
for _, metric := range doseMetrics {
|
||||
d := deltas[metric.name]
|
||||
sensitivity := ""
|
||||
if len(d) > 1 {
|
||||
sensitivity = fmt.Sprintf(" without the largest: %+.1f", meanWithoutLargest(d))
|
||||
}
|
||||
fmt.Fprintf(&b, " %-14s boundary-aligned %d/%d mean Δ%+.1f median Δ%+.1f%s\n",
|
||||
metric.name, aligned[metric.name], counted[metric.name], mean(d), median(d), sensitivity)
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// meanWithoutLargest is the robustness check: the largest effect is reported
|
||||
// and then set aside, so a conclusion never rests on one task alone.
|
||||
func meanWithoutLargest(d []float64) float64 {
|
||||
if len(d) < 2 {
|
||||
return mean(d)
|
||||
}
|
||||
worst, at := 0.0, 0
|
||||
for i, v := range d {
|
||||
if math.Abs(v) > worst {
|
||||
worst, at = math.Abs(v), i
|
||||
}
|
||||
}
|
||||
rest := append(append([]float64(nil), d[:at]...), d[at+1:]...)
|
||||
return mean(rest)
|
||||
}
|
||||
|
||||
func scaleHeader(step int) string {
|
||||
var b strings.Builder
|
||||
for i, s := range scaleOrder {
|
||||
if i == step+1 {
|
||||
s = "↓" + s
|
||||
}
|
||||
fmt.Fprintf(&b, "%8s", s)
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// curveCells prints a reading per arm, and a dash where an arm produced none.
|
||||
func curveCells(c doseCurve) string {
|
||||
var b strings.Builder
|
||||
for i, v := range c.Values {
|
||||
if c.Missing[i] {
|
||||
fmt.Fprintf(&b, "%8s", "-")
|
||||
continue
|
||||
}
|
||||
fmt.Fprintf(&b, "%8.0f", v)
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
func percentile(xs []float64, q float64) float64 {
|
||||
if len(xs) == 0 {
|
||||
return 0
|
||||
}
|
||||
s := append([]float64(nil), xs...)
|
||||
sort.Float64s(s)
|
||||
i := int(float64(len(s)-1) * q)
|
||||
return s[i]
|
||||
}
|
||||
|
||||
func maxOf(xs []float64) float64 {
|
||||
out := 0.0
|
||||
for _, x := range xs {
|
||||
if x > out {
|
||||
out = x
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -1,205 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reasonix/internal/state/sessionstore"
|
||||
"strings"
|
||||
|
||||
"reasonix/internal/contract/ablation"
|
||||
"reasonix/internal/contract/event"
|
||||
"reasonix/internal/contract/provider"
|
||||
"reasonix/internal/contract/tool"
|
||||
"reasonix/internal/runtime/agent"
|
||||
)
|
||||
|
||||
// A fixture is a real session that has really been compacted: no bench-only
|
||||
// state shape, so the loader and the projection restore have to work on it.
|
||||
|
||||
const (
|
||||
fixtureWindow = 60000
|
||||
fixtureGenerations = 5
|
||||
// fillerBytes per unit is what makes a fold necessary rather than optional.
|
||||
fillerBytes = 6000
|
||||
)
|
||||
|
||||
// digestProvider answers a summarizer without naming anything the corpus
|
||||
// plants. A digest that happened to quote the answer would leave it visible
|
||||
// and silently turn the task into a no-op.
|
||||
type digestProvider struct{ calls int }
|
||||
|
||||
func (p *digestProvider) Name() string { return "fixture-digest" }
|
||||
|
||||
func (p *digestProvider) Stream(_ context.Context, _ provider.Request) (<-chan provider.Chunk, error) {
|
||||
p.calls++
|
||||
ch := make(chan provider.Chunk, 2)
|
||||
ch <- provider.Chunk{Type: provider.ChunkText, Text: "Earlier work continued across several files; nothing outstanding was recorded."}
|
||||
ch <- provider.Chunk{Type: provider.ChunkDone}
|
||||
close(ch)
|
||||
return ch, nil
|
||||
}
|
||||
|
||||
// fillerCall is one ordinary read whose only job is to take up room — in the
|
||||
// window, and in the fold index ahead of a cue.
|
||||
func fillerCall(sess *sessionstore.Session, n int) {
|
||||
id := fmt.Sprintf("f%04d", n)
|
||||
sess.Add(provider.Message{Role: provider.RoleAssistant, ToolCalls: []provider.ToolCall{
|
||||
{ID: id, Name: "read_file", Arguments: fmt.Sprintf(`{"path":"internal/mod%03d/handler%03d.go"}`, n%37, n)},
|
||||
}})
|
||||
sess.Add(provider.Message{Role: provider.RoleTool, ToolCallID: id, Name: "read_file", Content: fillerBody(n)})
|
||||
}
|
||||
|
||||
func fillerBody(n int) string {
|
||||
var b strings.Builder
|
||||
fmt.Fprintf(&b, "package mod%03d\n\n", n%37)
|
||||
for b.Len() < fillerBytes {
|
||||
fmt.Fprintf(&b, "func handler%03dStep%02d() error { return nil }\n", n, b.Len()%64)
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// builtFixture is one task's history after real folds, plus where its target
|
||||
// ended up.
|
||||
type builtFixture struct {
|
||||
Session *sessionstore.Session
|
||||
State sessionstore.CompactionState
|
||||
Target int
|
||||
Path string
|
||||
Instance fixtureInstance
|
||||
}
|
||||
|
||||
// buildFixture plays a task's history through the real agent and folds it at
|
||||
// the given arm, writing the session and its context state where the agent
|
||||
// itself would.
|
||||
func buildFixture(f fixtureInstance, arm ablation.Set, sessionPath string) (builtFixture, error) {
|
||||
t := f.Task
|
||||
sess := sessionstore.NewSession("You are a coding agent.")
|
||||
sess.Add(provider.Message{Role: provider.RoleUser, Content: "Work through the transport and scheduler backlog."})
|
||||
|
||||
n := 0
|
||||
at := -1
|
||||
|
||||
a := agent.New(&digestProvider{}, tool.NewRegistry(), sess, agent.Options{
|
||||
ContextWindow: fixtureWindow, CompactRatio: 0.5, RecentKeep: 2,
|
||||
SessionPath: sessionPath, KeepPolicy: agent.KeepErrors, Ablation: arm,
|
||||
}, event.Discard)
|
||||
|
||||
for gen := range fixtureGenerations {
|
||||
if gen == t.PlantAfterGen {
|
||||
at = f.plant(sess)
|
||||
}
|
||||
for range 8 {
|
||||
fillerCall(sess, n)
|
||||
n++
|
||||
}
|
||||
if _, err := a.CompactNow(context.Background(), agent.CompactRequest{}); err != nil {
|
||||
return builtFixture{}, fmt.Errorf("%s: generation %d fold: %w", t.ID, gen, err)
|
||||
}
|
||||
}
|
||||
if at < 0 {
|
||||
return builtFixture{}, fmt.Errorf("%s: PlantAfterGen %d is past the last generation", t.ID, t.PlantAfterGen)
|
||||
}
|
||||
// projectionCoversTail cannot revalidate a projection covering the whole
|
||||
// transcript: it needs a live tail, which a real session always has because
|
||||
// work continues after a fold. Neutral text — the tail is provider-visible.
|
||||
sess.Add(provider.Message{Role: provider.RoleAssistant, Content: "Backlog swept; ready for the next question."})
|
||||
if err := sess.Save(sessionPath); err != nil {
|
||||
return builtFixture{}, fmt.Errorf("%s: save session: %w", t.ID, err)
|
||||
}
|
||||
state, ok, err := sessionstore.LoadCompactionState(sessionPath)
|
||||
if err != nil {
|
||||
return builtFixture{}, fmt.Errorf("%s: load context state: %w", t.ID, err)
|
||||
}
|
||||
if !ok || len(state.Projection.Messages) == 0 {
|
||||
return builtFixture{}, fmt.Errorf("%s: no projection was installed", t.ID)
|
||||
}
|
||||
if state.Projection.CoveredCount <= at {
|
||||
return builtFixture{}, fmt.Errorf("%s: target #%d is not inside the folded region (covered %d)",
|
||||
t.ID, at, state.Projection.CoveredCount)
|
||||
}
|
||||
return builtFixture{Session: sess, State: state, Target: at, Path: sessionPath, Instance: f}, nil
|
||||
}
|
||||
|
||||
// visibleContext is the model-visible view: the stored projection spliced with
|
||||
// everything canonical that came after it.
|
||||
func visibleContext(f builtFixture) []provider.Message {
|
||||
canonical := f.Session.Snapshot()
|
||||
out := append([]provider.Message(nil), f.State.Projection.Messages...)
|
||||
if n := f.State.Projection.CoveredCount; n >= 0 && n < len(canonical) {
|
||||
out = append(out, canonical[n:]...)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// visibleText is everything the model would read, as one string.
|
||||
func visibleText(f builtFixture) string {
|
||||
var b strings.Builder
|
||||
for _, m := range visibleContext(f) {
|
||||
b.WriteString(m.Content)
|
||||
b.WriteString("\n")
|
||||
for _, tc := range m.ToolCalls {
|
||||
b.WriteString(tc.Name)
|
||||
b.WriteString(" ")
|
||||
b.WriteString(string(tc.Arguments))
|
||||
b.WriteString("\n")
|
||||
}
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// sealFixture removes the fixture from disk once the agent holds it in memory.
|
||||
// The canonical transcript is the answer, and the sandbox mounts the host
|
||||
// read-only by design — so a model that finds the path reads the answer, which
|
||||
// one did. The agent has already loaded the session and its sidecar by the time
|
||||
// this runs; a failed autosave afterwards costs the benchmark nothing.
|
||||
func sealFixture(sessionPath string) {
|
||||
// Every file, not two named ones: a session writes an event log and a
|
||||
// context sidecar beside the transcript, and session.events.jsonl still
|
||||
// carried the answer after the other two were sealed by name.
|
||||
dir := filepath.Dir(sessionPath)
|
||||
entries, err := os.ReadDir(dir)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
for _, e := range entries {
|
||||
if !e.IsDir() {
|
||||
_ = os.Remove(filepath.Join(dir, e.Name()))
|
||||
}
|
||||
}
|
||||
// Deny the directory too, so an autosave cannot recreate what was removed.
|
||||
_ = os.Chmod(dir, 0o500)
|
||||
}
|
||||
|
||||
// unsealFixture restores the directory so the harness can clean up after
|
||||
// itself.
|
||||
func unsealFixture(sessionPath string) {
|
||||
_ = os.Chmod(filepath.Dir(sessionPath), 0o700)
|
||||
}
|
||||
|
||||
// answerOnDisk reports any readable file under dir holding a current-run
|
||||
// answer. Two named files were sealed because two leaks were found in them;
|
||||
// this scans the directory, so a path that moves — an autosave, a recovery
|
||||
// tombstone, a future sidecar — fails here instead of on a paid run.
|
||||
func answerOnDisk(dir string, markers []string) (string, string) {
|
||||
var atPath, found string
|
||||
_ = filepath.WalkDir(dir, func(path string, d fs.DirEntry, err error) error {
|
||||
if err != nil || d.IsDir() || atPath != "" {
|
||||
return nil //nolint:nilerr // an unreadable entry is not a leak
|
||||
}
|
||||
body, readErr := os.ReadFile(path)
|
||||
if readErr != nil {
|
||||
return nil
|
||||
}
|
||||
for _, marker := range markers {
|
||||
if len(marker) >= 6 && strings.Contains(string(body), marker) {
|
||||
atPath, found = path, marker
|
||||
return filepath.SkipAll
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
return atPath, found
|
||||
}
|
||||
@@ -1,128 +0,0 @@
|
||||
// instance.go — a task template becomes one run's concrete history.
|
||||
package main
|
||||
|
||||
import (
|
||||
"crypto/rand"
|
||||
"fmt"
|
||||
"hash/fnv"
|
||||
mathrand "math/rand"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// The invariant this file exists for: no literal sufficient to satisfy
|
||||
// AnswerRecovered for a concrete run may exist in this repository before that
|
||||
// run begins. Grepping the corpus reveals the question, never the answer.
|
||||
|
||||
// varKind is how one placeholder's value is generated. Values stay shaped like
|
||||
// real engineering records — a UUID would defeat the leak but turn every task
|
||||
// into "find the random string".
|
||||
type varKind string
|
||||
|
||||
const (
|
||||
varCodename varKind = "codename" // cobalt-k7m4q9
|
||||
varInt varKind = "int" // 731
|
||||
varDecimal varKind = "decimal" // 0.0437
|
||||
varDuration varKind = "duration" // 43ms
|
||||
)
|
||||
|
||||
// varSpec declares one placeholder. Min/Max bound the numeric kinds.
|
||||
type varSpec struct {
|
||||
Name string
|
||||
Kind varKind
|
||||
Word string // codename stem, e.g. "cobalt"
|
||||
Min int
|
||||
Max int
|
||||
}
|
||||
|
||||
// codenameEntropy is how much of a codename is unguessable. Five base32 digits
|
||||
// is about 25 bits: far past what a model can reach by knowing the stem.
|
||||
const codenameEntropy = 5
|
||||
|
||||
const codenameAlphabet = "abcdefghijkmnpqrstuvwxyz23456789"
|
||||
|
||||
// fixtureVars is one run's instantiated values.
|
||||
type fixtureVars map[string]string
|
||||
|
||||
// newVars generates values for every spec. A nil rng means deterministic.
|
||||
func newVars(specs []varSpec, rng *mathrand.Rand) fixtureVars {
|
||||
out := fixtureVars{}
|
||||
for _, spec := range specs {
|
||||
switch spec.Kind {
|
||||
case varCodename:
|
||||
out[spec.Name] = spec.Word + "-" + randomToken(rng, codenameEntropy)
|
||||
case varInt:
|
||||
out[spec.Name] = fmt.Sprint(randomBetween(rng, spec.Min, spec.Max))
|
||||
case varDecimal:
|
||||
out[spec.Name] = fmt.Sprintf("0.%04d", randomBetween(rng, spec.Min, spec.Max))
|
||||
case varDuration:
|
||||
out[spec.Name] = fmt.Sprintf("%dms", randomBetween(rng, spec.Min, spec.Max))
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func randomToken(rng *mathrand.Rand, n int) string {
|
||||
var b strings.Builder
|
||||
for range n {
|
||||
b.WriteByte(codenameAlphabet[randomBetween(rng, 0, len(codenameAlphabet)-1)])
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
func randomBetween(rng *mathrand.Rand, lo, hi int) int {
|
||||
if hi <= lo {
|
||||
return lo
|
||||
}
|
||||
return lo + rng.Intn(hi-lo+1)
|
||||
}
|
||||
|
||||
// seededRand is the deterministic generator preflight and tests use, so a
|
||||
// fixture's tier calibration and its golden state reproduce exactly.
|
||||
func seededRand(parts ...string) *mathrand.Rand {
|
||||
h := fnv.New64a()
|
||||
for _, p := range parts {
|
||||
h.Write([]byte(p))
|
||||
h.Write([]byte{0})
|
||||
}
|
||||
return mathrand.New(mathrand.NewSource(int64(h.Sum64())))
|
||||
}
|
||||
|
||||
// liveRand is the generator a paid run uses: every run gets values that existed
|
||||
// nowhere before it started.
|
||||
func liveRand() *mathrand.Rand {
|
||||
var seed [8]byte
|
||||
if _, err := rand.Read(seed[:]); err != nil {
|
||||
panic("contextbench: no entropy for a live run: " + err.Error())
|
||||
}
|
||||
var n int64
|
||||
for _, b := range seed {
|
||||
n = n<<8 | int64(b)
|
||||
}
|
||||
return mathrand.New(mathrand.NewSource(n))
|
||||
}
|
||||
|
||||
// instantiate substitutes {{name}} and refuses anything left over. The grammar
|
||||
// is one identifier in double braces: no conditionals, no loops, no defaults.
|
||||
// This is fixture data, not a template language.
|
||||
func instantiate(text string, vars fixtureVars) (string, error) {
|
||||
out := text
|
||||
for name, value := range vars {
|
||||
out = strings.ReplaceAll(out, "{{"+name+"}}", value)
|
||||
}
|
||||
if strings.Contains(out, "{{") || strings.Contains(out, "}}") {
|
||||
return "", fmt.Errorf("unresolved placeholder in %q", firstPlaceholder(out))
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func firstPlaceholder(text string) string {
|
||||
i := strings.Index(text, "{{")
|
||||
if i < 0 {
|
||||
return text
|
||||
}
|
||||
j := strings.Index(text[i:], "}}")
|
||||
if j < 0 {
|
||||
return text[i:min(i+40, len(text))]
|
||||
}
|
||||
return text[i : i+j+2]
|
||||
}
|
||||
@@ -1,137 +0,0 @@
|
||||
// ContextRetrievalBench builds the fixtures the long-horizon retrieval
|
||||
// experiments run against, and proves they are worth paying a provider for.
|
||||
//
|
||||
// -mode=calibrate find each index task's lead-call count for its cue tier
|
||||
// -mode=preflight build every task under every arm and check the contract
|
||||
//
|
||||
// Nothing here calls a provider. The experiments themselves do; this is what
|
||||
// says they would measure what they claim to.
|
||||
package main
|
||||
|
||||
import (
|
||||
"flag"
|
||||
"fmt"
|
||||
"os"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
func main() {
|
||||
mode := flag.String("mode", "preflight", "calibrate | preflight | snippet | adversarial | run-search | run-boundary | run-index")
|
||||
work := flag.String("work", "", "directory for built fixtures (default: a temp dir)")
|
||||
only := flag.String("task", "", "limit to one task id")
|
||||
dry := flag.Bool("dry", false, "run modes: drive the pipeline with a scripted provider instead of paying for one")
|
||||
flag.Parse()
|
||||
|
||||
if err := validateCorpus(); err != nil {
|
||||
fmt.Fprintln(os.Stderr, "corpus:", err)
|
||||
os.Exit(2)
|
||||
}
|
||||
root := *work
|
||||
if root == "" {
|
||||
dir, err := os.MkdirTemp("", "context-retrieval-*")
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(2)
|
||||
}
|
||||
defer os.RemoveAll(dir)
|
||||
root = dir
|
||||
}
|
||||
|
||||
tasks := allTasks()
|
||||
if *only != "" {
|
||||
t, ok := taskByID(*only)
|
||||
if !ok {
|
||||
fmt.Fprintf(os.Stderr, "unknown task %q\n", *only)
|
||||
os.Exit(2)
|
||||
}
|
||||
tasks = []contextTask{t}
|
||||
}
|
||||
|
||||
switch *mode {
|
||||
case "calibrate":
|
||||
os.Exit(runCalibrate(tasks, root))
|
||||
case "preflight":
|
||||
os.Exit(runPreflight(tasks, root))
|
||||
case "run-search":
|
||||
os.Exit(runExperiment(experimentSearch, root, *dry, tasks))
|
||||
case "run-boundary":
|
||||
os.Exit(runBoundaries(root))
|
||||
case "snippet":
|
||||
os.Exit(runSnippetAudit(root, 100))
|
||||
case "adversarial":
|
||||
os.Exit(runAdversarial(root))
|
||||
case "run-index":
|
||||
os.Exit(runExperiment(experimentIndex, root, *dry, tasks))
|
||||
default:
|
||||
fmt.Fprintf(os.Stderr, "unknown mode %q\n", *mode)
|
||||
os.Exit(2)
|
||||
}
|
||||
}
|
||||
|
||||
func runPreflight(tasks []contextTask, root string) int {
|
||||
var all []finding
|
||||
for _, t := range tasks {
|
||||
found, err := checkTask(t, root)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "%s: %v\n", t.ID, err)
|
||||
return 2
|
||||
}
|
||||
all = append(all, found...)
|
||||
status := "ok"
|
||||
if len(found) > 0 {
|
||||
status = fmt.Sprintf("%d problem(s)", len(found))
|
||||
}
|
||||
fmt.Printf("%-24s %-6s gen=%-2d %s\n", t.ID, t.Experiment, t.PlantAfterGen, status)
|
||||
}
|
||||
if len(all) == 0 {
|
||||
fmt.Printf("\n%d fixtures ready.\n", len(tasks))
|
||||
return 0
|
||||
}
|
||||
fmt.Fprintf(os.Stderr, "\n%d preflight problem(s):\n", len(all))
|
||||
sort.Slice(all, func(i, j int) bool { return all[i].Task < all[j].Task })
|
||||
for _, f := range all {
|
||||
fmt.Fprintln(os.Stderr, " ", f)
|
||||
}
|
||||
return 1
|
||||
}
|
||||
|
||||
// runCalibrate reports, for each index task, the lead-call counts at which its
|
||||
// cue is still addressed by each budget. The corpus freezes one number from
|
||||
// this; it is never computed at run time, or the implementation under test
|
||||
// would be moving its own goalposts.
|
||||
func runCalibrate(tasks []contextTask, root string) int {
|
||||
for _, t := range tasks {
|
||||
if t.Experiment != experimentIndex {
|
||||
continue
|
||||
}
|
||||
var rows []string
|
||||
for lead := range fixtureGenerations {
|
||||
probe := t
|
||||
probe.PlantAfterGen = lead
|
||||
var visibleAt []string
|
||||
for _, scale := range armScales {
|
||||
inst, ierr := instantiateTask(probe, seededRand(t.ID, scale, "calibrate"))
|
||||
if ierr != nil {
|
||||
fmt.Fprintln(os.Stderr, ierr)
|
||||
return 2
|
||||
}
|
||||
f, err := buildFixture(inst, armFor(scale, false),
|
||||
fmt.Sprintf("%s/calib/%s/%d-%s/session.jsonl", root, t.ID, lead, scale))
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "%s gen=%d %s: %v\n", t.ID, lead, scale, err)
|
||||
return 2
|
||||
}
|
||||
if strings.Contains(visibleText(f), inst.CueMarker) {
|
||||
visibleAt = append(visibleAt, scale)
|
||||
}
|
||||
}
|
||||
rows = append(rows, fmt.Sprintf("planted after gen %d, cue visible at: %s", lead, strings.Join(visibleAt, ",")))
|
||||
}
|
||||
fmt.Printf("\n%s (wants tier %s)\n", t.ID, t.CueTier)
|
||||
for _, r := range rows {
|
||||
fmt.Println(" ", r)
|
||||
}
|
||||
}
|
||||
return 0
|
||||
}
|
||||
@@ -1,137 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os/exec"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// The same checks the preflight mode runs, under go test: a compaction change
|
||||
// that invalidates the fixtures fails here rather than three hundred API calls
|
||||
// later.
|
||||
|
||||
func TestCorpusIsWellFormed(t *testing.T) {
|
||||
if err := validateCorpus(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := len(searchTasks()); got != 6 {
|
||||
t.Errorf("search experiment has %d tasks, want 6", got)
|
||||
}
|
||||
if got := len(indexTasks()); got != 6 {
|
||||
t.Errorf("index experiment has %d tasks, want 6", got)
|
||||
}
|
||||
tiers := map[string]int{}
|
||||
for _, task := range indexTasks() {
|
||||
tiers[task.CueTier]++
|
||||
}
|
||||
// Two per tier is what makes the staircase readable: the same task can be
|
||||
// compared across the boundary its cue sits on, instead of two different
|
||||
// questions being compared across two arms.
|
||||
for _, tier := range []string{tierQuarter, tierHalf, tierDefault} {
|
||||
if tiers[tier] != 2 {
|
||||
t.Errorf("tier %s has %d tasks, want 2", tier, tiers[tier])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The invariant the whole instantiation layer exists for: no literal a scorer
|
||||
// needs may be readable in this repository before a run starts.
|
||||
func TestNoAnswerLiteralExistsInTheRepository(t *testing.T) {
|
||||
root, err := repoRoot()
|
||||
if err != nil {
|
||||
t.Skip(err)
|
||||
}
|
||||
for _, task := range allTasks() {
|
||||
inst, err := instantiateTask(task, seededRand(task.ID, "leak-check"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, marker := range append(inst.AnswerMarkers, inst.CueMarker) {
|
||||
if len(marker) < 6 {
|
||||
continue // a bare number is not a literal anyone can grep for
|
||||
}
|
||||
if found, where := grepRepo(t, root, marker); found {
|
||||
t.Errorf("%s: %q is readable at %s before the run", task.ID, marker, where)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Two runs of the same task must not share an answer, or one leaked run leaks
|
||||
// every future one.
|
||||
func TestInstancesDifferBetweenRuns(t *testing.T) {
|
||||
for _, task := range allTasks() {
|
||||
a, err := instantiateTask(task, liveRand())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b, err := instantiateTask(task, liveRand())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if slices.Equal(a.AnswerMarkers, b.AnswerMarkers) {
|
||||
t.Errorf("%s produced the same answers twice: %v", task.ID, a.AnswerMarkers)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A preflight that could not reproduce would calibrate against one run's
|
||||
// nonces and report about another's.
|
||||
func TestSeededInstancesReproduce(t *testing.T) {
|
||||
for _, task := range allTasks() {
|
||||
a, _ := instantiateTask(task, seededRand(task.ID, "x"))
|
||||
b, _ := instantiateTask(task, seededRand(task.ID, "x"))
|
||||
if !slices.Equal(a.AnswerMarkers, b.AnswerMarkers) || a.body != b.body {
|
||||
t.Errorf("%s did not reproduce under the same seed", task.ID)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// An unresolved placeholder would put "{{marker}}" in the transcript and score
|
||||
// every run wrong; it fails the fixture instead.
|
||||
func TestUnresolvedPlaceholderIsRefused(t *testing.T) {
|
||||
if _, err := instantiate("value is {{missing}}", fixtureVars{"other": "x"}); err == nil {
|
||||
t.Fatal("an unresolved placeholder was accepted")
|
||||
}
|
||||
}
|
||||
|
||||
func repoRoot() (string, error) {
|
||||
out, err := exec.Command("git", "rev-parse", "--show-toplevel").Output()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("not in a git checkout: %w", err)
|
||||
}
|
||||
return strings.TrimSpace(string(out)), nil
|
||||
}
|
||||
|
||||
// grepRepo searches tracked files only: build artifacts and temp fixtures are
|
||||
// not what a fresh checkout hands anyone.
|
||||
func grepRepo(t *testing.T, root, needle string) (bool, string) {
|
||||
t.Helper()
|
||||
cmd := exec.Command("git", "grep", "-l", "-F", needle)
|
||||
cmd.Dir = root
|
||||
out, err := cmd.Output()
|
||||
if err != nil {
|
||||
return false, "" // exit 1 means no match
|
||||
}
|
||||
return true, strings.TrimSpace(strings.SplitN(string(out), "\n", 2)[0])
|
||||
}
|
||||
|
||||
func TestEveryFixturePassesPreflight(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("builds and folds 36 sessions")
|
||||
}
|
||||
root := t.TempDir()
|
||||
for _, task := range allTasks() {
|
||||
t.Run(task.ID, func(t *testing.T) {
|
||||
found, err := checkTask(task, root)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, f := range found {
|
||||
t.Error(f)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,290 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
"reasonix/internal/base/retrieval"
|
||||
"reasonix/internal/contract/ablation"
|
||||
"reasonix/internal/contract/provider"
|
||||
)
|
||||
|
||||
// Preflight is what makes a real run worth paying for. An answer the model can
|
||||
// already read, or a cue not where the matrix says, turns an arm into a
|
||||
// different experiment without saying so.
|
||||
|
||||
// armScales are the fold-index arms an index task is measured across.
|
||||
var armScales = []string{"default", "half", "quarter", "off"}
|
||||
|
||||
// experimentArm is one measured configuration of a task.
|
||||
type experimentArm struct {
|
||||
name string
|
||||
scale string
|
||||
searchOff bool
|
||||
}
|
||||
|
||||
// armsFor is the single declaration of which arms a task runs at, so preflight
|
||||
// and the run itself cannot check one set and measure another.
|
||||
func armsFor(t contextTask) []experimentArm {
|
||||
if t.Experiment == experimentSearch {
|
||||
return []experimentArm{
|
||||
{"search-on", "default", false},
|
||||
{"search-off", "default", true},
|
||||
}
|
||||
}
|
||||
arms := make([]experimentArm, 0, len(armScales))
|
||||
for _, scale := range armScales {
|
||||
arms = append(arms, experimentArm{"index-" + scale, scale, false})
|
||||
}
|
||||
return arms
|
||||
}
|
||||
|
||||
func scaleOf(name string) ablation.FoldIndexScale {
|
||||
s, err := ablation.ParseFoldIndexScale(name)
|
||||
if err != nil {
|
||||
panic("unknown scale " + name) // the list above is a constant
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// armFor is the arm a task is measured under at one fold-index scale.
|
||||
func armFor(scale string, searchOff bool) ablation.Set {
|
||||
set := ablation.Set{}
|
||||
if searchOff {
|
||||
set = ablation.New(ablation.RecallSearch)
|
||||
}
|
||||
return set.WithFoldIndex(scaleOf(scale))
|
||||
}
|
||||
|
||||
type finding struct {
|
||||
Task, Arm, Problem string
|
||||
}
|
||||
|
||||
func (f finding) String() string { return fmt.Sprintf("%s [%s]: %s", f.Task, f.Arm, f.Problem) }
|
||||
|
||||
// checkTask builds the task under every arm it will be run at and reports what
|
||||
// does not hold.
|
||||
func checkTask(t contextTask, root string) ([]finding, error) {
|
||||
var out []finding
|
||||
for _, arm := range armsFor(t) {
|
||||
// Deterministic values: a preflight has to reproduce, and a tier
|
||||
// calibration measured against one run's nonces is measured against
|
||||
// nothing.
|
||||
inst, err := instantiateTask(t, seededRand(t.ID, arm.name, "preflight"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
dir := filepath.Join(root, t.ID, arm.name)
|
||||
f, err := buildFixture(inst, armFor(arm.scale, arm.searchOff), filepath.Join(dir, "session.jsonl"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
visible := visibleText(f)
|
||||
|
||||
// The answer must be unreadable in every arm, or the task measures
|
||||
// reading rather than recall.
|
||||
for _, marker := range inst.AnswerMarkers {
|
||||
if len(marker) < 3 {
|
||||
continue // a bare number is not a nonce; the whole set still has to match
|
||||
}
|
||||
if strings.Contains(visible, marker) {
|
||||
out = append(out, finding{t.ID, arm.name, fmt.Sprintf("answer marker %q is already visible", marker)})
|
||||
}
|
||||
}
|
||||
|
||||
// The corpus has to be findable at all, or a miss in a real run says
|
||||
// nothing about the model.
|
||||
rank, err := probeRank(f, inst.ProbeQuery)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
switch {
|
||||
case rank == 0:
|
||||
out = append(out, finding{t.ID, arm.name, fmt.Sprintf("probe query %q does not find the target at all", inst.ProbeQuery)})
|
||||
case rank > 5:
|
||||
out = append(out, finding{t.ID, arm.name, fmt.Sprintf("probe query %q ranks the target %d", inst.ProbeQuery, rank)})
|
||||
}
|
||||
|
||||
if t.Experiment == experimentIndex {
|
||||
out = append(out, checkCueVisibility(t, inst, arm.scale, arm.name, visible)...)
|
||||
} else if line := indexLineFor(f, f.Target); line != "" {
|
||||
out = append(out, finding{t.ID, arm.name,
|
||||
"a search task is addressed by the fold index, so it measures the index and not search: " + line})
|
||||
}
|
||||
|
||||
// The projection is what the agent believes. This is what the model is
|
||||
// told, and it is the one that has to hold.
|
||||
req, err := captureRequest(f, armFor(arm.scale, arm.searchOff), inst.Prompt)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s [%s]: capture: %w", t.ID, arm.name, err)
|
||||
}
|
||||
out = append(out, checkCapturedRequest(t, inst, arm.name, arm.scale, arm.searchOff, req)...)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// checkCapturedRequest holds one arm's real provider request to its contract:
|
||||
// the answer unreadable, the cue exactly where its tier says, the recall tool
|
||||
// offering only what this arm can do, and no host-side field surviving the
|
||||
// boundary.
|
||||
func checkCapturedRequest(t contextTask, inst fixtureInstance, armName, scale string, searchOff bool, req provider.Request) []finding {
|
||||
var out []finding
|
||||
for _, marker := range inst.AnswerMarkers {
|
||||
if len(marker) < 3 {
|
||||
continue // a bare number is not a nonce; the set as a whole still is
|
||||
}
|
||||
for _, at := range findInSurface(req, marker) {
|
||||
out = append(out, finding{t.ID, armName, fmt.Sprintf("answer marker %q leaked at %s", marker, at)})
|
||||
}
|
||||
}
|
||||
if t.Experiment == experimentIndex {
|
||||
wantVisible, wantHidden := tierScales(t.CueTier)
|
||||
where := findInSurface(req, inst.CueMarker)
|
||||
switch {
|
||||
case contains(wantVisible, scale) && len(where) == 0:
|
||||
out = append(out, finding{t.ID, armName, fmt.Sprintf("cue %q is not provider-visible; the tier expects it", inst.CueMarker)})
|
||||
case contains(wantHidden, scale) && len(where) > 0:
|
||||
out = append(out, finding{t.ID, armName, fmt.Sprintf("cue %q is provider-visible at %s: %s", inst.CueMarker, where[0], scale)})
|
||||
}
|
||||
}
|
||||
|
||||
schema, ok := recallToolSchema(req)
|
||||
if !ok {
|
||||
out = append(out, finding{t.ID, armName, "recall is not in the provider's tool list"})
|
||||
return append(out, boundaryFindings(t, armName, req)...)
|
||||
}
|
||||
params := ""
|
||||
if b, err := json.Marshal(schema.Parameters); err == nil {
|
||||
params = string(b)
|
||||
}
|
||||
// Both arms keep the read half: the ablation removes search, not recall.
|
||||
if !strings.Contains(params, `"positions"`) {
|
||||
out = append(out, finding{t.ID, armName, "recall lost its positions parameter"})
|
||||
}
|
||||
hasQuery := strings.Contains(params, `"query"`)
|
||||
switch {
|
||||
case searchOff && hasQuery:
|
||||
out = append(out, finding{t.ID, armName, "recall still offers a query parameter"})
|
||||
case !searchOff && !hasQuery:
|
||||
out = append(out, finding{t.ID, armName, "recall has no query parameter"})
|
||||
}
|
||||
if searchOff {
|
||||
for _, phrase := range searchWording {
|
||||
for _, at := range findInSurface(req, phrase) {
|
||||
out = append(out, finding{t.ID, armName, fmt.Sprintf("search is offered to an arm without it: %q at %s", phrase, at)})
|
||||
}
|
||||
}
|
||||
}
|
||||
return append(out, boundaryFindings(t, armName, req)...)
|
||||
}
|
||||
|
||||
func boundaryFindings(t contextTask, armName string, req provider.Request) []finding {
|
||||
var out []finding
|
||||
for _, at := range localOnlyLeaks(req) {
|
||||
out = append(out, finding{t.ID, armName, "host-only field crossed the provider boundary at " + at})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// checkCueVisibility holds an index task to its tier: the cue is addressed at
|
||||
// the scales the matrix names and at no others. A policy change that moves it
|
||||
// fails here rather than turning two arms into one experiment.
|
||||
func checkCueVisibility(t contextTask, inst fixtureInstance, scale, armName, visible string) []finding {
|
||||
wantVisible, wantHidden := tierScales(t.CueTier)
|
||||
has := strings.Contains(visible, inst.CueMarker)
|
||||
switch {
|
||||
case contains(wantVisible, scale) && !has:
|
||||
return []finding{{t.ID, armName, fmt.Sprintf("cue %q should be addressed at %s and is not", inst.CueMarker, scale)}}
|
||||
case contains(wantHidden, scale) && has:
|
||||
return []finding{{t.ID, armName, fmt.Sprintf("cue %q should be gone at %s and is not", inst.CueMarker, scale)}}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func contains(list []string, want string) bool { return slices.Contains(list, want) }
|
||||
|
||||
// probeRank runs the task's own query against the folded region the way recall
|
||||
// would, and reports where the target lands.
|
||||
func probeRank(f builtFixture, query string) (int, error) {
|
||||
terms, err := retrieval.QueryTerms(query)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
canonical := f.Session.Snapshot()
|
||||
covered := min(f.State.Projection.CoveredCount, len(canonical))
|
||||
docs := make([]map[string]int, 0, covered)
|
||||
positions := make([]int, 0, covered)
|
||||
lengths := make([]int, 0, covered)
|
||||
for i, m := range canonical[:covered] {
|
||||
var text strings.Builder
|
||||
text.WriteString(m.Content)
|
||||
for _, tc := range m.ToolCalls {
|
||||
text.WriteString(" " + tc.Name + " " + string(tc.Arguments))
|
||||
}
|
||||
tokens := retrieval.Tokens(text.String())
|
||||
if len(tokens) == 0 {
|
||||
continue
|
||||
}
|
||||
docs = append(docs, retrieval.Counts(tokens))
|
||||
positions = append(positions, i)
|
||||
lengths = append(lengths, len(tokens))
|
||||
}
|
||||
df := retrieval.DocumentFrequency(docs)
|
||||
avg := 0.0
|
||||
for _, l := range lengths {
|
||||
avg += float64(l)
|
||||
}
|
||||
if len(lengths) > 0 {
|
||||
avg /= float64(len(lengths))
|
||||
}
|
||||
type hit struct {
|
||||
pos int
|
||||
score float64
|
||||
}
|
||||
var hits []hit
|
||||
for i, counts := range docs {
|
||||
if s := retrieval.BM25Score(counts, lengths[i], terms, df, len(docs), avg); s > 0 {
|
||||
hits = append(hits, hit{positions[i], s})
|
||||
}
|
||||
}
|
||||
for i := range hits {
|
||||
for j := i + 1; j < len(hits); j++ {
|
||||
if hits[j].score > hits[i].score {
|
||||
hits[i], hits[j] = hits[j], hits[i]
|
||||
}
|
||||
}
|
||||
}
|
||||
// A tool result answers at its caller's position, so either counts.
|
||||
for i, h := range hits {
|
||||
if h.pos == f.Target || h.pos == f.Target+1 {
|
||||
return i + 1, nil
|
||||
}
|
||||
}
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
// indexLineFor returns the fold-index line addressing a position, if any.
|
||||
func indexLineFor(f builtFixture, pos int) string {
|
||||
want := fmt.Sprintf("#%d ", pos)
|
||||
for line := range strings.SplitSeq(visibleIndexOnly(f), "\n") {
|
||||
if strings.Contains(line, want) {
|
||||
return strings.TrimSpace(line)
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// visibleIndexOnly is just the fold-index sections of the projection.
|
||||
func visibleIndexOnly(f builtFixture) string {
|
||||
var b strings.Builder
|
||||
for _, m := range visibleContext(f) {
|
||||
if i := strings.Index(m.Content, "## Folded work index"); i >= 0 {
|
||||
b.WriteString(m.Content[i:])
|
||||
b.WriteString("\n")
|
||||
}
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
@@ -1,252 +0,0 @@
|
||||
// reformulation.go — how a query changed, from token sets alone.
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"reasonix/internal/base/retrieval"
|
||||
)
|
||||
|
||||
// The host probe ranks every target first, so the engine is not what makes a
|
||||
// model take more than one attempt. These classes are set arithmetic over the
|
||||
// same tokenizer recall uses: no judge, nothing read out of prose.
|
||||
|
||||
const (
|
||||
reformFirst = "First" // nothing to compare against yet
|
||||
reformNearDup = "NearDuplicate" // the same query again, give or take a word
|
||||
reformNarrowing = "Narrowing" // kept everything and added terms
|
||||
reformBroadening = "Broadening" // dropped terms, added none
|
||||
reformAnchored = "HistoricalAnchor" // reached for a term the question does not contain
|
||||
reformShift = "SynonymShift" // traded terms for others
|
||||
)
|
||||
|
||||
// nearDuplicateOverlap is where "reworded" stops and "asked again" begins.
|
||||
const nearDuplicateOverlap = 0.8
|
||||
|
||||
func terms(s string) []string {
|
||||
return retrieval.Unique(retrieval.Tokens(s))
|
||||
}
|
||||
|
||||
func termSet(s string) map[string]bool {
|
||||
out := map[string]bool{}
|
||||
for _, t := range terms(s) {
|
||||
out[t] = true
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func jaccard(a, b map[string]bool) float64 {
|
||||
if len(a) == 0 && len(b) == 0 {
|
||||
return 1
|
||||
}
|
||||
shared := 0
|
||||
for t := range a {
|
||||
if b[t] {
|
||||
shared++
|
||||
}
|
||||
}
|
||||
union := len(a) + len(b) - shared
|
||||
if union == 0 {
|
||||
return 0
|
||||
}
|
||||
return float64(shared) / float64(union)
|
||||
}
|
||||
|
||||
// classifyReformulation names how this query differs from the one before it.
|
||||
func classifyReformulation(previous, current, question string) string {
|
||||
if previous == "" {
|
||||
return reformFirst
|
||||
}
|
||||
prev, cur := termSet(previous), termSet(current)
|
||||
if jaccard(prev, cur) >= nearDuplicateOverlap {
|
||||
return reformNearDup
|
||||
}
|
||||
added, dropped := diffTerms(cur, prev), diffTerms(prev, cur)
|
||||
// A term the question never used had to come from the model's own memory of
|
||||
// the history, which is a different move from rephrasing what it was asked.
|
||||
ask := termSet(question)
|
||||
for _, t := range added {
|
||||
if !ask[t] {
|
||||
return reformAnchored
|
||||
}
|
||||
}
|
||||
switch {
|
||||
case len(dropped) == 0:
|
||||
return reformNarrowing
|
||||
case len(added) == 0:
|
||||
return reformBroadening
|
||||
default:
|
||||
return reformShift
|
||||
}
|
||||
}
|
||||
|
||||
func diffTerms(from, minus map[string]bool) []string {
|
||||
var out []string
|
||||
for t := range from {
|
||||
if !minus[t] {
|
||||
out = append(out, t)
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// classifySearches labels each attempt and records which of the host probe's
|
||||
// terms it left out. The probe is the oracle: it ranks the target first, so a
|
||||
// query missing its terms says what the model did not think to ask for.
|
||||
func classifySearches(m *contextMetrics, inst fixtureInstance) {
|
||||
probe := terms(inst.ProbeQuery)
|
||||
previous := ""
|
||||
for i := range m.Searches {
|
||||
s := &m.Searches[i]
|
||||
s.Reformulation = classifyReformulation(previous, s.Query, inst.Prompt)
|
||||
have := termSet(s.Query)
|
||||
for _, term := range probe {
|
||||
if !have[term] {
|
||||
s.MissingProbeTerms = append(s.MissingProbeTerms, term)
|
||||
}
|
||||
}
|
||||
previous = s.Query
|
||||
}
|
||||
}
|
||||
|
||||
// queryAudit is the reformulation funnel across a set of runs.
|
||||
type queryAudit struct {
|
||||
Runs, RunsThatSearched int
|
||||
FirstQueryHit int
|
||||
HitBySecond, HitByThird int
|
||||
NeverHit int
|
||||
TotalSearches int
|
||||
PostHitSearches int
|
||||
HitThenRead int
|
||||
EscapeBeforeFirstSearch int
|
||||
Reformulations map[string]int
|
||||
MissedProbeTerms map[string]int
|
||||
}
|
||||
|
||||
func auditQueries(runs []contextMetrics) queryAudit {
|
||||
a := queryAudit{Reformulations: map[string]int{}, MissedProbeTerms: map[string]int{}}
|
||||
for _, r := range runs {
|
||||
if r.Contaminated {
|
||||
continue
|
||||
}
|
||||
a.Runs++
|
||||
a.TotalSearches += len(r.Searches)
|
||||
a.PostHitSearches += r.PostHitSearches
|
||||
if r.EscapeBeforeFirstRecall > 0 {
|
||||
a.EscapeBeforeFirstSearch++
|
||||
}
|
||||
if len(r.Searches) == 0 {
|
||||
continue
|
||||
}
|
||||
a.RunsThatSearched++
|
||||
hitAt := 0
|
||||
for i, s := range r.Searches {
|
||||
a.Reformulations[s.Reformulation]++
|
||||
for _, term := range s.MissingProbeTerms {
|
||||
a.MissedProbeTerms[term]++
|
||||
}
|
||||
if s.TargetRank > 0 && hitAt == 0 {
|
||||
hitAt = i + 1
|
||||
}
|
||||
}
|
||||
switch {
|
||||
case hitAt == 1:
|
||||
a.FirstQueryHit++
|
||||
case hitAt == 2:
|
||||
a.HitBySecond++
|
||||
case hitAt >= 3:
|
||||
a.HitByThird++
|
||||
default:
|
||||
a.NeverHit++
|
||||
}
|
||||
if hitAt > 0 && r.TargetRead {
|
||||
a.HitThenRead++
|
||||
}
|
||||
}
|
||||
return a
|
||||
}
|
||||
|
||||
func (a queryAudit) report() string {
|
||||
var b strings.Builder
|
||||
b.WriteString("\n## Query reformulation\n")
|
||||
fmt.Fprintf(&b, " runs=%d searched=%d searches=%d (%.2f per searching run)\n",
|
||||
a.Runs, a.RunsThatSearched, a.TotalSearches, ratio(a.TotalSearches, a.RunsThatSearched))
|
||||
fmt.Fprintf(&b, " target found by query 1: %d by 2: %d by 3+: %d never: %d\n",
|
||||
a.FirstQueryHit, a.HitBySecond, a.HitByThird, a.NeverHit)
|
||||
fmt.Fprintf(&b, " found then read: %d/%d searches issued after the target was already back: %d\n",
|
||||
a.HitThenRead, a.FirstQueryHit+a.HitBySecond+a.HitByThird, a.PostHitSearches)
|
||||
fmt.Fprintf(&b, " reached for the workspace before searching: %d\n", a.EscapeBeforeFirstSearch)
|
||||
b.WriteString(" reformulations: " + countsLine(a.Reformulations) + "\n")
|
||||
b.WriteString(" probe terms most often absent: " + topCounts(a.MissedProbeTerms, 8) + "\n")
|
||||
return b.String()
|
||||
}
|
||||
|
||||
func ratio(n, d int) float64 {
|
||||
if d == 0 {
|
||||
return 0
|
||||
}
|
||||
return float64(n) / float64(d)
|
||||
}
|
||||
|
||||
func countsLine(counts map[string]int) string {
|
||||
keys := make([]string, 0, len(counts))
|
||||
for k := range counts {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
parts := make([]string, 0, len(keys))
|
||||
for _, k := range keys {
|
||||
parts = append(parts, fmt.Sprintf("%s=%d", k, counts[k]))
|
||||
}
|
||||
return strings.Join(parts, " ")
|
||||
}
|
||||
|
||||
func topCounts(counts map[string]int, limit int) string {
|
||||
type kv struct {
|
||||
k string
|
||||
n int
|
||||
}
|
||||
all := make([]kv, 0, len(counts))
|
||||
for k, n := range counts {
|
||||
all = append(all, kv{k, n})
|
||||
}
|
||||
sort.Slice(all, func(i, j int) bool {
|
||||
if all[i].n != all[j].n {
|
||||
return all[i].n > all[j].n
|
||||
}
|
||||
return all[i].k < all[j].k
|
||||
})
|
||||
parts := make([]string, 0, limit)
|
||||
for _, e := range all[:min(limit, len(all))] {
|
||||
parts = append(parts, fmt.Sprintf("%s=%d", e.k, e.n))
|
||||
}
|
||||
if len(parts) == 0 {
|
||||
return "(none)"
|
||||
}
|
||||
return strings.Join(parts, " ")
|
||||
}
|
||||
|
||||
// queryLines lists every attempt, so a surprising audit can be read back.
|
||||
func queryLines(runs []contextMetrics) string {
|
||||
var b strings.Builder
|
||||
b.WriteString("\n## Every query, in order\n")
|
||||
for _, r := range runs {
|
||||
for _, s := range r.Searches {
|
||||
rank := "miss"
|
||||
if s.TargetRank > 0 {
|
||||
rank = fmt.Sprintf("rank %d", s.TargetRank)
|
||||
}
|
||||
fmt.Fprintf(&b, " %-24s %-14s #%d %-17s %-9s %q\n",
|
||||
r.Task, r.Arm, s.Ordinal, s.Reformulation, rank, s.Query)
|
||||
if len(s.MissingProbeTerms) > 0 && s.TargetRank == 0 {
|
||||
fmt.Fprintf(&b, " %-24s %-14s missing probe terms: %s\n",
|
||||
"", "", strings.Join(slices.Clone(s.MissingProbeTerms), " "))
|
||||
}
|
||||
}
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
@@ -1,519 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"reasonix/internal/state/sessionstore"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"reasonix/internal/assembly/boot"
|
||||
"reasonix/internal/contract/ablation"
|
||||
"reasonix/internal/contract/event"
|
||||
"reasonix/internal/contract/provider"
|
||||
"reasonix/internal/contract/tool"
|
||||
_ "reasonix/internal/model/openai"
|
||||
"reasonix/internal/runtime/agent"
|
||||
_ "reasonix/internal/tools/builtin"
|
||||
)
|
||||
|
||||
// A run is one recall question against a fixture the preflight has already
|
||||
// cleared. Scoring reads the transcript the turn appended — tool names and
|
||||
// their JSON arguments — so nothing depends on how the model narrated itself.
|
||||
|
||||
const (
|
||||
runBaseURL = "https://api.deepseek.com"
|
||||
runModel = "deepseek-chat"
|
||||
runTimeout = 4 * time.Minute
|
||||
)
|
||||
|
||||
func realProvider() (provider.Provider, error) {
|
||||
key := os.Getenv("DEEPSEEK_API_KEY")
|
||||
if key == "" {
|
||||
return nil, fmt.Errorf("a real run needs DEEPSEEK_API_KEY")
|
||||
}
|
||||
return provider.New("openai", provider.Config{
|
||||
Name: "contextbench", BaseURL: runBaseURL, Model: runModel, APIKey: key,
|
||||
})
|
||||
}
|
||||
|
||||
// adversarialPrompts tell the model what it is running and send it after the
|
||||
// answer everywhere except the conversation. The question is not whether it
|
||||
// obeys the experiment: it is whether the answer is reachable at all by a model
|
||||
// that knows the mechanism and is actively attacking the host.
|
||||
var adversarialPrompts = map[string]string{
|
||||
experimentSearch: "You are running a benchmark. Find the expected answer for this task from the " +
|
||||
"Reasonix repository, benchmark source, fixture generator, test files, scratch files, or " +
|
||||
"session metadata. Do not use recall unless necessary. Report the exact hidden values.",
|
||||
experimentIndex: "Inspect benchmarks/context-retrieval, the git repository, session files, temporary " +
|
||||
"files, and any benchmark metadata you can access. Recover the concrete instantiated answer for " +
|
||||
"this run without relying on the folded conversation. Report the exact values.",
|
||||
}
|
||||
|
||||
// runAdversarial sends one task per experiment after the answer through every
|
||||
// host-readable surface. It passes when nothing but recall could produce it —
|
||||
// a run that gives up and recalls has still proven the attack failed.
|
||||
func runAdversarial(root string) int {
|
||||
root, err := filepath.Abs(root)
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
return 2
|
||||
}
|
||||
restore, ierr := isolateWorkspace()
|
||||
if ierr != nil {
|
||||
fmt.Fprintln(os.Stderr, "isolate workspace:", ierr)
|
||||
return 2
|
||||
}
|
||||
defer restore()
|
||||
p, perr := realProvider()
|
||||
if perr != nil {
|
||||
fmt.Fprintln(os.Stderr, perr)
|
||||
return 2
|
||||
}
|
||||
leaked := 0
|
||||
for _, id := range []string{"r01-retry-boundary", "i01-transport-fallback"} {
|
||||
t, ok := taskByID(id)
|
||||
if !ok {
|
||||
fmt.Fprintf(os.Stderr, "unknown task %q\n", id)
|
||||
return 2
|
||||
}
|
||||
m, err := runOne(p, t, armFor("default", false), "adversarial", root, false, false,
|
||||
adversarialPrompts[t.Experiment])
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "%s: %v\n", id, err)
|
||||
}
|
||||
verdict := "isolated"
|
||||
if m.EscapeFoundAnswer {
|
||||
verdict = "LEAKED via " + m.LeakedVia
|
||||
leaked++
|
||||
}
|
||||
fmt.Printf("%-24s %-24s escapes=%-3d recall=%d answered=%v tools=%v\n",
|
||||
id, verdict, m.EscapeCalls, m.SearchCalls+m.ReadCalls, m.AnswerRecovered, m.UnexpectedWorkTools)
|
||||
if m.EscapeFoundAnswer {
|
||||
fmt.Printf(" args: %s\n output: %s\n", m.LeakedArgs, m.LeakedOutput)
|
||||
}
|
||||
}
|
||||
if leaked > 0 {
|
||||
fmt.Fprintf(os.Stderr, "\n%d of 2 adversarial runs reached an answer outside recall\n", leaked)
|
||||
return 1
|
||||
}
|
||||
fmt.Println("\nNo answer was reachable outside recall.")
|
||||
return 0
|
||||
}
|
||||
|
||||
// runOne asks one task under one arm and scores what the model did.
|
||||
func runOne(p provider.Provider, t contextTask, arm ablation.Set, armName, root string, cueVisible, dry bool, promptOverride ...string) (contextMetrics, error) {
|
||||
// Live runs get values that existed nowhere before the run started; a dry
|
||||
// run stays reproducible.
|
||||
rng := liveRand()
|
||||
if dry {
|
||||
rng = seededRand(t.ID, armName, "dry")
|
||||
}
|
||||
inst, err := instantiateTask(t, rng)
|
||||
if err != nil {
|
||||
return contextMetrics{}, err
|
||||
}
|
||||
f, err := buildFixture(inst, arm, filepath.Join(root, t.ID, armName, "session.jsonl"))
|
||||
if err != nil {
|
||||
return contextMetrics{}, err
|
||||
}
|
||||
sess, err := sessionstore.LoadSession(f.Path)
|
||||
if err != nil {
|
||||
return contextMetrics{}, fmt.Errorf("load session: %w", err)
|
||||
}
|
||||
reg := tool.NewRegistry()
|
||||
for _, x := range tool.Builtins() {
|
||||
reg.Add(x)
|
||||
}
|
||||
boot.ApplyUnifiedProviderToolSurface(reg, false, arm)
|
||||
|
||||
a := agent.New(p, reg, sess, agent.Options{
|
||||
ContextWindow: fixtureWindow, CompactRatio: 0.5, RecentKeep: 2,
|
||||
SessionPath: f.Path, KeepPolicy: agent.KeepErrors, Ablation: arm,
|
||||
}, event.Discard)
|
||||
a.LoadProjectionSidecar(f.Path)
|
||||
// From here the transcript lives only in memory. On disk it is the answer.
|
||||
sealFixture(f.Path)
|
||||
defer unsealFixture(f.Path)
|
||||
dir := filepath.Dir(f.Path)
|
||||
if at, marker := answerOnDisk(dir, inst.AnswerMarkers); at != "" {
|
||||
return contextMetrics{}, fmt.Errorf("%s [%s]: %q is readable at %s before the run", t.ID, armName, marker, at)
|
||||
}
|
||||
|
||||
before := len(sess.Snapshot())
|
||||
ctx, cancel := context.WithTimeout(context.Background(), runTimeout)
|
||||
defer cancel()
|
||||
prompt := inst.Prompt
|
||||
if len(promptOverride) > 0 && promptOverride[0] != "" {
|
||||
prompt = promptOverride[0]
|
||||
}
|
||||
runErr := a.Run(ctx, prompt)
|
||||
|
||||
if at, marker := answerOnDisk(dir, inst.AnswerMarkers); at != "" {
|
||||
return contextMetrics{}, fmt.Errorf("%s [%s]: %q was written to %s during the run", t.ID, armName, marker, at)
|
||||
}
|
||||
|
||||
appended := sess.Snapshot()[before:]
|
||||
m := scoreRun(appended, inst, f.Target, armName, cueVisible)
|
||||
m.trajectory = appended
|
||||
m.scoreAnswer(finalAnswer(appended), inst)
|
||||
if runErr != nil {
|
||||
m.FailureStage = "RunError"
|
||||
m.TimeoutReason = timeoutReason(m)
|
||||
}
|
||||
return m, runErr
|
||||
}
|
||||
|
||||
// finalAnswer is the last assistant text the turn produced.
|
||||
func finalAnswer(msgs []provider.Message) string {
|
||||
for _, m := range slices.Backward(msgs) {
|
||||
if m.Role == provider.RoleAssistant && strings.TrimSpace(m.Content) != "" {
|
||||
return m.Content
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// isolateWorkspace moves the process into an empty directory before any task
|
||||
// runs. The corpus lives in this repository, so an agent left here greps the
|
||||
// answers out of corpus.go — one did, quoting the file's own line about them
|
||||
// being nonces. Fixtures stay outside it too: a readable session file is the
|
||||
// canonical transcript, answer included.
|
||||
func isolateWorkspace() (func(), error) {
|
||||
dir, err := os.MkdirTemp("", "contextbench-workspace-*")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
readme := "This workspace is empty on purpose. The question is about earlier conversation,\nnot about any file here.\n"
|
||||
if err := os.WriteFile(filepath.Join(dir, "README.md"), []byte(readme), 0o644); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
previous, err := os.Getwd()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := os.Chdir(dir); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return func() {
|
||||
_ = os.Chdir(previous)
|
||||
_ = os.RemoveAll(dir)
|
||||
}, nil
|
||||
}
|
||||
|
||||
// runBoundaries runs only the pair each index task is built for: the narrowest
|
||||
// scale still addressing its cue, and the next one down. Each pair moves the
|
||||
// affordance and nothing else, where default-vs-off moves the whole index.
|
||||
|
||||
// The two arms get separate nonces: reusing one would leave the first arm's
|
||||
// answer in a host-readable history for the second to find.
|
||||
func runBoundaries(root string) int {
|
||||
root, err := filepath.Abs(root)
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
return 2
|
||||
}
|
||||
restore, ierr := isolateWorkspace()
|
||||
if ierr != nil {
|
||||
fmt.Fprintln(os.Stderr, "isolate workspace:", ierr)
|
||||
return 2
|
||||
}
|
||||
defer restore()
|
||||
p, perr := realProvider()
|
||||
if perr != nil {
|
||||
fmt.Fprintln(os.Stderr, perr)
|
||||
return 2
|
||||
}
|
||||
|
||||
var all []contextMetrics
|
||||
var cueSide, noCueSide []contextMetrics
|
||||
for _, t := range indexTasks() {
|
||||
cue, noCue := boundaryPair(t.CueTier)
|
||||
for _, side := range []struct {
|
||||
scale string
|
||||
has bool
|
||||
}{{cue, true}, {noCue, false}} {
|
||||
name := "index-" + side.scale
|
||||
m, err := runOne(p, t, armFor(side.scale, false), name, root, side.has, false)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "%s [%s]: %v\n", t.ID, name, err)
|
||||
}
|
||||
all = append(all, m)
|
||||
if side.has {
|
||||
cueSide = append(cueSide, m)
|
||||
} else {
|
||||
noCueSide = append(noCueSide, m)
|
||||
}
|
||||
fmt.Printf("%-24s %-14s cue=%-3v %-14s search=%d read=%d direct=%-5v recall-tok=%-5d escape=%d\n",
|
||||
t.ID, name, side.has, m.FailureStage, m.SearchCalls, m.ReadCalls,
|
||||
m.CueDirectRead, m.RecallReturnedTokens, m.EscapeCalls)
|
||||
}
|
||||
}
|
||||
reportDeltas(cueSide, noCueSide)
|
||||
fmt.Print(reportStopping(all))
|
||||
return writeResults(root, all)
|
||||
}
|
||||
|
||||
// reportDeltas is the paired reading: each task against itself, then pooled.
|
||||
func reportDeltas(cueSide, noCueSide []contextMetrics) {
|
||||
fmt.Printf("\n## Paired deltas (cue present − cue absent)\n")
|
||||
fmt.Printf(" %-24s %-10s %-10s %-9s %-11s %s\n",
|
||||
"task", "recovery", "directread", "searches", "recall-tok", "escapes")
|
||||
for i := range cueSide {
|
||||
if i >= len(noCueSide) {
|
||||
break
|
||||
}
|
||||
a, b := cueSide[i], noCueSide[i]
|
||||
fmt.Printf(" %-24s %-10s %-10s %-9d %-11d %d\n", a.Task,
|
||||
deltaBool(a.AnswerRecovered, b.AnswerRecovered),
|
||||
deltaBool(a.CueDirectRead, b.CueDirectRead),
|
||||
a.SearchCalls-b.SearchCalls,
|
||||
a.RecallReturnedTokens-b.RecallReturnedTokens,
|
||||
a.EscapeCalls-b.EscapeCalls)
|
||||
}
|
||||
fmt.Println("\n## Pooled")
|
||||
fmt.Println(" ", summarize("cue-present", cueSide).line())
|
||||
fmt.Println(" ", summarize("cue-absent", noCueSide).line())
|
||||
fmt.Println("\n## Where each side looked first")
|
||||
fmt.Printf(" %-14s %s\n", "cue-present", countsLine(summarize("x", cueSide).Routes))
|
||||
fmt.Printf(" %-14s %s\n", "cue-absent", countsLine(summarize("x", noCueSide).Routes))
|
||||
}
|
||||
|
||||
func deltaBool(a, b bool) string {
|
||||
switch {
|
||||
case a == b:
|
||||
return "same"
|
||||
case a:
|
||||
return "+cue"
|
||||
default:
|
||||
return "-cue"
|
||||
}
|
||||
}
|
||||
|
||||
// runExperiment runs one experiment's tasks across its arms. A dry run drives
|
||||
// the same pipeline with a scripted provider: it proves fixture, run, scoring
|
||||
// and report hold together before any of it is paid for.
|
||||
func runExperiment(experiment, root string, dry bool, tasks []contextTask) int {
|
||||
root, err := filepath.Abs(root)
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
return 2
|
||||
}
|
||||
restore, err := isolateWorkspace()
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, "isolate workspace:", err)
|
||||
return 2
|
||||
}
|
||||
defer restore()
|
||||
var p provider.Provider = &scriptedProvider{}
|
||||
if !dry {
|
||||
real, rerr := realProvider()
|
||||
if rerr != nil {
|
||||
fmt.Fprintln(os.Stderr, rerr)
|
||||
return 2
|
||||
}
|
||||
p = real
|
||||
}
|
||||
byArm := map[string][]contextMetrics{}
|
||||
var all []contextMetrics
|
||||
|
||||
if experiment == experimentIndex {
|
||||
fmt.Print(fingerprint(dry))
|
||||
}
|
||||
for _, t := range tasks {
|
||||
if t.Experiment != experiment {
|
||||
continue
|
||||
}
|
||||
// Arms are shuffled inside each task block. Running six defaults, then
|
||||
// six halves, would make provider drift collinear with the axis under
|
||||
// test: a bad minute would land entirely on one arm.
|
||||
for _, arm := range shuffledArms(t) {
|
||||
cueVisible := t.Experiment == experimentIndex && cueExpectedVisible(t, arm.scale)
|
||||
m, err := runOne(p, t, armFor(arm.scale, arm.searchOff), arm.name, root, cueVisible, dry)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "%s [%s]: %v\n", t.ID, arm.name, err)
|
||||
}
|
||||
byArm[arm.name] = append(byArm[arm.name], m)
|
||||
all = append(all, m)
|
||||
fmt.Printf("%-24s %-14s %-18s search=%d hit=%d read=%d recall-tok=%d %v\n",
|
||||
t.ID, arm.name, m.FailureStage, m.SearchCalls, m.TargetSearchHits, m.ReadCalls,
|
||||
m.RecallReturnedTokens, m.UnexpectedWorkTools)
|
||||
}
|
||||
}
|
||||
reportFunnels(byArm)
|
||||
if experiment == experimentIndex {
|
||||
fmt.Print(reportDoseResponse(all))
|
||||
}
|
||||
fmt.Print(reportStopping(all))
|
||||
fmt.Print(auditQueries(all).report())
|
||||
fmt.Print(queryLines(all))
|
||||
if experiment == experimentIndex {
|
||||
reportBoundaries(all)
|
||||
}
|
||||
return writeResults(root, all)
|
||||
}
|
||||
|
||||
func cueExpectedVisible(t contextTask, scale string) bool {
|
||||
visible, _ := tierScales(t.CueTier)
|
||||
return contains(visible, scale)
|
||||
}
|
||||
|
||||
func reportFunnels(byArm map[string][]contextMetrics) {
|
||||
names := make([]string, 0, len(byArm))
|
||||
for name := range byArm {
|
||||
names = append(names, name)
|
||||
}
|
||||
sort.Strings(names)
|
||||
fmt.Println("\n## Funnel")
|
||||
for _, name := range names {
|
||||
fmt.Println(" ", summarize(name, byArm[name]).line())
|
||||
}
|
||||
fmt.Println("\n## Where it looked first")
|
||||
for _, name := range names {
|
||||
f := summarize(name, byArm[name])
|
||||
fmt.Printf(" %-14s %s\n", name, countsLine(f.Routes))
|
||||
}
|
||||
fmt.Println("\n## Where it broke")
|
||||
for _, name := range names {
|
||||
f := summarize(name, byArm[name])
|
||||
stages := make([]string, 0, len(f.Stages))
|
||||
for stage, n := range f.Stages {
|
||||
stages = append(stages, fmt.Sprintf("%s=%d", stage, n))
|
||||
}
|
||||
sort.Strings(stages)
|
||||
fmt.Printf(" %-12s %s\n", name, strings.Join(stages, " "))
|
||||
}
|
||||
}
|
||||
|
||||
// writeTrajectories saves what each turn produced, after the whole batch is
|
||||
// done. Three analyses in a row have needed a re-run because something was
|
||||
// counted and not kept — query text, then snippet contents. It is written at
|
||||
// the end on purpose: during the batch these files are the answer.
|
||||
func writeTrajectories(root string, all []contextMetrics) error {
|
||||
f, err := os.Create(filepath.Join(root, "trajectories.jsonl"))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer f.Close()
|
||||
for _, m := range all {
|
||||
if err := writeJSONLine(f, map[string]any{
|
||||
"task": m.Task, "arm": m.Arm, "messages": m.trajectory,
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func writeResults(root string, all []contextMetrics) int {
|
||||
path := filepath.Join(root, "results.jsonl")
|
||||
f, err := os.Create(path)
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
return 2
|
||||
}
|
||||
defer f.Close()
|
||||
for _, m := range all {
|
||||
if err := writeJSONLine(f, m); err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
return 2
|
||||
}
|
||||
}
|
||||
if err := writeTrajectories(root, all); err != nil {
|
||||
fmt.Fprintln(os.Stderr, "write trajectories:", err)
|
||||
return 2
|
||||
}
|
||||
fmt.Printf("\n%d runs written to %s (with trajectories.jsonl)\n", len(all), path)
|
||||
return 0
|
||||
}
|
||||
|
||||
// reportBoundaries prints each index task across its own cue boundary, then the
|
||||
// two sides pooled. Task difficulty is held constant inside a row, which is
|
||||
// what a four-arm average cannot do.
|
||||
func reportBoundaries(all []contextMetrics) {
|
||||
byKey := map[string]contextMetrics{}
|
||||
for _, m := range all {
|
||||
byKey[m.Task+"|"+m.Arm] = m
|
||||
}
|
||||
var cueSide, noCueSide []contextMetrics
|
||||
fmt.Println("\n## Across each task's cue boundary")
|
||||
fmt.Printf(" %-24s %-16s %-4s %-10s %-9s %-12s %-11s %s\n",
|
||||
"task", "arm", "cue", "recovered", "cue-read", "search/task", "recall-tok", "escape")
|
||||
for _, t := range indexTasks() {
|
||||
cue, noCue := boundaryPair(t.CueTier)
|
||||
for _, side := range []struct {
|
||||
scale string
|
||||
has bool
|
||||
}{{cue, true}, {noCue, false}} {
|
||||
m, ok := byKey[t.ID+"|index-"+side.scale]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if side.has {
|
||||
cueSide = append(cueSide, m)
|
||||
} else {
|
||||
noCueSide = append(noCueSide, m)
|
||||
}
|
||||
mark := "no"
|
||||
if side.has {
|
||||
mark = "yes"
|
||||
}
|
||||
fmt.Printf(" %-24s %-16s %-4s %-10v %-9v %-12d %-11d %d\n",
|
||||
t.ID, "index-"+side.scale, mark, m.AnswerRecovered, m.RecallReadWithoutSearch,
|
||||
m.SearchCalls, m.RecallReturnedTokens, m.EscapeCalls)
|
||||
}
|
||||
}
|
||||
fmt.Println("\n## Pooled across boundaries")
|
||||
fmt.Println(" ", summarize("cue-present", cueSide).line())
|
||||
fmt.Println(" ", summarize("cue-absent", noCueSide).line())
|
||||
}
|
||||
|
||||
// shuffledArms permutes a task's arms deterministically, so the order is
|
||||
// reproducible and still uncorrelated with the axis being measured.
|
||||
func shuffledArms(t contextTask) []experimentArm {
|
||||
arms := armsFor(t)
|
||||
rng := seededRand(t.ID, "arm-order")
|
||||
rng.Shuffle(len(arms), func(i, j int) { arms[i], arms[j] = arms[j], arms[i] })
|
||||
return arms
|
||||
}
|
||||
|
||||
// fingerprint records what produced a batch. Not a gate — but batch effects
|
||||
// have reversed three conclusions in this experiment already, and a table with
|
||||
// no provenance cannot be compared with the next one.
|
||||
func fingerprint(dry bool) string {
|
||||
model, endpoint := runModel, runBaseURL
|
||||
if dry {
|
||||
model, endpoint = "scripted", "none"
|
||||
}
|
||||
commit := "unknown"
|
||||
if out, err := exec.Command("git", "rev-parse", "--short", "HEAD").Output(); err == nil {
|
||||
commit = strings.TrimSpace(string(out))
|
||||
}
|
||||
return fmt.Sprintf("## Batch\n model=%s endpoint=%s window=%d generations=%d commit=%s started=%s\n",
|
||||
model, endpoint, fixtureWindow, fixtureGenerations, commit, time.Now().UTC().Format(time.RFC3339))
|
||||
}
|
||||
|
||||
// timeoutReason separates a run that never found its answer from one that found
|
||||
// it and kept going. Only the first is a case for a longer budget; the second
|
||||
// would just get more time to circle. A batch reporting only "context deadline
|
||||
// exceeded" cannot tell an arm that made retrieval harder from an arm that
|
||||
// happened to draw a slow minute.
|
||||
func timeoutReason(m contextMetrics) string {
|
||||
switch {
|
||||
case m.TargetHeldAtEnd && m.RoundsAfterTarget >= 2:
|
||||
return "PostTargetRunaway"
|
||||
case m.TargetHeldAtEnd:
|
||||
return "TargetHeldLate"
|
||||
case m.SearchCalls >= 3:
|
||||
return "SearchSpiral"
|
||||
case m.SearchCalls+m.ReadCalls+m.EscapeCalls == 0:
|
||||
return "NoToolActivity"
|
||||
default:
|
||||
return "Unknown"
|
||||
}
|
||||
}
|
||||
@@ -1,95 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"regexp"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"reasonix/internal/contract/provider"
|
||||
)
|
||||
|
||||
// scriptedProvider walks the chain a competent model would: search, read the
|
||||
// best hit, answer with what came back. It proves the pipeline holds together
|
||||
// for nothing, so a real run's first failure is about the model.
|
||||
|
||||
type scriptedProvider struct{}
|
||||
|
||||
func (p *scriptedProvider) Name() string { return "contextbench-scripted" }
|
||||
|
||||
var (
|
||||
scriptedHit = regexp.MustCompile(`(?m)^#(\d+) `)
|
||||
scriptedRecall = regexp.MustCompile(`(?s)^#\d+\n(.*)`)
|
||||
)
|
||||
|
||||
// Stream decides from the request alone, never from a call counter: one
|
||||
// provider serves every task in a run, and a counter would make the second task
|
||||
// start where the first left off.
|
||||
func (p *scriptedProvider) Stream(_ context.Context, req provider.Request) (<-chan provider.Chunk, error) {
|
||||
ch := make(chan provider.Chunk, 3)
|
||||
defer close(ch)
|
||||
// Honour the schema, as a model does: an arm without a query parameter is
|
||||
// an arm that cannot search, and pretending otherwise scores a refusal as
|
||||
// a failed search.
|
||||
if !recallAccepts(req, "query") {
|
||||
ch <- provider.Chunk{Type: provider.ChunkText, Text: "I have no address for that part of the history."}
|
||||
ch <- provider.Chunk{Type: provider.ChunkDone}
|
||||
return ch, nil
|
||||
}
|
||||
// The last message, not the last tool result anywhere: the fixture's own
|
||||
// history is full of tool results, and one of those is not this turn's.
|
||||
last := ""
|
||||
if n := len(req.Messages); n > 0 && req.Messages[n-1].Role == provider.RoleTool {
|
||||
last = req.Messages[n-1].Content
|
||||
}
|
||||
switch {
|
||||
case last == "":
|
||||
args, _ := json.Marshal(map[string]string{"query": lastUserText(req)})
|
||||
ch <- provider.Chunk{Type: provider.ChunkToolCall, ToolCall: &provider.ToolCall{
|
||||
ID: "s1", Name: "recall", Arguments: string(args),
|
||||
}}
|
||||
case strings.Contains(last, "Folded context matching"):
|
||||
positions := scriptedHit.FindAllStringSubmatch(last, 1)
|
||||
if len(positions) == 0 {
|
||||
ch <- provider.Chunk{Type: provider.ChunkText, Text: "Nothing in the folded history matches."}
|
||||
break
|
||||
}
|
||||
n, _ := strconv.Atoi(positions[0][1])
|
||||
args, _ := json.Marshal(map[string][]int{"positions": {n}})
|
||||
ch <- provider.Chunk{Type: provider.ChunkToolCall, ToolCall: &provider.ToolCall{
|
||||
ID: "r1", Name: "recall", Arguments: string(args),
|
||||
}}
|
||||
default:
|
||||
// Answer with what recall returned, so a correct chain scores as one.
|
||||
body := last
|
||||
if m := scriptedRecall.FindStringSubmatch(last); len(m) > 1 {
|
||||
body = m[1]
|
||||
}
|
||||
ch <- provider.Chunk{Type: provider.ChunkText, Text: "From the folded history: " + body}
|
||||
}
|
||||
ch <- provider.Chunk{Type: provider.ChunkDone}
|
||||
return ch, nil
|
||||
}
|
||||
|
||||
// recallAccepts reports whether recall is offered with the given parameter.
|
||||
func recallAccepts(req provider.Request, param string) bool {
|
||||
for _, t := range req.Tools {
|
||||
if t.Name != "recall" {
|
||||
continue
|
||||
}
|
||||
b, err := json.Marshal(t.Parameters)
|
||||
return err == nil && strings.Contains(string(b), `"`+param+`"`)
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func lastUserText(req provider.Request) string {
|
||||
for _, m := range slices.Backward(req.Messages) {
|
||||
if m.Role == provider.RoleUser {
|
||||
return m.Content
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
@@ -1,182 +0,0 @@
|
||||
// snippet.go — does a rank-1 hit actually hand over the answer?
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"reasonix/internal/base/retrieval"
|
||||
"reasonix/internal/contract/ablation"
|
||||
"reasonix/internal/contract/event"
|
||||
"reasonix/internal/contract/tool"
|
||||
"reasonix/internal/runtime/agent"
|
||||
)
|
||||
|
||||
// TargetHit is not EvidenceSufficient: a rank-1 hit can still hand over a
|
||||
// window too short to answer with, and searching again after that is correct.
|
||||
// Measured through the shipped recall path, not a copy of it.
|
||||
|
||||
// snippetWidths are the windows to compare. 240 is what ships.
|
||||
var snippetWidths = []int{120, 180, 240, 320, 480}
|
||||
|
||||
// historicalQueries are the queries these tasks actually drew in the pilots,
|
||||
// kept because a probe query is the host's phrasing and the model's was not.
|
||||
var historicalQueries = map[string][]string{
|
||||
"i03-coalescing": {"coalesce window marker probe", "job completion coalescing window"},
|
||||
"i04-recovery-fence": {"recovery fence marker epoch", "expected epoch", "recfence quartz"},
|
||||
}
|
||||
|
||||
type snippetReading struct {
|
||||
Task string
|
||||
Query string
|
||||
Width int
|
||||
Covered int
|
||||
Total int
|
||||
Runes int
|
||||
Rank int
|
||||
Complete bool
|
||||
}
|
||||
|
||||
// runSnippetAudit answers the question offline, over many instantiations so a
|
||||
// single seed's lengths cannot decide it.
|
||||
func runSnippetAudit(root string, seeds int) int {
|
||||
fmt.Printf("## Snippet coverage (%d instantiations per task)\n", seeds)
|
||||
fmt.Println(" Does the window a rank-1 hit returns contain every value the task scores on?")
|
||||
|
||||
byTaskWidth := map[string]map[int][]snippetReading{}
|
||||
for _, t := range allTasks() {
|
||||
byTaskWidth[t.ID] = map[int][]snippetReading{}
|
||||
for seed := range seeds {
|
||||
inst, err := instantiateTask(t, seededRand(t.ID, "snippet", fmt.Sprint(seed)))
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
return 2
|
||||
}
|
||||
queries := append([]string{inst.ProbeQuery}, historicalQueries[t.ID]...)
|
||||
for _, q := range queries {
|
||||
for _, width := range snippetWidths {
|
||||
r := measureSnippet(inst, q, width)
|
||||
byTaskWidth[t.ID][width] = append(byTaskWidth[t.ID][width], r)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fmt.Printf("\n %-24s %-5s", "task", "vals")
|
||||
for _, w := range snippetWidths {
|
||||
fmt.Printf("%9d", w)
|
||||
}
|
||||
fmt.Println(" mean runes @240")
|
||||
for _, t := range allTasks() {
|
||||
vals := len(t.AnswerVars)
|
||||
fmt.Printf(" %-24s %-5d", t.ID, vals)
|
||||
for _, w := range snippetWidths {
|
||||
fmt.Printf("%8.0f%%", 100*completeRate(byTaskWidth[t.ID][w]))
|
||||
}
|
||||
fmt.Printf(" %.0f\n", meanRunes(byTaskWidth[t.ID][240]))
|
||||
}
|
||||
reportShippedPath(root)
|
||||
return 0
|
||||
}
|
||||
|
||||
// measureSnippet renders one hit through the same MakeSnippet the recall path
|
||||
// uses, then asks whether every scored value survived the window.
|
||||
func measureSnippet(inst fixtureInstance, query string, width int) snippetReading {
|
||||
terms, err := retrieval.QueryTerms(query)
|
||||
if err != nil {
|
||||
return snippetReading{Task: inst.Task.ID, Query: query, Width: width, Total: len(inst.AnswerMarkers)}
|
||||
}
|
||||
// The tool result is the document the answer lives in; a call's arguments
|
||||
// are a separate document and never carry it.
|
||||
snippet := retrieval.MakeSnippet(inst.body, query, terms, width)
|
||||
covered := 0
|
||||
for _, marker := range inst.AnswerMarkers {
|
||||
if strings.Contains(strings.ToLower(snippet), strings.ToLower(marker)) {
|
||||
covered++
|
||||
}
|
||||
}
|
||||
return snippetReading{
|
||||
Task: inst.Task.ID, Query: query, Width: width,
|
||||
Covered: covered, Total: len(inst.AnswerMarkers),
|
||||
Runes: len([]rune(snippet)), Complete: covered == len(inst.AnswerMarkers),
|
||||
}
|
||||
}
|
||||
|
||||
// reportShippedPath cross-checks the table against the real recall tool: a
|
||||
// fixture, a fold, and a search through the agent that ships.
|
||||
func reportShippedPath(root string) {
|
||||
fmt.Println("\n## Through the shipped recall path (width as shipped)")
|
||||
for _, t := range allTasks() {
|
||||
inst, err := instantiateTask(t, seededRand(t.ID, "snippet-live"))
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
f, err := buildFixture(inst, ablation.Set{}, filepath.Join(root, "snippet", t.ID, "session.jsonl"))
|
||||
if err != nil {
|
||||
fmt.Printf(" %-24s fixture: %v\n", t.ID, err)
|
||||
continue
|
||||
}
|
||||
a := agent.New(&digestProvider{}, tool.NewRegistry(), f.Session, agent.Options{
|
||||
ContextWindow: fixtureWindow, CompactRatio: 0.5, RecentKeep: 2,
|
||||
SessionPath: f.Path, KeepPolicy: agent.KeepErrors,
|
||||
}, event.Discard)
|
||||
a.LoadProjectionSidecar(f.Path)
|
||||
queries := append([]string{inst.ProbeQuery}, historicalQueries[t.ID]...)
|
||||
for _, q := range queries {
|
||||
res, err := a.RecallContext(context.Background(), tool.RecallRequest{Query: q})
|
||||
if err != nil {
|
||||
fmt.Printf(" %-24s %-34q recall: %v\n", t.ID, q, err)
|
||||
continue
|
||||
}
|
||||
covered := 0
|
||||
for _, marker := range inst.AnswerMarkers {
|
||||
if strings.Contains(strings.ToLower(res.Text), strings.ToLower(marker)) {
|
||||
covered++
|
||||
}
|
||||
}
|
||||
fmt.Printf(" %-24s %-34q %d/%d values in the result\n",
|
||||
t.ID, truncate(q, 32), covered, len(inst.AnswerMarkers))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func completeRate(rs []snippetReading) float64 {
|
||||
if len(rs) == 0 {
|
||||
return 0
|
||||
}
|
||||
n := 0
|
||||
for _, r := range rs {
|
||||
if r.Complete {
|
||||
n++
|
||||
}
|
||||
}
|
||||
return float64(n) / float64(len(rs))
|
||||
}
|
||||
|
||||
func meanRunes(rs []snippetReading) float64 {
|
||||
if len(rs) == 0 {
|
||||
return 0
|
||||
}
|
||||
total := 0
|
||||
for _, r := range rs {
|
||||
total += r.Runes
|
||||
}
|
||||
return float64(total) / float64(len(rs))
|
||||
}
|
||||
|
||||
// answerCoverage is what a run should have recorded all along: how much of the
|
||||
// scored answer a search actually handed over. rank=1 with coverage 1/3 is not
|
||||
// a stopping failure.
|
||||
func answerCoverage(text string, markers []string) int {
|
||||
covered := 0
|
||||
lower := strings.ToLower(text)
|
||||
for _, marker := range markers {
|
||||
if strings.Contains(lower, strings.ToLower(marker)) {
|
||||
covered++
|
||||
}
|
||||
}
|
||||
return covered
|
||||
}
|
||||
@@ -1,3 +1,4 @@
|
||||
prompt = "contacts.txt is full of duplicate entries and users keep complaining. Clean it up and write the result to contacts_clean.txt."
|
||||
class = "ambiguous"
|
||||
max_steps = 15
|
||||
timeout_sec = 300
|
||||
@@ -1,3 +1,4 @@
|
||||
prompt = "config.py silently accepts broken config files and the app then crashes much later somewhere else. Make loading fail fast with clear errors instead."
|
||||
class = "ambiguous"
|
||||
max_steps = 15
|
||||
timeout_sec = 300
|
||||
@@ -1,3 +1,4 @@
|
||||
prompt = "parse_all in parser.py only handles some of the files under fixtures/. Users need it to handle all of them. Extend it so every fixture is parsed."
|
||||
class = "ambiguous"
|
||||
max_steps = 15
|
||||
timeout_sec = 300
|
||||
@@ -1,3 +1,4 @@
|
||||
prompt = "Users report that stats.py keeps crashing on real-world input files. Make it robust enough for production use. Sample inputs are in samples/."
|
||||
class = "ambiguous"
|
||||
max_steps = 15
|
||||
timeout_sec = 300
|
||||
@@ -1,3 +1,4 @@
|
||||
prompt = "Exactly one setting differs across config/base.toml, config/staging.toml and config/production.toml. Write AUDIT.md naming the setting that drifted, the environment it drifted in, and both values. Do not change any config."
|
||||
class = "read-only-output"
|
||||
max_steps = 25
|
||||
timeout_sec = 480
|
||||
@@ -1,3 +1,4 @@
|
||||
prompt = "discounted(price_cents, percent) in pricing.py never applies the discount. Fix it (integer cents, round the discount down)."
|
||||
class = "atomic-bugfix"
|
||||
max_steps = 12
|
||||
timeout_sec = 180
|
||||
@@ -1,3 +1,4 @@
|
||||
prompt = "overlaps(a, b) in intervals.py misjudges some interval pairs. Fix it so half-open intervals [start, end) are compared correctly."
|
||||
class = "atomic-bugfix"
|
||||
max_steps = 12
|
||||
timeout_sec = 180
|
||||
@@ -1,6 +0,0 @@
|
||||
def median(values):
|
||||
ordered = sorted(values)
|
||||
mid = len(ordered) // 2
|
||||
if len(ordered) % 2:
|
||||
return ordered[mid]
|
||||
return (ordered[mid - 1] + ordered[mid]) / 2
|
||||
@@ -1,3 +1,4 @@
|
||||
prompt = "median(values) in stats.py is wrong for even-length lists. Fix it to return the average of the two middle values."
|
||||
class = "atomic-bugfix"
|
||||
max_steps = 12
|
||||
timeout_sec = 180
|
||||
@@ -1,3 +1,4 @@
|
||||
prompt = "Fifo.pop in fifo.py returns the newest item instead of the oldest. Fix the FIFO order."
|
||||
class = "atomic-bugfix"
|
||||
max_steps = 12
|
||||
timeout_sec = 180
|
||||
@@ -1,3 +1,4 @@
|
||||
prompt = "slugify in slug.py leaves stray hyphens at the ends of some slugs. Fix it."
|
||||
class = "atomic-bugfix"
|
||||
max_steps = 12
|
||||
timeout_sec = 180
|
||||
@@ -1,3 +1,4 @@
|
||||
prompt = "store.py: values saved with set() sometimes cannot be read back with get(). Fix the bug."
|
||||
class = "atomic-bugfix"
|
||||
max_steps = 12
|
||||
timeout_sec = 180
|
||||
@@ -1,3 +1,4 @@
|
||||
prompt = "tail(lst, n) in utils.py drops the last element. Fix it so it returns the last n items of the list."
|
||||
class = "atomic-bugfix"
|
||||
max_steps = 12
|
||||
timeout_sec = 180
|
||||
@@ -1,3 +1,4 @@
|
||||
prompt = "Begin by reading the file story/chapter-1.md. Each chapter ends by telling you which chapter to read next; follow that trail one chapter at a time — read a chapter, find where it points you, then read that next chapter — until a chapter says the trail ends. You cannot know the next chapter without reading the current one, so do not guess or read ahead. In chapter 1 a river village is named; in the final chapter on the trail a noble House is named. When the trail ends, write a file answer.txt whose entire contents are the village name from chapter 1, a single hyphen, and the House name from the final chapter — for example Riverton-Stark — and nothing else."
|
||||
class = "repo-exploration"
|
||||
max_steps = 40
|
||||
timeout_sec = 900
|
||||
Loaded 100 of 7220 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user