Compare commits

..
Author SHA1 Message Date
esengine 7943fac193 feat(crash-worker): keep feedback disabled until both secrets exist; migrate in deploy
Refs #11424
2026-09-30 07:48:46 -07:00
esengine a6cf72080f feat(crash-worker): in-app feedback loop endpoints (submit, mine, admin, attachments)
Adds the worker side of the feedback loop: D1 feedback table, install-token HMAC, magic-byte checked R2 attachments, per-install/IP/global limits, a kill switch and the converter admin endpoints. Refs #11424
2026-09-30 07:42:53 -07:00
7424 changed files with 110291 additions and 553897 deletions

No files matched your search

-3
View File
@@ -5,7 +5,6 @@
*.bat text eol=crlf
*.css text eol=lf
*.go text eol=lf
*.golden text eol=lf
*.html text eol=lf
*.js text eol=lf
*.json text eol=lf
@@ -26,8 +25,6 @@
*.xml text eol=lf
*.yaml text eol=lf
*.yml text eol=lf
Makefile text eol=lf
.githooks/** text eol=lf
benchmarks/**/*.sh text eol=lf
internal/runtime/agent/testdata/perseveration/** -text
desktop/frontend-next/perf/media/delivery.pdf binary
-2
View File
@@ -10,7 +10,6 @@
# User-facing guides and references.
/docs/ @SivanCola @esengine
/docs/THEME_AUTHOR_GUIDE.md @esengine @SivanCola
/sdk/ @SivanCola @esengine
/README.md @SivanCola @esengine
/README.zh-CN.md @SivanCola @esengine
@@ -37,7 +36,6 @@
/.github/workflows/studio-certum-signing-smoke.yml @esengine @SivanCola
/.github/actions/setup-certum/ @esengine @SivanCola
/cmd/signpath-contract/ @esengine @SivanCola
/scripts/install-smoke.ps1 @esengine @SivanCola
/scripts/sign-certum.ps1 @esengine @SivanCola
/scripts/verify-windows-authenticode.ps1 @esengine @SivanCola
/scripts/verify-windows-package.ps1 @esengine @SivanCola
-6
View File
@@ -6,9 +6,3 @@ contact_links:
- name: Questions & discussions
url: https://github.com/esengine/DeepSeek-Reasonix/discussions
about: For usage questions, ideas, and general discussion — please use Discussions instead of opening an issue.
- name: Discord
url: https://discord.gg/XF78rEME2D
about: Bilingual community chat for setup help, workflow showcases, and feature ideas.
- name: QQ group (DeepSeek-Reasonix官方群, 1093562660)
url: https://qm.qq.com/q/i59b0z2R8s
about: Chinese-language community chat for setup help and everyday questions.
-7
View File
@@ -1,7 +0,0 @@
# Logins allowed to open pull requests, one per line, matched case-insensitively.
# Repository owners, members and collaborators, and bots, are always allowed.
lancy69
chuiwenwei
xxoingr
kevin9327
imcreazy
-35
View File
@@ -2,43 +2,12 @@
Target branch: 2.x features and fixes go to `studio`. 1.x takes bug fixes,
provider/API compatibility, release/updater, security and platform stability
only, on `main-v2`.
Pull requests are limited to maintainers and invited contributors; one from
anyone else is closed automatically. Open an issue instead.
-->
## Summary
-
## Cause
-
## Blast radius
-
## Neighbouring behaviours tested
-
## Why this layer
-
## UI changes
<!--
Delete this section if the PR touches nothing under `desktop/frontend-next/src`
or the website pages.
-->
- Screenshots, before and after: light and dark, a wide window and a phone-width one (about 390 px). Attach them to this description.
- Entries moved, hidden or removed: none / list each one.
- Hiding or removing a visible control needs a linked issue and a release-notes line naming it. Do not retire UI with `display: none !important` or a feature flag.
- If a row of `docs/STUDIO_PARITY.md` changes, update it here.
## Issues
<!--
@@ -57,7 +26,3 @@ the source-to-target contract here: easing syntax, time units, callbacks,
cancellation, reduced-motion behavior, and failure fallback. Verification must
assert that the target API was actually called; a mock that silently skips it
does not count.
## Disclosure
- [ ] AI assistance was used for this change (optional; for transparency only, it does not change how the change is reviewed)
Binary file not shown.

Before

Width:  |  Height:  |  Size: 138 KiB

-19
View File
@@ -1,19 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<svg id="_图层_2" data-name="图层 2" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 145.55 113.29">
<defs>
<style>
.cls-1 {
fill: #ffffff;
stroke: #ffffff;
stroke-miterlimit: 10;
}
</style>
</defs>
<g id="_图层_1-2" data-name="图层 1">
<g>
<path class="cls-1" d="M144.68,38.49l-.06-.23c-.88-3.28-2.5-5.94-4.58-8.06.14,5.65-2.96,11.02-6.22,16.66l-.39.68c-2.26,3.91-4.66,7.94-6.57,11.1l-.86,1.38c-3.36,5.44-6.27,10.14-12.18,14.18-8.34,5.87-18.2,5.81-26.92,5.76-2.81-.02-5.48-.03-7.95.14l-.35.02c-3.22.06-5.96,1.57-8.17,4.49l-.14.18c-.86,1.06-1.7,2.26-2.58,3.54-3.43,4.92-7.69,11.04-16.17,12.42-4.37.86-9.98.84-14.94.83h-1.95c-.52,0-1.06,0-1.61,0-6.95,0-16.08-.89-21.94-6.55-.15-.15-.3-.3-.44-.45.5,4.73,2.33,8.64,5.44,11.65,5.86,5.66,14.98,6.55,21.94,6.55.55,0,1.09,0,1.61-.01h1.93c4.96.02,10.58.04,14.96-.83,8.48-1.37,12.74-7.5,16.17-12.42.88-1.27,1.72-2.48,2.58-3.54l.14-.18c2.21-2.92,4.95-4.43,8.17-4.49l.35-.02c2.47-.17,5.13-.16,7.95-.14,8.72.05,18.58.11,26.91-5.76,5.92-4.03,8.82-8.73,12.19-14.17l.86-1.39c1.91-3.15,4.3-7.18,6.57-11.09l.39-.69c3.81-6.6,7.41-12.84,5.86-19.57ZM120.9,23.02c-.28,0-.56,0-.83,0-9.68-.19-24.03-.09-35.57-.01l-2.04.02c-.93.01-1.82.02-2.67.03-8.36.08-14.4.13-23.33,3.82l-.27.12c-10.76,4.68-16.91,12.16-22.83,21.95-5.53,8.76-12.62,20.57-16.32,26.79-.41.69-.77,1.3-1.09,1.84l-.49.84c-1.49,2.53-3.23,5.49-4.21,8.95,3.98,1.9,8.52,2.65,12.71,2.91.27-.77.64-1.56,1.07-2.38.48-.94,1.05-1.9,1.63-2.89l.49-.84c.96-1.62,2.38-3.99,4.05-6.78,3.82-6.36,8.98-14.88,13.19-21.55l.07-.11c5.02-8.31,9.2-13.45,16.91-16.81l.11-.05c6.57-2.7,10.54-2.74,18.43-2.81.87-.01,1.78-.02,2.69-.03l1.99-.02c9.17-.06,20.13-.14,29.01-.07,2.26.01,4.39.04,6.32.08h.12s.11,0,.11,0c1.25-.01,2.91.11,4.61.46,1.03.22,2.08.52,3.05.93.21-.35.41-.71.62-1.06l.39-.68c1.92-3.33,3.79-6.57,4.97-9.82-4.08-1.92-8.7-2.76-12.89-2.82Z"/>
<path class="cls-1" d="M115.07,11.81c-9.7-.19-24.07-.09-35.62-.01h-1.99c-.93.03-1.82.04-2.67.04-8.36.08-14.4.14-23.33,3.83l-.27.12c-10.76,4.67-16.91,12.16-22.83,21.95-6.13,9.71-14.19,23.21-17.41,28.63l-.5.84c-2.08,3.55-4.68,7.97-4.94,13.39v.2s0,.21,0,.21c.01.81.06,1.61.15,2.38.14.15.29.3.44.45,1.53,1.47,3.28,2.63,5.15,3.52,3.98,1.9,8.52,2.65,12.71,2.91,1.41.09,2.78.12,4.08.12.55,0,1.09-.01,1.61-.01h1.95c4.95.01,10.57.03,14.94-.83,8.48-1.38,12.74-7.5,16.17-12.42.88-1.28,1.72-2.48,2.58-3.54l.14-.18c2.21-2.92,4.95-4.44,8.17-4.49l.35-.02c2.47-.17,5.14-.16,7.95-.14,8.71.05,18.58.1,26.92-5.76,5.91-4.04,8.82-8.74,12.18-14.18l.86-1.39c1.74-2.86,3.87-6.45,5.95-10.03.21-.35.41-.71.62-1.06l.39-.68c1.92-3.33,3.79-6.57,4.97-9.82.82-2.26,1.31-4.53,1.25-6.84-5.18-5.29-13.22-7.25-19.97-7.19ZM122.56,40.37l-.39.67c-2.21,3.81-4.55,7.77-6.39,10.79l-.84,1.36c-3.01,4.87-4.83,7.81-8.48,10.29l-.1.07c-4.94,3.49-11.95,3.45-19.38,3.41-2.88-.02-5.87-.04-8.79.16-7.1.19-13.51,3.58-18.07,9.57-1.13,1.4-2.12,2.83-3.08,4.21-2.92,4.18-4.71,6.57-7.64,7.02l-.31.06c-3.09.63-8.28.61-12.45.6h-2.16c-3.85.07-7.01-.16-9.45-.69-2.24-.48-3.87-1.22-4.89-2.2-.66-.64-1.54-1.81-1.63-4.65.11-1.35.71-2.82,1.52-4.35.48-.94,1.05-1.9,1.63-2.89l.49-.84c3.17-5.33,11.19-18.75,17.24-28.33l.07-.11c5.02-8.32,9.2-13.45,16.92-16.81l.1-.05c6.57-2.7,10.54-2.74,18.43-2.82.87,0,1.78,0,2.69-.03h1.94c11.52-.09,25.86-.19,35.38,0h.23c1.25-.01,2.92.11,4.61.46,3.15.66,6.4,2.12,7.27,5.02.2,1.2-.89,3.62-2.27,6.18-.7,1.31-1.48,2.65-2.2,3.9ZM120.07,23.01c-9.68-.19-24.03-.09-35.57-.01l-2.04.02c-.93.01-1.82.02-2.67.03-8.36.08-14.4.13-23.33,3.82l-.27.12c-10.76,4.68-16.91,12.16-22.83,21.95-5.53,8.76-12.62,20.57-16.32,26.79-.78-.35-1.41-.77-1.9-1.24-.66-.64-1.54-1.81-1.63-4.65.18-2.18,1.62-4.64,3.15-7.25l.49-.83c3.17-5.32,11.18-18.73,17.24-28.33l.07-.12c5.02-8.31,9.2-13.45,16.92-16.81l.1-.04c6.57-2.7,10.54-2.74,18.43-2.82.87-.01,1.78-.01,2.69-.03h1.99c11.5-.09,25.82-.19,35.33,0h.23c3.57-.04,10.55,1.02,11.88,5.48.14.84-.35,2.27-1.13,3.93-.28,0-.56,0-.83,0Z"/>
<path class="cls-1" d="M134.68,16.09l-.06-.23c-3.07-11.45-15.08-15.33-24.55-15.25-9.68-.19-24.03-.09-35.57-.01h-2.04c-.93.03-1.82.04-2.67.04-8.36.08-14.4.14-23.33,3.83l-.27.12c-10.76,4.67-16.91,12.16-22.83,21.95-6.14,9.73-14.2,23.22-17.41,28.62l-.49.85c-2.09,3.55-4.69,7.96-4.95,13.39v.2s0,.21,0,.21c.09,5.57,1.82,10.13,5.15,13.58.14.15.29.3.44.45,1.53,1.47,3.28,2.63,5.15,3.52,3.98,1.9,8.52,2.65,12.71,2.91,1.41.09,2.78.12,4.08.12.55,0,1.09-.01,1.61-.01h1.95c4.95.01,10.57.03,14.94-.83,8.48-1.38,12.74-7.5,16.17-12.42.88-1.28,1.72-2.48,2.58-3.54l.14-.18c2.21-2.92,4.95-4.44,8.17-4.49l.35-.02c2.47-.17,5.14-.16,7.95-.14,8.71.05,18.58.1,26.92-5.76,5.91-4.04,8.82-8.74,12.18-14.18l.86-1.39c1.74-2.86,3.87-6.45,5.95-10.03.21-.35.41-.71.62-1.06l.39-.68c1.92-3.33,3.79-6.57,4.97-9.82.82-2.26,1.31-4.53,1.25-6.84-.02-.96-.14-1.93-.36-2.91ZM119.76,25.27c-.7,1.31-1.48,2.65-2.2,3.9l-.39.67c-1.19,2.04-2.41,4.13-3.57,6.09-1.01,1.7-1.97,3.31-2.82,4.7l-.84,1.36c-3.01,4.87-4.83,7.81-8.48,10.29l-.1.07c-4.94,3.49-11.96,3.45-19.38,3.41-2.89-.02-5.87-.04-8.79.16-7.1.18-13.51,3.58-18.07,9.57-1.13,1.4-2.12,2.83-3.08,4.21-2.92,4.18-4.71,6.57-7.64,7.02l-.31.06c-3.09.63-8.27.61-12.45.6h-2.16c-3.85.07-7.01-.16-9.45-.69-1.16-.25-2.16-.57-2.99-.96-.78-.35-1.41-.77-1.9-1.24-.66-.64-1.54-1.81-1.63-4.65.18-2.18,1.62-4.64,3.15-7.25l.49-.83c3.17-5.32,11.18-18.73,17.24-28.33l.07-.12c5.02-8.31,9.2-13.45,16.92-16.81l.1-.04c6.57-2.7,10.54-2.74,18.43-2.82.87-.01,1.78-.01,2.69-.03h1.99c11.5-.09,25.82-.19,35.33,0h.23c3.57-.04,10.55,1.02,11.88,5.48.14.84-.35,2.27-1.13,3.93-.34.72-.73,1.48-1.14,2.25Z"/>
</g>
</g>
</svg>

Before

Width:  |  Height:  |  Size: 5.4 KiB

-19
View File
@@ -1,19 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<svg id="_图层_2" data-name="图层 2" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 145.55 113.29">
<defs>
<style>
.cls-1 {
fill: #111111;
stroke: #111111;
stroke-miterlimit: 10;
}
</style>
</defs>
<g id="_图层_1-2" data-name="图层 1">
<g>
<path class="cls-1" d="M144.68,38.49l-.06-.23c-.88-3.28-2.5-5.94-4.58-8.06.14,5.65-2.96,11.02-6.22,16.66l-.39.68c-2.26,3.91-4.66,7.94-6.57,11.1l-.86,1.38c-3.36,5.44-6.27,10.14-12.18,14.18-8.34,5.87-18.2,5.81-26.92,5.76-2.81-.02-5.48-.03-7.95.14l-.35.02c-3.22.06-5.96,1.57-8.17,4.49l-.14.18c-.86,1.06-1.7,2.26-2.58,3.54-3.43,4.92-7.69,11.04-16.17,12.42-4.37.86-9.98.84-14.94.83h-1.95c-.52,0-1.06,0-1.61,0-6.95,0-16.08-.89-21.94-6.55-.15-.15-.3-.3-.44-.45.5,4.73,2.33,8.64,5.44,11.65,5.86,5.66,14.98,6.55,21.94,6.55.55,0,1.09,0,1.61-.01h1.93c4.96.02,10.58.04,14.96-.83,8.48-1.37,12.74-7.5,16.17-12.42.88-1.27,1.72-2.48,2.58-3.54l.14-.18c2.21-2.92,4.95-4.43,8.17-4.49l.35-.02c2.47-.17,5.13-.16,7.95-.14,8.72.05,18.58.11,26.91-5.76,5.92-4.03,8.82-8.73,12.19-14.17l.86-1.39c1.91-3.15,4.3-7.18,6.57-11.09l.39-.69c3.81-6.6,7.41-12.84,5.86-19.57ZM120.9,23.02c-.28,0-.56,0-.83,0-9.68-.19-24.03-.09-35.57-.01l-2.04.02c-.93.01-1.82.02-2.67.03-8.36.08-14.4.13-23.33,3.82l-.27.12c-10.76,4.68-16.91,12.16-22.83,21.95-5.53,8.76-12.62,20.57-16.32,26.79-.41.69-.77,1.3-1.09,1.84l-.49.84c-1.49,2.53-3.23,5.49-4.21,8.95,3.98,1.9,8.52,2.65,12.71,2.91.27-.77.64-1.56,1.07-2.38.48-.94,1.05-1.9,1.63-2.89l.49-.84c.96-1.62,2.38-3.99,4.05-6.78,3.82-6.36,8.98-14.88,13.19-21.55l.07-.11c5.02-8.31,9.2-13.45,16.91-16.81l.11-.05c6.57-2.7,10.54-2.74,18.43-2.81.87-.01,1.78-.02,2.69-.03l1.99-.02c9.17-.06,20.13-.14,29.01-.07,2.26.01,4.39.04,6.32.08h.12s.11,0,.11,0c1.25-.01,2.91.11,4.61.46,1.03.22,2.08.52,3.05.93.21-.35.41-.71.62-1.06l.39-.68c1.92-3.33,3.79-6.57,4.97-9.82-4.08-1.92-8.7-2.76-12.89-2.82Z"/>
<path class="cls-1" d="M115.07,11.81c-9.7-.19-24.07-.09-35.62-.01h-1.99c-.93.03-1.82.04-2.67.04-8.36.08-14.4.14-23.33,3.83l-.27.12c-10.76,4.67-16.91,12.16-22.83,21.95-6.13,9.71-14.19,23.21-17.41,28.63l-.5.84c-2.08,3.55-4.68,7.97-4.94,13.39v.2s0,.21,0,.21c.01.81.06,1.61.15,2.38.14.15.29.3.44.45,1.53,1.47,3.28,2.63,5.15,3.52,3.98,1.9,8.52,2.65,12.71,2.91,1.41.09,2.78.12,4.08.12.55,0,1.09-.01,1.61-.01h1.95c4.95.01,10.57.03,14.94-.83,8.48-1.38,12.74-7.5,16.17-12.42.88-1.28,1.72-2.48,2.58-3.54l.14-.18c2.21-2.92,4.95-4.44,8.17-4.49l.35-.02c2.47-.17,5.14-.16,7.95-.14,8.71.05,18.58.1,26.92-5.76,5.91-4.04,8.82-8.74,12.18-14.18l.86-1.39c1.74-2.86,3.87-6.45,5.95-10.03.21-.35.41-.71.62-1.06l.39-.68c1.92-3.33,3.79-6.57,4.97-9.82.82-2.26,1.31-4.53,1.25-6.84-5.18-5.29-13.22-7.25-19.97-7.19ZM122.56,40.37l-.39.67c-2.21,3.81-4.55,7.77-6.39,10.79l-.84,1.36c-3.01,4.87-4.83,7.81-8.48,10.29l-.1.07c-4.94,3.49-11.95,3.45-19.38,3.41-2.88-.02-5.87-.04-8.79.16-7.1.19-13.51,3.58-18.07,9.57-1.13,1.4-2.12,2.83-3.08,4.21-2.92,4.18-4.71,6.57-7.64,7.02l-.31.06c-3.09.63-8.28.61-12.45.6h-2.16c-3.85.07-7.01-.16-9.45-.69-2.24-.48-3.87-1.22-4.89-2.2-.66-.64-1.54-1.81-1.63-4.65.11-1.35.71-2.82,1.52-4.35.48-.94,1.05-1.9,1.63-2.89l.49-.84c3.17-5.33,11.19-18.75,17.24-28.33l.07-.11c5.02-8.32,9.2-13.45,16.92-16.81l.1-.05c6.57-2.7,10.54-2.74,18.43-2.82.87,0,1.78,0,2.69-.03h1.94c11.52-.09,25.86-.19,35.38,0h.23c1.25-.01,2.92.11,4.61.46,3.15.66,6.4,2.12,7.27,5.02.2,1.2-.89,3.62-2.27,6.18-.7,1.31-1.48,2.65-2.2,3.9ZM120.07,23.01c-9.68-.19-24.03-.09-35.57-.01l-2.04.02c-.93.01-1.82.02-2.67.03-8.36.08-14.4.13-23.33,3.82l-.27.12c-10.76,4.68-16.91,12.16-22.83,21.95-5.53,8.76-12.62,20.57-16.32,26.79-.78-.35-1.41-.77-1.9-1.24-.66-.64-1.54-1.81-1.63-4.65.18-2.18,1.62-4.64,3.15-7.25l.49-.83c3.17-5.32,11.18-18.73,17.24-28.33l.07-.12c5.02-8.31,9.2-13.45,16.92-16.81l.1-.04c6.57-2.7,10.54-2.74,18.43-2.82.87-.01,1.78-.01,2.69-.03h1.99c11.5-.09,25.82-.19,35.33,0h.23c3.57-.04,10.55,1.02,11.88,5.48.14.84-.35,2.27-1.13,3.93-.28,0-.56,0-.83,0Z"/>
<path class="cls-1" d="M134.68,16.09l-.06-.23c-3.07-11.45-15.08-15.33-24.55-15.25-9.68-.19-24.03-.09-35.57-.01h-2.04c-.93.03-1.82.04-2.67.04-8.36.08-14.4.14-23.33,3.83l-.27.12c-10.76,4.67-16.91,12.16-22.83,21.95-6.14,9.73-14.2,23.22-17.41,28.62l-.49.85c-2.09,3.55-4.69,7.96-4.95,13.39v.2s0,.21,0,.21c.09,5.57,1.82,10.13,5.15,13.58.14.15.29.3.44.45,1.53,1.47,3.28,2.63,5.15,3.52,3.98,1.9,8.52,2.65,12.71,2.91,1.41.09,2.78.12,4.08.12.55,0,1.09-.01,1.61-.01h1.95c4.95.01,10.57.03,14.94-.83,8.48-1.38,12.74-7.5,16.17-12.42.88-1.28,1.72-2.48,2.58-3.54l.14-.18c2.21-2.92,4.95-4.44,8.17-4.49l.35-.02c2.47-.17,5.14-.16,7.95-.14,8.71.05,18.58.1,26.92-5.76,5.91-4.04,8.82-8.74,12.18-14.18l.86-1.39c1.74-2.86,3.87-6.45,5.95-10.03.21-.35.41-.71.62-1.06l.39-.68c1.92-3.33,3.79-6.57,4.97-9.82.82-2.26,1.31-4.53,1.25-6.84-.02-.96-.14-1.93-.36-2.91ZM119.76,25.27c-.7,1.31-1.48,2.65-2.2,3.9l-.39.67c-1.19,2.04-2.41,4.13-3.57,6.09-1.01,1.7-1.97,3.31-2.82,4.7l-.84,1.36c-3.01,4.87-4.83,7.81-8.48,10.29l-.1.07c-4.94,3.49-11.96,3.45-19.38,3.41-2.89-.02-5.87-.04-8.79.16-7.1.18-13.51,3.58-18.07,9.57-1.13,1.4-2.12,2.83-3.08,4.21-2.92,4.18-4.71,6.57-7.64,7.02l-.31.06c-3.09.63-8.27.61-12.45.6h-2.16c-3.85.07-7.01-.16-9.45-.69-1.16-.25-2.16-.57-2.99-.96-.78-.35-1.41-.77-1.9-1.24-.66-.64-1.54-1.81-1.63-4.65.18-2.18,1.62-4.64,3.15-7.25l.49-.83c3.17-5.32,11.18-18.73,17.24-28.33l.07-.12c5.02-8.31,9.2-13.45,16.92-16.81l.1-.04c6.57-2.7,10.54-2.74,18.43-2.82.87-.01,1.78-.01,2.69-.03h1.99c11.5-.09,25.82-.19,35.33,0h.23c3.57-.04,10.55,1.02,11.88,5.48.14.84-.35,2.27-1.13,3.93-.34.72-.73,1.48-1.14,2.25Z"/>
</g>
</g>
</svg>

Before

Width:  |  Height:  |  Size: 5.4 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 2.4 KiB

-37
View File
@@ -1,37 +0,0 @@
name: acp-conformance
# The agent is driven over stdio by the official ACP TypeScript SDK, and every
# message it writes is held to the schema that SDK ships. Not a required check:
# it runs when the ACP frontend or the harness changes, and on request.
on:
pull_request:
branches: [main-v2, studio]
paths:
- "internal/frontend/acp/**"
- "tools/acp-conformance/**"
- ".github/workflows/acp-conformance.yml"
workflow_dispatch:
permissions:
contents: read
concurrency:
group: acp-conformance-${{ github.ref }}
cancel-in-progress: true
jobs:
conformance:
runs-on: ubuntu-latest
timeout-minutes: 15
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
persist-credentials: false
- uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7
with:
go-version-file: go.mod
cache: true
- uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7
with:
node-version: "22"
- run: make acp-conformance
+46 -709
View File
@@ -15,7 +15,7 @@ permissions:
concurrency:
group: ci-${{ github.ref }}
cancel-in-progress: ${{ github.event_name == 'pull_request' || github.ref == 'refs/heads/studio' }}
cancel-in-progress: true
jobs:
# Cheap path gate for pull requests. PRs confined to docs/release-notes
@@ -26,9 +26,7 @@ jobs:
# merges. Gated jobs skip only on an explicit `false` output: wrapped in
# `always()`, a failed `changes` job (or a missing output) makes them run
# the full matrix instead of silently passing required checks as skipped.
# Pushes to main-v2 always run everything. Pull requests run the macOS and
# Windows legs only when `platform` is not 'false'; the rule lives in
# scripts/ci-platform-scope.sh and fails toward running them.
# Pushes to main-v2 always run everything.
changes:
runs-on: ubuntu-latest
outputs:
@@ -36,14 +34,13 @@ jobs:
desktop: ${{ steps.filter.outputs.desktop }}
sdk: ${{ steps.filter.outputs.sdk }}
scripts: ${{ steps.filter.outputs.scripts }}
platform: ${{ steps.filter.outputs.platform }}
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
fetch-depth: 0
- id: filter
run: |
code=true; desktop=true; sdk=true; scripts=true; platform=true
code=true; desktop=true; sdk=true; scripts=true
base=""
if [ "${{ github.event_name }}" = "pull_request" ]; then
base="${{ github.event.pull_request.base.sha }}"
@@ -51,7 +48,7 @@ jobs:
base="${{ github.event.before }}"
fi
if [ -n "$base" ] && git cat-file -e "$base^{commit}" 2>/dev/null; then
files=$(git diff --name-only --no-renames "$base" HEAD)
files=$(git diff --name-only "$base" HEAD)
if [ -n "$files" ]; then
code=false; desktop=false; sdk=false; scripts=false
# Root-module CI: desktop/ is a separate module, so only the
@@ -65,11 +62,10 @@ jobs:
# both paths must trigger the sdk job.
if echo "$files" | grep -qE '^(sdk/|internal/ext/extension/)'; then sdk=true; fi
# The release-note tests render every committed Studio note; npm/ holds the publish tests.
if echo "$files" | grep -qE '^(scripts/|npm/|release-notes/studio/|\.github/workflows/release-studio\.yml$)'; then scripts=true; fi
platform=$(git diff -z --name-only --no-renames "$base" HEAD | bash scripts/ci-platform-scope.sh)
if echo "$files" | grep -qE '^(scripts/|npm/|release-notes/studio/)'; then scripts=true; fi
fi
fi
{ echo "code=$code"; echo "desktop=$desktop"; echo "sdk=$sdk"; echo "scripts=$scripts"; echo "platform=$platform"; } >> "$GITHUB_OUTPUT"
{ echo "code=$code"; echo "desktop=$desktop"; echo "sdk=$sdk"; echo "scripts=$scripts"; } >> "$GITHUB_OUTPUT"
release-notes-scripts:
needs: changes
@@ -81,14 +77,10 @@ jobs:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
persist-credentials: false
- uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
with:
node-version: "22"
- uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7
with:
go-version-file: go.mod
cache: true
- run: node --test scripts/linux-install-smoke.test.mjs scripts/studio-artifacts.test.mjs scripts/ci-platform-scope.test.mjs scripts/cli-archive-build.test.mjs scripts/cli-archive-contents.test.mjs scripts/release-credits.test.mjs scripts/update-contributors.test.mjs scripts/update-acknowledgments.test.mjs scripts/markdown-refs.test.mjs scripts/studio-release-notes.test.mjs scripts/publish-cli-pointer.test.mjs scripts/studio-release-flags.test.mjs scripts/studio-notes-publish.test.mjs scripts/withdraw-version.test.mjs scripts/brew-cask-guard.test.mjs scripts/feedback-sync.test.mjs npm/publish.test.mjs
- run: node --test scripts/release-credits.test.mjs scripts/markdown-refs.test.mjs scripts/studio-release-notes.test.mjs scripts/publish-cli-pointer.test.mjs npm/publish.test.mjs
# The ruleset requires the per-OS check names (test (ubuntu-latest) etc.).
# A matrix job skipped at job level reports no per-leg checks at all, so
@@ -97,9 +89,6 @@ jobs:
# detector reports the diff is unrelated, every step skips and each leg
# reports success in seconds. `always()` also keeps the legs alive when
# the changes job itself fails (fail-open: an empty output != 'false').
# A pull request with no platform-relevant change moves the macOS and
# Windows legs onto an ubuntu runner with every step skipped: the check
# names the ruleset requires still report, without a scarce runner.
test:
needs: changes
if: always()
@@ -107,9 +96,9 @@ jobs:
fail-fast: false
matrix:
os: [ubuntu-latest, macos-latest, windows-latest]
runs-on: ${{ github.event_name == 'pull_request' && needs.changes.outputs.platform == 'false' && 'ubuntu-latest' || matrix.os }}
runs-on: ${{ matrix.os }}
env:
RUN_STEPS: ${{ github.event_name != 'pull_request' || (needs.changes.outputs.code != 'false' && (matrix.os == 'ubuntu-latest' || needs.changes.outputs.platform != 'false')) }}
RUN_STEPS: ${{ github.event_name != 'pull_request' || needs.changes.outputs.code != 'false' }}
steps:
- if: env.RUN_STEPS == 'true'
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
@@ -179,34 +168,19 @@ jobs:
REASONIX_RELEASE_CACHE_GUARD: "1"
run: go test ./...
- name: test (Windows composition links)
if: env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request'
timeout-minutes: 5
shell: bash
run: |
set -o pipefail
go test -count=1 -timeout=3m -json -run '^Test(CaptureDirectoryMaterializesContainedLinks|CaptureDirectoryRefusesUnsafeLinks|CaptureHookFileFlattensContainedLink|MaterializeStillRejectsLinkedRetainedFilesAndRoot|WindowsCompositionJunctions|WindowsLiveSkillJunctions|CompositionRetainsLinkedSkillAcrossRestart|CompositionLinkedSkillRejectsLinkedResources|CompositionLinkedExtensionResources|CompositionLinkedHookResources)$' ./internal/state/agentpreset ./internal/ext/skill ./internal/assembly/boot | tee "$RUNNER_TEMP/composition-links.jsonl"
jq -se '
["TestCaptureDirectoryMaterializesContainedLinks", "TestCaptureDirectoryRefusesUnsafeLinks",
"TestCaptureHookFileFlattensContainedLink", "TestMaterializeStillRejectsLinkedRetainedFilesAndRoot",
"TestWindowsCompositionJunctions", "TestWindowsLiveSkillJunctions", "TestCompositionRetainsLinkedSkillAcrossRestart",
"TestCompositionLinkedSkillRejectsLinkedResources", "TestCompositionLinkedExtensionResources",
"TestCompositionLinkedHookResources"] as $required |
[.[] | select(.Action == "pass") | .Test] as $passed |
all(.[]; .Action != "skip") and (($required - $passed) | length == 0)
' "$RUNNER_TEMP/composition-links.jsonl"
# Pull requests run a Windows smoke suite: the packages that actually
# carry *_windows.go code, the startup/checkpoint packages with portable
# filesystem contracts, plus cmd/. Isolation/worktree/mutation have a
# complete JSON inventory below. The full ./... sweep stays on pushes
# filesystem contracts, plus cmd/. The full ./... sweep stays on pushes
# to main-v2; the Unix legs always run the full suite. Keep the job-level
# budget above normal 7-9 minute runner variance; each package test binary
# remains independently bounded by Go's per-package timeout below.
#
# The per-package timeout is 18m: internal/assembly/boot took 478-717s
# (12 push runs) on the Windows runner, so 18m is 1.5x the worst. A hang
# still trips the step budget (25m smoke, 35m full).
# That timeout is 8m, not 3m: across four consecutive runs of the same
# code, internal/assembly/boot measured 133s, 146s, 162s and then hit 180s, and
# internal/runtime/agent 82s, 92s, 144s and 180s — a 2.2x spread from runner
# variance alone, with four heavy packages sharing four vCPUs. At 3m the
# step failed on slow runners without a code change; a hang still trips
# the 15 minute job budget above.
#
# Every root-module `go test` that runs on Windows passes -count=1. With
# the result cache on, go test records each file and environment access
@@ -217,548 +191,11 @@ jobs:
# internal/base/testenv holds every such step to it.
- name: test (Windows smoke)
if: env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request'
timeout-minutes: 25
timeout-minutes: 15
env:
REASONIX_RELEASE_CACHE_GUARD: "1"
WINDOWS_SANDBOX_WAIT_MS: "20000"
run: go test -count=1 -p 4 -timeout=18m ./internal/runtime/agent/... ./internal/platform/appidentity/... ./internal/assembly/boot/... ./internal/state/checkpoint/... ./internal/frontend/cli/... ./internal/session/control/... ./internal/base/filelock/... ./internal/base/fileutil/... ./internal/ext/hook/... ./internal/state/instruction/... ./internal/ext/mcplaunch/... ./internal/platform/notify/... ./internal/base/proc/... ./internal/platform/remote/... ./internal/platform/repair/... ./internal/safety/sandbox/... ./internal/base/sysproxy/... ./internal/state/workspacelease/... ./cmd/...
- name: test (Windows published command job owner)
if: env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request'
shell: bash
run: |
set -euo pipefail
evidence="$RUNNER_TEMP/published-job-owner"
mkdir -p "$evidence"
go test -count=1 -timeout=3m -json -run '^TestPublishedJob' ./internal/base/proc | tee "$evidence/positive.jsonl"
node scripts/published-job-evidence.mjs positive "$evidence/positive.jsonl"
node scripts/published-job-evidence.mjs overlay "$evidence"
set +e
go test -count=1 -timeout=3m -json -overlay="$evidence/overlay.json" -run '^TestPublishedJob' ./internal/base/proc > "$evidence/negative.jsonl"
old_result=$?
set -e
test "$old_result" -eq 1
node scripts/published-job-evidence.mjs negative "$evidence/negative.jsonl"
- name: upload Windows published command job owner evidence
if: always() && env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request'
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
with:
name: published-job-owner-windows-${{ github.run_attempt }}
path: ${{ runner.temp }}/published-job-owner/
if-no-files-found: error
retention-days: 7
- name: test (Windows isolation owners)
if: env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request'
timeout-minutes: 25
shell: bash
run: |
set -o pipefail
go test -count=1 -p 4 -timeout=18m -json ./internal/platform/worktree/... ./internal/state/mutation/... | tee "$RUNNER_TEMP/isolation-owners.jsonl" |
jq -rj --unbuffered 'select(.Action == "output" and .Test == null) | .Output'
jq -se '
[.[] | select(.Test == null and .Action == "pass") | .Package] as $passed |
[.[] | select(.Test != null and .Action == "pass") | .Test] as $tests |
(["reasonix/internal/platform/worktree", "reasonix/internal/state/mutation"] - $passed | length) == 0 and
(["TestHeldPathsCoverOnlyTheSameFileAcrossCaseSpellings",
"TestLegacyWaitDoesNotConsumeExecutionCapacity"] - $tests | length) == 0
' "$RUNNER_TEMP/isolation-owners.jsonl"
jq -se '
. as $events |
{"reasonix/internal/platform/worktree": [
"TestWriteChangesRejectsRedirectedDeletion",
"TestWriteChangesUnlinksLeafDirectoryLink"
]} |
to_entries | all(.[]; . as $package |
([ $events[] | select(.Package == $package.key and .Test == null and
(.Action == "pass" or .Action == "fail" or .Action == "skip")) ] |
length == 1 and .[0].Action == "pass") and
all(.value[]; . as $test |
[ $events[] | select(.Package == $package.key and .Test == $test and
(.Action == "pass" or .Action == "fail" or .Action == "skip")) ] |
length == 1 and .[0].Action == "pass"))
' "$RUNNER_TEMP/isolation-owners.jsonl"
- name: upload Windows isolation test inventory
if: always() && env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request'
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
with:
name: isolation-owners-windows-${{ github.run_attempt }}
path: ${{ runner.temp }}/isolation-owners.jsonl
if-no-files-found: ignore
retention-days: 7
- name: test (Windows delegated Goal usage scope)
if: env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request'
shell: bash
run: |
set -o pipefail
go test -count=1 -timeout=3m -json -run '^Test(UsageScope.*)$' ./internal/contract/event ./internal/session/control ./internal/assembly/boot | tee "$RUNNER_TEMP/goal-usage-scope.jsonl"
jq -se '
["TestUsageScopeKeepsEarlierEventOwner",
"TestUsageScopeWithoutCapability" ] as $required |
[.[] | select(.Test != null and .Action == "pass") | .Test] as $passed |
[.[] | select(.Test == null and .Action == "pass") | .Package] as $packages |
all(.[]; .Action != "fail" and .Action != "skip") and
(($required - $passed) | length == 0) and
((["reasonix/internal/contract/event", "reasonix/internal/session/control", "reasonix/internal/assembly/boot"] - $packages) | length == 0)
' "$RUNNER_TEMP/goal-usage-scope.jsonl"
- name: upload Windows delegated Goal usage scope inventory
if: always() && env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request'
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
with:
name: goal-usage-scope-windows-${{ github.run_attempt }}
path: ${{ runner.temp }}/goal-usage-scope.jsonl
if-no-files-found: error
retention-days: 7
- name: test (Windows Controller routing root budget)
if: env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request'
timeout-minutes: 5
shell: bash
run: |
set -o pipefail
go test -count=1 -timeout=3m -json -run '^Test(SemanticRootUsage)$' ./internal/runtime/capability | tee "$RUNNER_TEMP/controller-routing-budget.jsonl"
jq -se '
[.[] | select(.Test == null and .Action == "pass") | .Package] as $packages |
[.[] | select(.Test != null and .Action == "pass") | .Test] as $tests |
all(.[]; .Action != "skip" and .Action != "fail") and
(["reasonix/internal/runtime/capability"] - $packages | length) == 0 and
([
"TestSemanticRootUsage",
"TestSemanticRootUsage/billed_error",
"TestSemanticRootUsage/cache",
"TestSemanticRootUsage/estimated",
"TestSemanticRootUsage/invalid_answer",
"TestSemanticRootUsage/missing",
"TestSemanticRootUsage/nil_sink",
"TestSemanticRootUsage/no_observer",
"TestSemanticRootUsage/reported"
] - $tests | length) == 0
' "$RUNNER_TEMP/controller-routing-budget.jsonl"
- name: upload Windows Controller routing root budget inventory
if: always() && env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request'
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
with:
name: controller-routing-budget-windows-${{ github.run_attempt }}
path: ${{ runner.temp }}/controller-routing-budget.jsonl
if-no-files-found: ignore
retention-days: 7
- name: test (Windows Coordinator root budget)
if: env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request'
timeout-minutes: 5
shell: bash
run: |
set -o pipefail
go test -count=1 -timeout=3m -json -run '^Test(PreparedRootBudget|CoordinatorRootBudget|EffectCoordinatorRoot)' ./internal/runtime/agent ./internal/runtime/coordinator | tee "$RUNNER_TEMP/coordinator-root-budget.jsonl"
jq -se '
[.[] | select(.Test == null and .Action == "pass") | .Package] as $packages |
[.[] | select(.Test != null and .Action == "pass") | .Test] as $tests |
all(.[]; .Action != "skip" and .Action != "fail") and
(["reasonix/internal/runtime/agent", "reasonix/internal/runtime/coordinator"] - $packages | length) == 0 and
(["TestCoordinatorRootBudgetRoutes",
"TestCoordinatorRootBudgetRoutes/raw", "TestCoordinatorRootBudgetRoutes/tool_planner",
"TestCoordinatorRootBudgetRoutes/failure_fallback", "TestCoordinatorRootBudgetRoutes/plan_only_continuation",
"TestCoordinatorRootBudgetRoutes/approval_declined_continuation", "TestCoordinatorRootBudgetRoutes/approval_allowed"
] - $tests | length) == 0
' "$RUNNER_TEMP/coordinator-root-budget.jsonl"
- name: upload Windows Coordinator root budget inventory
if: always() && env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request'
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
with:
name: coordinator-root-budget-windows-${{ github.run_attempt }}
path: ${{ runner.temp }}/coordinator-root-budget.jsonl
if-no-files-found: ignore
retention-days: 7
- name: test (Windows resumed Goal planner budget)
if: env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request'
shell: bash
run: |
set -o pipefail
go test -count=1 -timeout=3m -json -run '^Test(GoalResumeRenewsPlannerBudget|GoalNonBudgetResumeKeepsPlannerBudget|GoalTokenBudgetPausesAndResumes|EffectGoalResumeRenewsPlannerBudget)$' ./internal/session/control ./internal/assembly/boot | tee "$RUNNER_TEMP/goal-planner-resume.jsonl"
jq -se '
["TestGoalResumeRenewsPlannerBudget",
"TestGoalNonBudgetResumeKeepsPlannerBudget",
"TestGoalNonBudgetResumeKeepsPlannerBudget/manual_pause",
"TestGoalNonBudgetResumeKeepsPlannerBudget/not_resumable",
"TestGoalTokenBudgetPausesAndResumes",
"TestEffectGoalResumeRenewsPlannerBudget"] as $required |
[.[] | select(.Test != null and .Action == "pass") | .Test] as $passed |
[.[] | select(.Test == null and .Action == "pass") | .Package] as $packages |
all(.[]; .Action != "fail" and .Action != "skip") and
(($required - $passed) | length == 0) and
((["reasonix/internal/session/control", "reasonix/internal/assembly/boot"] - $packages) | length == 0)
' "$RUNNER_TEMP/goal-planner-resume.jsonl"
- name: upload Windows resumed Goal planner budget inventory
if: always() && env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request'
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
with:
name: goal-planner-resume-windows-${{ github.run_attempt }}
path: ${{ runner.temp }}/goal-planner-resume.jsonl
if-no-files-found: error
retention-days: 7
- name: test (Windows root wall budget)
if: env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request'
timeout-minutes: 5
shell: bash
run: |
set -o pipefail
go test -count=1 -timeout=3m -json -run '^Test(RootWall|EffectRootWall)' ./internal/runtime/agent ./internal/assembly/boot | tee "$RUNNER_TEMP/root-wall-budget.jsonl"
jq -se '
[.[] | select(.Test == null and .Action == "pass") | .Package] as $packages |
[.[] | select(.Test != null and .Action == "pass") | .Test] as $tests |
all(.[]; .Action != "skip" and .Action != "fail") and
(["reasonix/internal/runtime/agent", "reasonix/internal/assembly/boot"] - $packages | length) == 0 and
(["TestRootWallBudgetReachesDelegatedAgents", "TestRootWallBudgetReachesDelegatedAgents/foreground",
"TestRootWallBudgetReachesDelegatedAgents/nested", "TestRootWallBudgetReachesDelegatedAgents/siblings",
"TestRootWallBudgetReachesDelegatedAgents/context", "TestRootWallBudgetReachesDelegatedAgents/unconfigured",
"TestRootWallBudgetReachesDelegatedAgents/explicit_zero", "TestRootWallBudgetContinuationAndReset",
"TestRootWallBudgetContinuationAndReset/continuation", "TestRootWallBudgetContinuationAndReset/new_task",
"TestRootWallBudgetContinuationAndReset/reset", "TestRootWallBudgetContinuationAndReset/override"
] - $tests | length) == 0
' "$RUNNER_TEMP/root-wall-budget.jsonl"
- name: upload Windows root wall budget inventory
if: always() && env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request'
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
with:
name: root-wall-budget-windows-${{ github.run_attempt }}
path: ${{ runner.temp }}/root-wall-budget.jsonl
if-no-files-found: ignore
retention-days: 7
- name: test (Windows root token budget)
if: env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request'
timeout-minutes: 5
shell: bash
run: |
set -o pipefail
go test -count=1 -timeout=3m -json -run '^Test(RootToken|EffectRootToken|BudgetCountsKnownReportedTokens|ReportedToken)' ./internal/runtime/agent | tee "$RUNNER_TEMP/root-token-budget.jsonl"
jq -se '
[.[] | select(.Test == null and .Action == "pass") | .Package] as $packages |
[.[] | select(.Test != null and .Action == "pass") | .Test] as $tests |
all(.[]; .Action != "skip" and .Action != "fail") and
(["reasonix/internal/runtime/agent"] - $packages | length) == 0 and
(["TestRootTokenBudgetIncludesChildProviderUsage", "TestRootTokenBudgetIncludesChildProviderUsage/configured",
"TestRootTokenBudgetIncludesChildProviderUsage/context", "TestRootTokenBudgetIncludesChildProviderUsage/unconfigured",
"TestRootTokenBudgetIncludesChildProviderUsage/explicit_zero", "TestRootTokenBudgetSurvivesNestedAndConcurrentChildren",
"TestRootTokenBudgetSurvivesNestedAndConcurrentChildren/nested", "TestRootTokenBudgetSurvivesNestedAndConcurrentChildren/siblings",
"TestRootTokenBudgetSurvivesNestedAndConcurrentChildren/child_override", "TestRootTokenBudgetContinuationAndReset",
"TestRootTokenBudgetContinuationAndReset/continuation", "TestRootTokenBudgetContinuationAndReset/new_task",
"TestRootTokenBudgetContinuationAndReset/reset" ] - $tests | length) == 0
' "$RUNNER_TEMP/root-token-budget.jsonl"
jq -se '
. as $events |
{"reasonix/internal/runtime/agent": [
"TestBudgetCountsKnownReportedTokens",
"TestBudgetCountsKnownReportedTokens/split",
"TestBudgetCountsKnownReportedTokens/total_only",
"TestBudgetCountsKnownReportedTokens/partial_split",
"TestBudgetCountsKnownReportedTokens/lower_total",
"TestBudgetCountsKnownReportedTokens/missing_total",
"TestBudgetCountsKnownReportedTokens/negative_split",
"TestBudgetCountsKnownReportedTokens/zero",
"TestBudgetCountsKnownReportedTokens/negative",
"TestReportedTokenCountersSaturate"]} |
to_entries | all(.[]; . as $package |
([ $events[] | select(.Package == $package.key and .Test == null and
(.Action == "pass" or .Action == "fail" or .Action == "skip")) ] |
length == 1 and .[0].Action == "pass") and
all(.value[]; . as $test |
[ $events[] | select(.Package == $package.key and .Test == $test and
(.Action == "pass" or .Action == "fail" or .Action == "skip")) ] |
length == 1 and .[0].Action == "pass"))
' "$RUNNER_TEMP/root-token-budget.jsonl"
- name: upload Windows root token budget inventory
if: always() && env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request'
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
with:
name: root-token-budget-windows-${{ github.run_attempt }}
path: ${{ runner.temp }}/root-token-budget.jsonl
if-no-files-found: ignore
retention-days: 7
- name: test (Windows budget usage completeness)
if: env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request'
timeout-minutes: 5
shell: bash
run: |
set -o pipefail
go test -count=1 -timeout=3m -json -run '^Test(RequestOnlyUsage|RunBudgetMissingUsage|TaskBudgetKnownCost|RunBudgetExplicitZero|TaskBudgetUsageCompleteness|TaskBudgetCostUses|EffectBudgetCompleteness|TaskBudgetGateIgnoresCostWhenUnpriced)' ./internal/contract/provider ./internal/runtime/agent ./internal/assembly/boot | tee "$RUNNER_TEMP/budget-usage.jsonl"
jq -se '
[.[] | select(.Test == null and .Action == "pass") | .Package] as $packages |
[.[] | select(.Test != null and .Action == "pass") | .Test] as $tests |
all(.[]; .Action != "skip" and .Action != "fail") and
(["reasonix/internal/contract/provider", "reasonix/internal/runtime/agent", "reasonix/internal/assembly/boot"] - $packages | length) == 0 and
(["TestRequestOnlyUsageDoesNotCertifyTokenAccounting", "TestRunBudgetMissingUsageKeepsCostIncomplete",
"TestRunBudgetMissingUsageKeepsCostIncomplete/complete", "TestRunBudgetMissingUsageKeepsCostIncomplete/missing_first",
"TestRunBudgetMissingUsageKeepsCostIncomplete/missing_last", "TestRunBudgetMissingUsageKeepsCostIncomplete/all_missing",
"TestTaskBudgetKnownCostStillLandsAfterMissingUsage", "TestRunBudgetExplicitZeroUsageIsPriced",
"TestTaskBudgetUsageCompletenessFollowsItsLifetime", "TestTaskBudgetUsageCompletenessFollowsItsLifetime/continuation",
"TestTaskBudgetUsageCompletenessFollowsItsLifetime/new_task", "TestTaskBudgetUsageCompletenessFollowsItsLifetime/explicit_reset",
"TestTaskBudgetCostUsesOnlyTheReportedSubtotal", "TestTaskBudgetGateIgnoresCostWhenUnpriced",
"TestEffectBudgetCompletenessFromOpenAIStream", "TestEffectBudgetCompletenessFromOpenAIStream/missing_0",
"TestEffectBudgetCompletenessFromOpenAIStream/missing_1", "TestEffectBudgetCompletenessFromOpenAIStream/missing_2"] - $tests | length) == 0
' "$RUNNER_TEMP/budget-usage.jsonl"
- name: upload Windows budget usage inventory
if: always() && env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request'
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
with:
name: budget-usage-windows-${{ github.run_attempt }}
path: ${{ runner.temp }}/budget-usage.jsonl
if-no-files-found: ignore
retention-days: 7
- name: test (Windows Git storage environment)
if: env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request'
timeout-minutes: 5
shell: bash
run: |
set -o pipefail
go test -count=1 -timeout=3m -json -run '^Test(HostStorageEnvironment|HostObjectStorage|HostExplicitStorageControls|DetachedStorageEnvironment)$' ./internal/platform/gitcmd ./internal/platform/worktree ./internal/assembly/boot | tee "$RUNNER_TEMP/git-storage-environment.jsonl"
jq -se '
[.[] | select(.Test == null and .Action == "pass") | .Package] as $packages |
[.[] | select(.Test != null and .Action == "pass") | .Test] as $tests |
all(.[]; .Action != "skip" and .Action != "fail") and
(["reasonix/internal/platform/gitcmd", "reasonix/internal/platform/worktree", "reasonix/internal/assembly/boot"] - $packages | length) == 0 and
([
"TestDetachedStorageEnvironment",
"TestDetachedStorageEnvironment/explicit_env",
"TestDetachedStorageEnvironment/inherited_env",
"TestHostExplicitStorageControls",
"TestHostExplicitStorageControls/explicit_object_directory",
"TestHostExplicitStorageControls/private_index",
"TestHostExplicitStorageControls/repository_declared_alternate",
"TestHostObjectStorage",
"TestHostObjectStorage/ambient_alternate_not_read",
"TestHostObjectStorage/writes_stay_in_repository",
"TestHostStorageEnvironment",
"TestHostStorageEnvironment/GIT_ALTERNATE_OBJECT_DIRECTORIES",
"TestHostStorageEnvironment/GIT_INDEX_FILE",
"TestHostStorageEnvironment/GIT_OBJECT_DIRECTORY"
] - $tests | length) == 0
' "$RUNNER_TEMP/git-storage-environment.jsonl"
- name: upload Windows Git storage environment inventory
if: always() && env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request'
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
with:
name: git-storage-environment-windows-${{ github.run_attempt }}
path: ${{ runner.temp }}/git-storage-environment.jsonl
if-no-files-found: ignore
retention-days: 7
- name: test (Windows Anthropic interrupted usage)
if: env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request'
timeout-minutes: 5
shell: bash
run: |
set -o pipefail
go test -count=1 -timeout=3m -json -run '^Test(InterruptedUsage|EffectAnthropicPartialUsageRecovery)' ./internal/model/anthropic ./internal/assembly/boot | tee "$RUNNER_TEMP/anthropic-interrupted-usage.jsonl"
jq -se '
[.[] | select(.Test == null and .Action == "pass") | .Package] as $packages |
[.[] | select(.Test != null and .Action == "pass") | .Test] as $tests |
all(.[]; .Action != "skip" and .Action != "fail") and
(["reasonix/internal/model/anthropic", "reasonix/internal/assembly/boot"] - $packages | length) == 0 and
(["TestInterruptedUsageTerminal", "TestInterruptedUsageTerminal/upstream_error",
"TestInterruptedUsageTerminal/payload_error", "TestInterruptedUsageTerminal/premature_eof",
"TestInterruptedUsageTerminal/decode_error", "TestInterruptedUsageTerminal/reported_zero",
"TestInterruptedUsageTerminal/missing_control", "TestInterruptedUsageTerminal/complete_control",
"TestInterruptedUsageBoundedConsumer", "TestInterruptedUsageReadErrorAndCancellation",
"TestInterruptedUsageReadErrorAndCancellation/read_error", "TestInterruptedUsageReadErrorAndCancellation/cancelled_buffered_reader",
"TestInterruptedUsageAbandonedConsumer", "TestInterruptedUsageIdleTimeout",
"TestEffectAnthropicPartialUsageRecovery", "TestEffectAnthropicPartialUsageRecovery/upstream_error",
"TestEffectAnthropicPartialUsageRecovery/premature_eof", "TestEffectAnthropicPartialUsageRecovery/complete_control"] - $tests | length) == 0
' "$RUNNER_TEMP/anthropic-interrupted-usage.jsonl"
- name: upload Windows Anthropic interrupted usage inventory
if: always() && env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request'
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
with:
name: anthropic-interrupted-usage-windows-${{ github.run_attempt }}
path: ${{ runner.temp }}/anthropic-interrupted-usage.jsonl
if-no-files-found: ignore
retention-days: 7
- name: test (Windows root websearch budget)
if: env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request'
timeout-minutes: 5
shell: bash
run: |
set -o pipefail
go test -count=1 -timeout=3m -json -run '^Test(RootWebSearch|EffectRootWebSearch)' ./internal/tools/websearch | tee "$RUNNER_TEMP/root-websearch-budget.jsonl"
jq -se '
[.[] | select(.Test == null and .Action == "pass") | .Package] as $packages |
[.[] | select(.Test != null and .Action == "pass") | .Test] as $tests |
all(.[]; .Action != "skip" and .Action != "fail") and
(["reasonix/internal/tools/websearch"] - $packages | length) == 0 and
(["TestRootWebSearchUsageTerminal", "TestRootWebSearchUsageTerminal/reported",
"TestRootWebSearchUsageTerminal/no_reporter", "TestRootWebSearchUsageTerminal/missing",
"TestRootWebSearchUsageTerminal/no_observer", "TestRootWebSearchUsageTerminal/input_refused",
"TestRootWebSearchUsageTerminal/no_factory", "TestRootWebSearchUsageTerminal/factory_error",
"TestRootWebSearchUsageTerminal/billed_failure", "TestRootWebSearchUsageTerminal/cancel_after_usage",
"TestRootWebSearchUsageTerminal/cancel_before_start", "TestRootWebSearchUsageTerminal/stream_error"
] - $tests | length) == 0
' "$RUNNER_TEMP/root-websearch-budget.jsonl"
- name: upload Windows root websearch budget inventory
if: always() && env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request'
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
with:
name: root-websearch-budget-windows-${{ github.run_attempt }}
path: ${{ runner.temp }}/root-websearch-budget.jsonl
if-no-files-found: ignore
retention-days: 7
- name: test (Windows explicit skill root budget)
if: env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request'
timeout-minutes: 5
shell: bash
run: |
set -o pipefail
go test -count=1 -timeout=3m -json -run '^(TestProfileRootBudgetLifetime|TestProfileBudgetPreparationUsesTurnAdmission|TestEffectExplicitSkillRootBudget)$' ./internal/session/control | tee "$RUNNER_TEMP/explicit-skill-root-budget.jsonl"
jq -se '
[.[] | select(.Test == null and .Action == "pass") | .Package] as $packages |
[.[] | select(.Test != null and .Action == "pass") | .Test] as $tests |
all(.[]; .Action != "skip" and .Action != "fail") and
(["reasonix/internal/session/control"] - $packages | length) == 0 and
(["TestProfileBudgetPreparationUsesTurnAdmission" ] - $tests | length) == 0
' "$RUNNER_TEMP/explicit-skill-root-budget.jsonl"
- name: upload Windows explicit skill root budget inventory
if: always() && env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request'
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
with:
name: explicit-skill-root-budget-windows-${{ github.run_attempt }}
path: ${{ runner.temp }}/explicit-skill-root-budget.jsonl
if-no-files-found: ignore
retention-days: 7
- name: test (Windows root triage budget)
if: env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request'
timeout-minutes: 5
shell: bash
run: |
set -o pipefail
go test -count=1 -timeout=3m -json -run '^Test(RootTriage|EffectRootTriage)' ./internal/runtime/agent | tee "$RUNNER_TEMP/root-triage-budget.jsonl"
jq -se '
[.[] | select(.Test == null and .Action == "pass") | .Package] as $packages |
[.[] | select(.Test != null and .Action == "pass") | .Test] as $tests |
all(.[]; .Action != "skip" and .Action != "fail") and
(["reasonix/internal/runtime/agent"] - $packages | length) == 0 and
(["TestRootTriageTokenBudget", "TestRootTriageTokenBudget/classifier", "TestRootTriageTokenBudget/screen",
"TestRootTriageTokenBudget/billed_failure", "TestRootTriageTokenBudget/missing", "TestRootTriageTokenBudget/disabled",
"TestRootTriageTokenBudget/no_sink", "TestRootTriageTokenBudget/counts_without_total",
"TestRootTriageOutsideRunDoesNotCreateScope" ] - $tests | length) == 0
' "$RUNNER_TEMP/root-triage-budget.jsonl"
- name: upload Windows root triage budget inventory
if: always() && env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request'
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
with:
name: root-triage-budget-windows-${{ github.run_attempt }}
path: ${{ runner.temp }}/root-triage-budget.jsonl
if-no-files-found: ignore
retention-days: 7
- name: test (Windows local cost quote)
if: env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request'
shell: bash
run: |
set -o pipefail
go test -count=1 -timeout=3m -json -run '^Test(LocalBudgetUsesOccurrenceQuote|LocalCost.*|UsageQuote.*)$' ./internal/contract/event ./internal/runtime/agent | tee "$RUNNER_TEMP/local-cost-quote.jsonl"
jq -se '
["TestLocalBudgetUsesOccurrenceQuote", "TestLocalBudgetUsesOccurrenceQuote/peak", "TestLocalBudgetUsesOccurrenceQuote/offpeak",
"TestLocalCostKeepsOriginalCurrencyBuckets", "TestLocalCostFixedPointThresholdAndUnknown",
"TestUsageQuoteUsesOneOwnerAndOccurrence", "TestUsageQuoteLegacyAndNilSinks"
] as $required |
[.[] | select(.Test != null and .Action == "pass") | .Test] as $passed |
[.[] | select(.Test == null and .Action == "pass") | .Package] as $packages |
all(.[]; .Action != "fail" and .Action != "skip") and
(($required - $passed) | length == 0) and
((["reasonix/internal/contract/event", "reasonix/internal/runtime/agent"] - $packages) | length == 0)
' "$RUNNER_TEMP/local-cost-quote.jsonl"
- name: upload Windows local cost quote inventory
if: always() && env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request'
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
with:
name: local-cost-quote-windows-${{ github.run_attempt }}
path: ${{ runner.temp }}/local-cost-quote.jsonl
if-no-files-found: error
retention-days: 7
- name: test (Windows estimated usage quotes)
if: env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request'
timeout-minutes: 5
shell: bash
run: |
set -o pipefail
go test -count=1 -timeout=3m -json -run '^Test(EstimatedUsage|EffectEstimatedUsage|ToWireUsagePayloadJSON$|StatusExtensionTracksMultipleSessionsAndUsage$|CacheSplitProvenanceIsRecorded$|RecordedPlainCallIsUnchanged$)' ./internal/contract/pricing ./internal/contract/event ./internal/contract/config ./internal/assembly/boot ./internal/contract/eventwire ./internal/frontend/acp ./internal/state/stats | tee "$RUNNER_TEMP/estimated-quote.jsonl"
jq -se '
[.[] | select(.Test == null and .Action == "pass") | .Package] as $packages |
[.[] | select(.Test != null and .Action == "pass") | .Test] as $tests |
all(.[]; .Action != "skip" and .Action != "fail") and
(["reasonix/internal/contract/pricing", "reasonix/internal/contract/event", "reasonix/internal/contract/config", "reasonix/internal/assembly/boot",
"reasonix/internal/contract/eventwire", "reasonix/internal/frontend/acp", "reasonix/internal/state/stats"] - $packages | length) == 0 and
(["TestEstimatedUsageQuotePreservesAmountsWithoutCompleteFacts", "TestEstimatedUsageQuotePreservesAmountsWithoutCompleteFacts/CNY",
"TestEstimatedUsageQuotePreservesAmountsWithoutCompleteFacts/USD", "TestEstimatedUsageLedgerRetainsPartialCoverage",
"TestEstimatedUsageLedgerRetainsPartialCoverage/separate_keys", "TestEstimatedUsageLedgerRetainsPartialCoverage/same_key",
"TestEstimatedUsageSavedQuoteReader", "TestEstimatedUsageQuoteSinkCoversTurnAndGuardian",
"TestEstimatedUsageQuoteSinkCoversTurnAndGuardian/turn", "TestEstimatedUsageQuoteSinkCoversTurnAndGuardian/guardian",
"TestEstimatedUsageConfiguredQuote", "TestEffectEstimatedUsageQuoteAfterStreamRecovery",
"TestEffectEstimatedUsageQuoteAfterStreamRecovery/reported_control", "TestEffectEstimatedUsageQuoteAfterStreamRecovery/interrupted",
"TestToWireUsagePayloadJSON", "TestStatusExtensionTracksMultipleSessionsAndUsage", "TestCacheSplitProvenanceIsRecorded", "TestRecordedPlainCallIsUnchanged"] - $tests | length) == 0
' "$RUNNER_TEMP/estimated-quote.jsonl"
- name: upload Windows estimated quote inventory
if: always() && env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request'
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
with:
name: estimated-quote-windows-${{ github.run_attempt }}
path: ${{ runner.temp }}/estimated-quote.jsonl
if-no-files-found: ignore
retention-days: 7
- name: test (Windows bounded root budget)
if: env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request'
timeout-minutes: 5
shell: bash
run: |
set -o pipefail
go test -count=1 -timeout=3m -json -run '^Test(AuxiliaryUsageObserver|BoundedUsageObserver|RootBounded|EffectRootBounded)' ./internal/contract/provider ./internal/model/boundedllm ./internal/runtime/agent | tee "$RUNNER_TEMP/bounded-root-budget.jsonl"
jq -se '
[.[] | select(.Test == null and .Action == "pass") | .Package] as $packages |
[.[] | select(.Test != null and .Action == "pass") | .Test] as $tests |
all(.[]; .Action != "skip" and .Action != "fail") and
(["reasonix/internal/contract/provider", "reasonix/internal/model/boundedllm",
"reasonix/internal/runtime/agent"] - $packages | length) == 0 and
(["TestAuxiliaryUsageObserverReplacesAndCopies",
"TestBoundedUsageObserverOwnsOneTerminal", "TestBoundedUsageObserverOwnsOneTerminal/reported",
"TestBoundedUsageObserverOwnsOneTerminal/billed_failure", "TestBoundedUsageObserverOwnsOneTerminal/without_sink",
"TestBoundedUsageObserverOwnsOneTerminal/missing", "TestBoundedUsageObserverOwnsOneTerminal/input_refused"
] - $tests | length) == 0
' "$RUNNER_TEMP/bounded-root-budget.jsonl"
- name: upload Windows bounded root budget inventory
if: always() && env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request'
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
with:
name: bounded-root-budget-windows-${{ github.run_attempt }}
path: ${{ runner.temp }}/bounded-root-budget.jsonl
if-no-files-found: ignore
retention-days: 7
run: go test -count=1 -p 4 -timeout=8m ./internal/runtime/agent/... ./internal/platform/appidentity/... ./internal/assembly/boot/... ./internal/state/checkpoint/... ./internal/frontend/cli/... ./internal/session/control/... ./internal/base/filelock/... ./internal/base/fileutil/... ./internal/ext/hook/... ./internal/state/instruction/... ./internal/ext/mcplaunch/... ./internal/platform/notify/... ./internal/base/proc/... ./internal/platform/remote/... ./internal/platform/repair/... ./internal/safety/sandbox/... ./internal/base/sysproxy/... ./internal/state/workspacelease/... ./cmd/...
# The general Windows PR smoke list intentionally omits internal/contract/tool.
# Keep the session-temp portability contract covered without widening the
@@ -777,6 +214,24 @@ jobs:
timeout-minutes: 5
run: go test -count=1 -timeout=3m -run '^TestBashPowerShellExecuteDetailedContract$|^TestBashPowerShell51PreflightRejectsAndAndDetailed$|^TestBashPowerShellOutputIsUTF8$|^TestBashPowerShellSurfacesNonZeroExit$|^TestBashPowerShellRejectsChaining$' ./internal/tools/builtin
# Full push suite still uses the same 8m package budget as Windows smoke:
# agent/boot/control already hit 180s package timeouts under runner load,
# so 3m fails on slow machines without a code regression. It runs with
# -count=1 for the reason given at the smoke step. A hang still dies here
# rather than on a package.
- name: test (full)
if: env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name != 'pull_request'
timeout-minutes: 30
env:
# Run the prompt-cache prefix-stability guard (TestCacheHit*) in CI: a
# regression there silently tanks the cache hit rate the project is
# built around.
REASONIX_RELEASE_CACHE_GUARD: "1"
# Bound sandbox helper children in Windows tests so a failed OS-level
# launch cannot pin the Actions step after Go's package timeout fires.
WINDOWS_SANDBOX_WAIT_MS: "20000"
run: go test -count=1 -p 4 -timeout=8m ./...
race:
needs: changes
if: always() && (github.event_name != 'pull_request' || needs.changes.outputs.code != 'false')
@@ -811,130 +266,13 @@ jobs:
if: github.event_name == 'pull_request'
env:
REASONIX_RELEASE_CACHE_GUARD: "1"
run: go test -race ./internal/runtime/agent/... ./internal/runtime/delegation/... ./internal/platform/worktree/... ./internal/state/mutation/... ./internal/ext/plugin/... ./internal/tools/jobs/... ./internal/base/proc/... ./internal/safety/sandbox/... ./internal/base/filelock/... ./internal/contract/eventwire/... ./internal/platform/remote/... ./internal/ext/extension/... ./internal/assembly/boot/... ./internal/session/control/... ./internal/contract/tool/...
run: go test -race ./internal/runtime/agent/... ./internal/ext/plugin/... ./internal/tools/jobs/... ./internal/base/proc/... ./internal/safety/sandbox/... ./internal/base/filelock/... ./internal/contract/eventwire/... ./internal/platform/remote/... ./internal/ext/extension/... ./internal/assembly/boot/... ./internal/session/control/... ./internal/contract/tool/...
# Push-only full suites. internal/frontend/serve and internal/assembly/boot run
# hundreds of serial tests and set the wall time of one whole-tree `go test`,
# so each is cut by test name into shards that run on their own runners. The
# -run / -skip patterns partition the names exactly: every test lands in one
# shard. `rest` is every other package. Pull requests keep the focused steps
# in the `test` and `race` jobs above.
test-windows-shards:
if: github.event_name != 'pull_request'
name: test (windows-latest, ${{ matrix.shard }})
runs-on: windows-latest
strategy:
fail-fast: false
matrix:
include:
- { shard: rest }
- { shard: serve-1, pkg: ./internal/frontend/serve, flag: -run, filter: '^Test[A-E]' }
- { shard: serve-2, pkg: ./internal/frontend/serve, flag: -run, filter: '^Test[F-R]' }
- { shard: serve-3, pkg: ./internal/frontend/serve, flag: -skip, filter: '^Test[A-R]' }
- { shard: boot-1, pkg: ./internal/assembly/boot, flag: -skip, filter: '^TestEffect([^0-9A-C]|$)' }
- { shard: boot-2, pkg: ./internal/assembly/boot, flag: -run, filter: '^TestEffect[D-O]' }
- { shard: boot-3, pkg: ./internal/assembly/boot, flag: -run, filter: '^TestEffect([^0-9A-O]|$)' }
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
- uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7
with:
go-version-file: go.mod
cache: true
- name: test (shard)
timeout-minutes: 40
env:
# Run the prompt-cache prefix-stability guard (TestCacheHit*) in CI: a
# regression there silently tanks the cache hit rate the project is
# built around.
REASONIX_RELEASE_CACHE_GUARD: "1"
# Bound sandbox helper children in Windows tests so a failed OS-level
# launch cannot pin the Actions step after Go's package timeout fires.
WINDOWS_SANDBOX_WAIT_MS: "20000"
SHARD: ${{ matrix.shard }}
PKG: ${{ matrix.pkg }}
FLAG: ${{ matrix.flag }}
FILTER: ${{ matrix.filter }}
shell: bash
run: |
if [ "$SHARD" = rest ]; then
mapfile -t pkgs < <(go list ./... | tr -d '\r' | grep -v -E '/internal/(frontend/serve|assembly/boot)$')
else
pkgs=("$PKG")
fi
filter=()
if [ -n "$FILTER" ]; then filter=("$FLAG" "$FILTER"); fi
go test -count=1 -p 4 -timeout=36m -json "${filter[@]}" "${pkgs[@]}" | tee "$RUNNER_TEMP/shard.jsonl" |
jq -rj --unbuffered 'select(.Action == "output" and .Test == null) | .Output'
- name: test (shard) failure detail
if: failure()
shell: bash
run: |
f="$RUNNER_TEMP/shard.jsonl"
[ -s "$f" ] || exit 0
echo "::group::output of failed tests"
jq -rn '[inputs | select(.Test != null)] | group_by(.Package + "/" + .Test)
| map(select(any(.[]; .Action == "fail")))[]
| map(select(.Action == "output") | .Output) | join("")' "$f" || true
echo "::endgroup::"
echo "slowest tests (seconds, package, name):"
jq -rn '[inputs | select(.Test != null and .Action == "pass")]
| sort_by(-.Elapsed)[:15][] | "\(.Elapsed) \(.Package) \(.Test)"' "$f" || true
race-shards:
if: github.event_name != 'pull_request'
name: race (${{ matrix.shard }})
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
include:
- { shard: rest }
- { shard: serve-1, pkg: ./internal/frontend/serve, flag: -run, filter: '^Test[A-E]' }
- { shard: serve-2, pkg: ./internal/frontend/serve, flag: -run, filter: '^Test[F-R]' }
- { shard: serve-3, pkg: ./internal/frontend/serve, flag: -skip, filter: '^Test[A-R]' }
- { shard: boot-1, pkg: ./internal/assembly/boot, flag: -skip, filter: '^TestEffect([^0-9A-C]|$)' }
- { shard: boot-2, pkg: ./internal/assembly/boot, flag: -run, filter: '^TestEffect[D-O]' }
- { shard: boot-3, pkg: ./internal/assembly/boot, flag: -run, filter: '^TestEffect([^0-9A-O]|$)' }
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
- uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7
with:
go-version-file: go.mod
cache: true
- name: Install and verify Linux sandbox backend
run: |
sudo apt-get update
sudo apt-get install -y bubblewrap
if sysctl -n kernel.unprivileged_userns_clone >/dev/null 2>&1; then
sudo sysctl -w kernel.unprivileged_userns_clone=1
fi
if sysctl -n kernel.apparmor_restrict_unprivileged_userns >/dev/null 2>&1; then
sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0
fi
bwrap --ro-bind / / --dev /dev --proc /proc -- true || echo "::warning::bubblewrap is installed but this runner denies user namespaces; unavailable-backend tests will run fail-closed"
- name: test -race (shard)
timeout-minutes: 30
- name: test -race (full)
if: github.event_name != 'pull_request'
env:
REASONIX_RELEASE_CACHE_GUARD: "1"
SHARD: ${{ matrix.shard }}
PKG: ${{ matrix.pkg }}
FLAG: ${{ matrix.flag }}
FILTER: ${{ matrix.filter }}
run: |
if [ "$SHARD" = rest ]; then
mapfile -t pkgs < <(go list ./... | grep -v -E '/internal/(frontend/serve|assembly/boot)$')
else
pkgs=("$PKG")
fi
filter=()
if [ -n "$FILTER" ]; then filter=("$FLAG" "$FILTER"); fi
go test -race -timeout=25m "${filter[@]}" "${pkgs[@]}"
run: go test -race ./...
# sdk/go is a nested stdlib-only module invisible to root `go test ./...`.
# Its DTOs are generated from internal/ext/extension/protocol, and the
@@ -947,9 +285,9 @@ jobs:
fail-fast: false
matrix:
os: [ubuntu-latest, macos-latest, windows-latest]
runs-on: ${{ github.event_name == 'pull_request' && needs.changes.outputs.platform == 'false' && 'ubuntu-latest' || matrix.os }}
runs-on: ${{ matrix.os }}
env:
RUN_STEPS: ${{ github.event_name != 'pull_request' || (needs.changes.outputs.sdk != 'false' && (matrix.os == 'ubuntu-latest' || needs.changes.outputs.platform != 'false')) }}
RUN_STEPS: ${{ github.event_name != 'pull_request' || needs.changes.outputs.sdk != 'false' }}
defaults:
run:
working-directory: sdk/go
@@ -1054,7 +392,7 @@ jobs:
# compile it against a real macOS toolchain.
desktop-macos:
needs: changes
if: always() && (github.event_name != 'pull_request' || (needs.changes.outputs.desktop != 'false' && needs.changes.outputs.platform != 'false'))
if: always() && (github.event_name != 'pull_request' || needs.changes.outputs.desktop != 'false')
runs-on: macos-latest
defaults:
run:
@@ -1082,7 +420,7 @@ jobs:
# (//go:build windows), so the ubuntu leg above never compiles them.
desktop-windows:
needs: changes
if: always() && (github.event_name != 'pull_request' || (needs.changes.outputs.desktop != 'false' && needs.changes.outputs.platform != 'false'))
if: always() && (github.event_name != 'pull_request' || needs.changes.outputs.desktop != 'false')
runs-on: windows-latest
defaults:
run:
@@ -1153,7 +491,6 @@ jobs:
run: |
go run github.com/rhysd/actionlint/cmd/[email protected] \
-ignore 'label "windows-11-arm" is unknown' \
-ignore 'label "ubuntu-22.04-arm" is unknown' \
.github/workflows/release-studio.yml \
.github/workflows/studio-certum-signing-smoke.yml \
.github/workflows/ci.yml
@@ -1193,7 +530,7 @@ jobs:
run: go test -coverprofile=coverage.out -covermode=atomic ./...
- name: upload coverage
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
with:
name: coverage-report
path: coverage.out
+3 -3
View File
@@ -43,20 +43,20 @@ jobs:
if: matrix.branch != ''
run: echo "sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT"
- name: Initialize CodeQL
uses: github/codeql-action/init@33d7b85da1a04171e211befe2c739303b9a4c066 # v4.39.3
uses: github/codeql-action/init@2892aa5e19bbd11bc0cff5427e3b750a04d9e3c2 # v4.38.2
with:
languages: ${{ matrix.language }}
build-mode: ${{ matrix.language == 'go' && 'autobuild' || 'none' }}
- name: Perform CodeQL Analysis
if: matrix.branch == ''
uses: github/codeql-action/analyze@33d7b85da1a04171e211befe2c739303b9a4c066 # v4.39.3
uses: github/codeql-action/analyze@2892aa5e19bbd11bc0cff5427e3b750a04d9e3c2 # v4.38.2
with:
category: "/language:${{ matrix.language }}"
# Results from a checked-out line are filed under that line's ref, not the
# default branch the schedule happened to run from.
- name: Perform CodeQL Analysis (${{ matrix.branch }})
if: matrix.branch != ''
uses: github/codeql-action/analyze@33d7b85da1a04171e211befe2c739303b9a4c066 # v4.39.3
uses: github/codeql-action/analyze@2892aa5e19bbd11bc0cff5427e3b750a04d9e3c2 # v4.38.2
with:
category: "/language:${{ matrix.language }}"
ref: refs/heads/${{ matrix.branch }}
+2 -2
View File
@@ -20,11 +20,11 @@ jobs:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
- uses: pnpm/action-setup@73e46992e05b19abd31f2f3c20e0e7370d762115 # v6.1.1
- uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413 # v6.1.0
with:
version: 10
run_install: false
- uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
with:
node-version: "24"
cache: pnpm
+11 -1
View File
@@ -20,7 +20,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
- uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
with:
node-version: '22'
cache: npm
@@ -36,6 +36,16 @@ jobs:
env:
CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
run: npm run migrate:diagnostics-v2
- name: Apply and verify feedback D1 migration
working-directory: workers/crash-report
env:
CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
run: |
npx wrangler d1 execute reasonix-crash --remote --file=migrate-feedback.sql
out=$(npx wrangler d1 execute reasonix-crash --remote --json --command "SELECT name FROM sqlite_master WHERE name IN ('feedback','feedback_status_created','feedback_install_created')")
for name in feedback feedback_status_created feedback_install_created; do
printf '%s' "$out" | grep -q "\"$name\"" || { echo "missing $name after migration"; exit 1; }
done
- name: Deploy
working-directory: workers/crash-report
env:
+2 -2
View File
@@ -20,11 +20,11 @@ jobs:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
- uses: pnpm/action-setup@73e46992e05b19abd31f2f3c20e0e7370d762115 # v6.1.1
- uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413 # v6.1.0
with:
version: 10
run_install: false
- uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
with:
node-version: "24"
cache: pnpm
-44
View File
@@ -1,44 +0,0 @@
name: Deploy ops events relay
on:
workflow_dispatch:
permissions:
contents: read
concurrency:
group: deploy-ops-events
cancel-in-progress: false
jobs:
deploy:
if: github.ref == 'refs/heads/platform'
runs-on: ubuntu-latest
environment: production
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
- uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7
with:
node-version: '22'
cache: npm
cache-dependency-path: workers/ops-events/package-lock.json
- name: Validate
working-directory: workers/ops-events
run: |
npm ci
npm run typecheck
npm test
- name: Deploy
working-directory: workers/ops-events
env:
CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
run: npx wrangler deploy
- name: Sync secrets
working-directory: workers/ops-events
env:
CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
GITHUB_WEBHOOK_SECRET: ${{ secrets.OPS_GITHUB_WEBHOOK_SECRET }}
OPS_LISTEN_TOKEN: ${{ secrets.OPS_LISTEN_TOKEN }}
OPS_EMIT_TOKEN: ${{ secrets.OPS_EMIT_TOKEN }}
run: |
node -e 'process.stdout.write(JSON.stringify({GITHUB_WEBHOOK_SECRET:process.env.GITHUB_WEBHOOK_SECRET||null,OPS_LISTEN_TOKEN:process.env.OPS_LISTEN_TOKEN||null,OPS_EMIT_TOKEN:process.env.OPS_EMIT_TOKEN||null}))' | npx wrangler secret bulk
+2 -2
View File
@@ -23,12 +23,12 @@ jobs:
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
- uses: pnpm/action-setup@73e46992e05b19abd31f2f3c20e0e7370d762115 # v6.1.1
- uses: pnpm/action-setup@0ebf47130e4866e96fce0953f49152a61190b271 # v6.0.9
with:
version: 10
run_install: false
- uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
with:
node-version: "24"
cache: pnpm
+1 -1
View File
@@ -51,7 +51,7 @@ jobs:
go-version-file: go.mod
cache: true
- uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7
- uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7
with:
python-version: '3.12'
-77
View File
@@ -1,77 +0,0 @@
name: Sync in-app feedback
# Turns feedback submitted from Reasonix Studio into issues and reports issue
# state (in progress, fixed in a tag, won't fix, duplicate) back to the worker,
# and mirrors the reporter's replies onto their issue as bot comments.
# Text from the public reaches the script only through the worker's JSON and
# the GitHub API, never through a shell command line or an expression.
on:
schedule:
- cron: '*/10 * * * *'
- cron: '5 3 * * *'
workflow_dispatch:
inputs:
dry_run:
description: Print what would happen without creating issues or writing back
required: false
default: false
type: boolean
full:
description: Re-read every open entry instead of only recently changed issues
required: false
default: false
type: boolean
permissions:
contents: read
issues: write
concurrency:
group: feedback-sync
cancel-in-progress: false
jobs:
sync:
if: github.repository == 'esengine/DeepSeek-Reasonix'
runs-on: ubuntu-latest
timeout-minutes: 8
steps:
- name: Check secret
id: gate
env:
FEEDBACK_ADMIN_TOKEN: ${{ secrets.FEEDBACK_ADMIN_TOKEN }}
run: |
if [ -z "$FEEDBACK_ADMIN_TOKEN" ]; then
echo "::notice::FEEDBACK_ADMIN_TOKEN is not set; feedback sync skipped"
echo "run=false" >> "$GITHUB_OUTPUT"
else
echo "run=true" >> "$GITHUB_OUTPUT"
fi
- if: steps.gate.outputs.run == 'true'
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
# git tag --contains needs full history; the tag API would cost a request per entry.
fetch-depth: 0
persist-credentials: false
- if: steps.gate.outputs.run == 'true'
uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7
with:
node-version: '22'
- name: Sync
if: steps.gate.outputs.run == 'true'
env:
FEEDBACK_ADMIN_TOKEN: ${{ secrets.FEEDBACK_ADMIN_TOKEN }}
# crash.reasonix.io answers runner IPs with a managed challenge; workers.dev serves only the converter's admin paths.
FEEDBACK_BASE_URL: https://reasonix-crash-report.reasonix.workers.dev
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
DRY_RUN: ${{ github.event.inputs.dry_run || 'false' }}
FULL: ${{ github.event.inputs.full || 'false' }}
SCHEDULE: ${{ github.event.schedule }}
run: |
set -euo pipefail
args=()
if [ "$DRY_RUN" = "true" ]; then args+=(--dry-run); fi
# The daily schedule sweeps everything the change window could have missed.
if [ "$FULL" = "true" ] || [ "$SCHEDULE" = "5 3 * * *" ]; then args+=(--full); fi
node scripts/feedback-sync.mjs "${args[@]}"
-32
View File
@@ -1,32 +0,0 @@
name: Deploy website
# GitHub requires workflow_dispatch to be registered on the default branch.
# The production implementation is the same workflow path on website.
on:
workflow_dispatch:
inputs:
release_version:
description: "Optional product release version associated with this publication"
required: false
default: ""
type: string
release_request:
description: "Optional release workflow correlation ID"
required: false
default: ""
type: string
studio_version:
description: "Studio version that must be present in the generated changelog"
required: false
default: ""
type: string
permissions: {}
jobs:
require-website:
runs-on: ubuntu-latest
steps:
- run: |
echo "::error::Deploy the website with --ref website; production is restricted to that branch."
exit 1
-27
View File
@@ -1,27 +0,0 @@
name: PR author gate
# Pull requests are limited to maintainers and the logins in .github/pr-authors.txt.
# This runs in the base repository's context, checks out the base branch only, and
# never executes pull-request code; the allowlist it reads is the base branch's.
on:
pull_request_target:
types: [opened]
permissions:
contents: read
pull-requests: write
concurrency:
group: pr-author-gate-${{ github.event.pull_request.number }}
cancel-in-progress: false
jobs:
gate:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
persist-credentials: false
- run: node scripts/pr-author-gate.mjs
env:
GH_TOKEN: ${{ github.token }}
+143 -369
View File
@@ -5,12 +5,10 @@ run-name: Release Studio ${{ inputs.tag || github.ref_name }}
# (studio-v*), its own environment gate, and its own rollback catalog at
# dl.reasonix.io/studio/versions.json.
#
# What it must never touch: the root dl.reasonix.io/versions.json, which is the
# desktop line's catalog. Studio artifacts land under the tag prefix, which
# cannot collide because the tag name itself differs. The highest stable tag
# is the repository's GitHub latest and a candidate is a prerelease; the GitHub
# release carries no root latest.json, because the 1.x updaters' last-resort
# URL is releases/latest/download/latest.json and must keep answering 404.
# What it must never touch: the repository-wide "latest" release (every Studio
# release is a prerelease) and the root dl.reasonix.io/versions.json, which is
# the desktop line's catalog. Studio artifacts land under the tag prefix, which
# cannot collide because the tag name itself differs.
#
# Windows artifacts are Authenticode-signed with a Certum SimplySign cloud
# certificate in two stages: every PE file in the application tree, then the
@@ -76,7 +74,8 @@ jobs:
fi
version="${tag#studio-}"
# Semver's prerelease segment, group 4, read from the grammar rather
# than the spelling.
# than the spelling. Every Studio release carries GitHub's prerelease
# flag, so the tag-level distinction needs a name of its own.
candidate=false
if [[ "${version#v}" =~ $semver ]] && [ -n "${BASH_REMATCH[4]}" ]; then
candidate=true
@@ -149,21 +148,10 @@ jobs:
# vocabularies disagree (x64 / amd64) and a reviewer has to see both.
# Absent means the runner's own, which is what Windows and Linux build.
# artifact is the exact name publish downloads this leg's packages by.
# goarch and unpacked belong to the Windows legs: the release
# architecture the package is named for, and the directory
# electron-builder leaves its bundle in (win-unpacked for x64 alone).
include:
- platform: windows/amd64
runner: windows-latest
goarch: amd64
unpacked: win-unpacked
artifact: studio-build-windows-amd64
- platform: windows/arm64
runner: windows-11-arm
ebarch: arm64
goarch: arm64
unpacked: win-arm64-unpacked
artifact: studio-build-windows-arm64
- platform: darwin/amd64
runner: macos-latest
ebarch: x64
@@ -175,9 +163,6 @@ jobs:
- platform: linux/amd64
runner: ubuntu-22.04
artifact: studio-build-linux-amd64
- platform: linux/arm64
runner: ubuntu-22.04-arm
artifact: studio-build-linux-arm64
steps:
# Real-time scanning inspects every file the job writes, and packaging
# writes a quarter of a gigabyte twice. The runner is thrown away after
@@ -200,12 +185,12 @@ jobs:
cache: true
cache-dependency-path: go.sum
- uses: pnpm/action-setup@73e46992e05b19abd31f2f3c20e0e7370d762115 # v6.1.1
- uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413 # v6.1.0
with:
version: 10
run_install: false
- uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
with:
node-version: "24"
cache: pnpm
@@ -213,22 +198,12 @@ jobs:
desktop/frontend-next/pnpm-lock.yaml
desktop/electron/pnpm-lock.yaml
# electron-builder shells out to dpkg-deb through fakeroot for the .deb and
# to rpmbuild for the .rpm, and carries its own NSIS -- the toolchain the
# Wails line installed by hand (GTK, WebKitGTK, nfpm, a pinned NSIS) is
# not this shell's.
# electron-builder shells out to dpkg-deb through fakeroot for the .deb,
# and carries its own NSIS -- the toolchain the Wails line installed by
# hand (GTK, WebKitGTK, nfpm, a pinned NSIS) is not this shell's.
- name: Install Linux packaging deps
if: runner.os == 'Linux'
run: sudo apt-get update && sudo apt-get install -y fakeroot rpm cpio xvfb xauth policykit-1
# The fpm electron-builder carries is an x86_64 program, so on an ARM64
# runner the .deb and .rpm are built with one installed from RubyGems.
- name: Use a native fpm on ARM64 Linux
if: runner.os == 'Linux' && runner.arch == 'ARM64'
run: |
sudo apt-get install -y ruby ruby-dev build-essential
sudo gem install --no-document fpm -v 1.16.0
echo "USE_SYSTEM_FPM=true" >> "$GITHUB_ENV"
run: sudo apt-get update && sudo apt-get install -y fakeroot
# The same Developer ID and notary credentials the desktop line uses: one
# certificate signs any bundle identifier its team owns, so Studio does not
@@ -291,7 +266,6 @@ jobs:
# kernel on a remote machine, and those routes name a release by it.
env:
REASONIX_VERSION: ${{ needs.resolve.outputs.version }}
REASONIX_GOARCH: ${{ matrix.goarch }}
run: pnpm install --frozen-lockfile && pnpm run build:host
# The helper the Linux self-update runs under pkexec, staged where the
@@ -314,14 +288,6 @@ jobs:
if [ -n "${{ matrix.ebarch }}" ]; then arch="--${{ matrix.ebarch }}"; fi
pnpm exec electron-builder --dir $arch -c.extraMetadata.version="${version#v}"
# An emulated x64 program runs on ARM64 Windows, so a bundle that starts
# proves nothing about its architecture: every PE image in it has to be
# built for the architecture the package is named for.
- name: Every PE image is built for the package's architecture
if: runner.os == 'Windows'
shell: bash
run: node desktop/electron/packaging/pe.js "dist/${{ matrix.unpacked }}" "${{ matrix.goarch }}"
# Notarization happens to the bundle, so it sits between the two packaging
# steps the way Authenticode does on Windows. Stapling here is what lets a
# first launch work offline; a ticket Apple has but the bundle does not is
@@ -372,39 +338,14 @@ jobs:
pnpm exec electron-builder --prepackaged "../../$app" $arch \
-c.extraMetadata.version="${version#v}"
- name: Verify both Linux package formats
if: runner.os == 'Linux'
shell: bash
run: |
platform="${{ matrix.platform }}"
arch="${platform#linux/}"
bash scripts/verify-linux-package.sh "dist/ReasonixStudio-linux-$arch.deb" "dist/ReasonixStudio-linux-$arch.rpm" "$arch"
- name: Accept the installed Linux package
if: runner.os == 'Linux'
shell: bash
run: |
platform="${{ matrix.platform }}"
arch="${platform#linux/}"
version="${{ needs.resolve.outputs.version }}"
bash scripts/linux-install-smoke.sh "dist/ReasonixStudio-linux-$arch.deb" "$arch" "${version#v}"
- name: Verify the AppImage
if: runner.os == 'Linux'
shell: bash
run: |
platform="${{ matrix.platform }}"
arch="${platform#linux/}"
bash scripts/verify-linux-appimage.sh "dist/ReasonixStudio-linux-$arch.AppImage" "$arch"
# With signing on, this leg only hands the bundle on: the signing jobs
# run nothing but the signer, and windows-package packages in between.
- name: Upload the unsigned Windows bundle for signing
if: runner.os == 'Windows' && env.SIGN_WINDOWS == 'true'
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: unsigned-studio-windows-${{ matrix.goarch }}-bundle
path: dist/${{ matrix.unpacked }}
name: unsigned-studio-windows-bundle
path: dist/win-unpacked
if-no-files-found: error
retention-days: 1
@@ -415,26 +356,19 @@ jobs:
if: runner.os == 'Windows' && env.SIGN_WINDOWS != 'true'
working-directory: desktop/electron
shell: bash
env:
# The arm64 installer has to be packed with the BCJ filter: 7-Zip
# applies an ARM64 filter to ARM64 executables by default, the NSIS
# extractor cannot decode it, and the installer then exits 0 with
# every executable missing.
ELECTRON_BUILDER_7Z_FILTER: ${{ matrix.goarch == 'arm64' && 'BCJ' || '' }}
run: |
version="${{ needs.resolve.outputs.version }}"
arch="--${{ matrix.ebarch || 'x64' }}"
rm -rf ../../zip-src ../../dist-zip
cp -r ../../dist/${{ matrix.unpacked }} ../../zip-src
pnpm exec electron-builder --prepackaged ../../zip-src --win zip $arch \
cp -r ../../dist/win-unpacked ../../zip-src
pnpm exec electron-builder --prepackaged ../../zip-src --win zip \
-c.extraMetadata.version="${version#v}" -c.directories.output=../../dist-zip &
zip_pid=$!
pnpm exec electron-builder --prepackaged ../../dist/${{ matrix.unpacked }} --win nsis $arch \
pnpm exec electron-builder --prepackaged ../../dist/win-unpacked --win nsis \
-c.extraMetadata.version="${version#v}"
wait "$zip_pid"
mv ../../dist-zip/ReasonixStudio-windows-${{ matrix.goarch }}.zip ../../dist/
mv ../../dist-zip/ReasonixStudio-windows-amd64.zip ../../dist/
- uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
if: runner.os != 'Windows' || env.SIGN_WINDOWS != 'true'
with:
name: ${{ matrix.artifact }}
@@ -448,36 +382,22 @@ jobs:
# package check run between them on their own runners with no secrets. Both
# signing jobs share one group with the Studio smoke test, so no two runs
# drive the same SimplySign account at once.
#
# Each signing stage is a matrix over the Windows architectures with one tree
# per job. max-parallel: 1 keeps the legs strictly serial, which the shared
# certum-signing group needs: a group holds one running and one waiting job,
# so two legs started together would have the waiting one cancelled by the
# next arrival. A failed leg reruns alone; the others' results are kept.
windows-sign-payload:
name: sign the Windows PE files (Certum, ${{ matrix.arch }})
name: sign the Windows PE files (Certum)
needs: [resolve, signing-contract, build]
if: vars.STUDIO_SIGNING_ENABLED == 'true'
runs-on: windows-2022
timeout-minutes: 30
strategy:
fail-fast: false
max-parallel: 1
matrix:
arch: [amd64, arm64]
environment: studio-release
permissions:
contents: read
concurrency:
group: certum-signing
cancel-in-progress: false
# One output per architecture, because matrix legs share a job's output map:
# a leg writes only its own and an empty value never replaces another's.
# Recorded here because no later job can write this job's outputs: the
# installer signing job refuses a payload artifact that no longer hashes to it.
outputs:
payload-digest-amd64: ${{ matrix.arch == 'amd64' && steps.verify.outputs.digest || '' }}
payload-digest-arm64: ${{ matrix.arch == 'arm64' && steps.verify.outputs.digest || '' }}
payload-digest: ${{ steps.verify.outputs.digest }}
steps:
# Only the workflow's own commit is checked out: these jobs run the signer
# and its checks, never anything the release tag builds.
@@ -516,9 +436,9 @@ jobs:
}
if ($failed) { exit 1 }
- uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
name: unsigned-studio-windows-${{ matrix.arch }}-bundle
name: unsigned-studio-windows-bundle
path: signing/app
# Stage one: every PE file in the application tree that Microsoft has
@@ -565,15 +485,14 @@ jobs:
$ErrorActionPreference = 'Stop'
./scripts/verify-windows-authenticode.ps1 `
-PayloadDirectory signing/app `
-Architecture ${{ matrix.arch }} `
-DigestPath "$env:RUNNER_TEMP/payload-digest" `
-ExpectedThumbprint $env:CERTUM_KEY_ID `
-ExpectedSubject $env:EXPECTED_SUBJECT
"digest=$(Get-Content -Raw "$env:RUNNER_TEMP/payload-digest")" >> $env:GITHUB_OUTPUT
- uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: signed-studio-windows-payload-${{ matrix.arch }}
name: signed-studio-windows-payload
path: signing/app
if-no-files-found: error
retention-days: 1
@@ -581,20 +500,10 @@ jobs:
# No environment and no secrets: electron-builder and its dependencies run
# here, on a runner that never held a signing session.
windows-package:
name: package Windows from the signed tree (${{ matrix.arch }})
name: package Windows from the signed tree
needs: [resolve, windows-sign-payload]
runs-on: windows-latest
timeout-minutes: 30
strategy:
fail-fast: false
matrix:
include:
- arch: amd64
ebarch: x64
unpacked: win-unpacked
- arch: arm64
ebarch: arm64
unpacked: win-arm64-unpacked
permissions:
contents: read
steps:
@@ -608,12 +517,12 @@ jobs:
ref: ${{ needs.resolve.outputs.sha }}
persist-credentials: false
- uses: pnpm/action-setup@73e46992e05b19abd31f2f3c20e0e7370d762115 # v6.1.1
- uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413 # v6.1.0
with:
version: 10
run_install: false
- uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
with:
node-version: "24"
cache: pnpm
@@ -625,10 +534,10 @@ jobs:
run: pnpm install --frozen-lockfile
# The signed payload is the whole bundle, so it is packaged as it came.
- uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
name: signed-studio-windows-payload-${{ matrix.arch }}
path: dist/${{ matrix.unpacked }}
name: signed-studio-windows-payload
path: dist/win-unpacked
# Both packages come from the signed tree, so the archive carries the
# signed files too rather than a second unsigned copy. The zip is packed
@@ -636,27 +545,24 @@ jobs:
- name: Package Windows from the signed bundle
working-directory: desktop/electron
shell: bash
env:
# See the build job: the arm64 installer must be packed with BCJ.
ELECTRON_BUILDER_7Z_FILTER: ${{ matrix.arch == 'arm64' && 'BCJ' || '' }}
run: |
version="${{ needs.resolve.outputs.version }}"
rm -rf ../../zip-src ../../dist-zip
cp -r ../../dist/${{ matrix.unpacked }} ../../zip-src
pnpm exec electron-builder --prepackaged ../../zip-src --win zip --${{ matrix.ebarch }} \
cp -r ../../dist/win-unpacked ../../zip-src
pnpm exec electron-builder --prepackaged ../../zip-src --win zip \
-c.extraMetadata.version="${version#v}" -c.directories.output=../../dist-zip &
zip_pid=$!
pnpm exec electron-builder --prepackaged ../../dist/${{ matrix.unpacked }} --win nsis --${{ matrix.ebarch }} \
pnpm exec electron-builder --prepackaged ../../dist/win-unpacked --win nsis \
-c.extraMetadata.version="${version#v}"
wait "$zip_pid"
mv ../../dist-zip/ReasonixStudio-windows-${{ matrix.arch }}.zip ../../dist/
mv ../../dist-zip/ReasonixStudio-windows-amd64.zip ../../dist/
- uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: packaged-studio-windows-${{ matrix.arch }}
name: packaged-studio-windows
path: |
dist/ReasonixStudio-windows-${{ matrix.arch }}-installer.exe
dist/ReasonixStudio-windows-${{ matrix.arch }}.zip
dist/ReasonixStudio-windows-amd64-installer.exe
dist/ReasonixStudio-windows-amd64.zip
if-no-files-found: error
retention-days: 1
@@ -665,35 +571,29 @@ jobs:
# secrets, never in a job that holds the Certum credentials. Its outputs are
# the hashes of the two files it checked, which the signing job requires.
windows-verify-package:
name: check the Windows packages against the signed tree (${{ matrix.arch }})
name: check the Windows packages against the signed tree
needs: [resolve, windows-sign-payload, windows-package]
runs-on: windows-2022
timeout-minutes: 30
strategy:
fail-fast: false
matrix:
arch: [amd64, arm64]
permissions:
contents: read
outputs:
installer-sha256-amd64: ${{ matrix.arch == 'amd64' && steps.check.outputs.installer-sha256 || '' }}
installer-sha256-arm64: ${{ matrix.arch == 'arm64' && steps.check.outputs.installer-sha256 || '' }}
archive-sha256-amd64: ${{ matrix.arch == 'amd64' && steps.check.outputs.archive-sha256 || '' }}
archive-sha256-arm64: ${{ matrix.arch == 'arm64' && steps.check.outputs.archive-sha256 || '' }}
installer-sha256: ${{ steps.check.outputs.installer-sha256 }}
archive-sha256: ${{ steps.check.outputs.archive-sha256 }}
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
ref: ${{ github.workflow_sha }}
persist-credentials: false
- uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
name: signed-studio-windows-payload-${{ matrix.arch }}
name: signed-studio-windows-payload
path: payload
- uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
name: packaged-studio-windows-${{ matrix.arch }}
name: packaged-studio-windows
path: dist
# 7-Zip is the runner image's, not the tag's.
@@ -701,26 +601,20 @@ jobs:
id: check
shell: pwsh
env:
PAYLOAD_DIGEST: ${{ needs.windows-sign-payload.outputs[format('payload-digest-{0}', matrix.arch)] }}
PAYLOAD_DIGEST: ${{ needs.windows-sign-payload.outputs.payload-digest }}
run: |
./scripts/verify-windows-package.ps1 `
-PayloadDirectory payload `
-ExpectedPayloadDigest $env:PAYLOAD_DIGEST `
-InstallerPath "dist/ReasonixStudio-windows-${{ matrix.arch }}-installer.exe" `
-PortableArchivePath "dist/ReasonixStudio-windows-${{ matrix.arch }}.zip" `
-OutputPath $env:GITHUB_OUTPUT `
-Architecture ${{ matrix.arch }}
-InstallerPath "dist/ReasonixStudio-windows-amd64-installer.exe" `
-PortableArchivePath "dist/ReasonixStudio-windows-amd64.zip" `
-OutputPath $env:GITHUB_OUTPUT
windows-sign-installer:
name: sign the Windows installer (Certum, ${{ matrix.arch }})
name: sign the Windows installer (Certum)
needs: [resolve, windows-package, windows-verify-package]
runs-on: windows-2022
timeout-minutes: 30
strategy:
fail-fast: false
max-parallel: 1
matrix:
arch: [amd64, arm64]
environment: studio-release
permissions:
contents: read
@@ -729,8 +623,7 @@ jobs:
cancel-in-progress: false
# The signed installer's hash, which publish requires of what it downloads.
outputs:
installer-sha256-amd64: ${{ matrix.arch == 'amd64' && steps.signed.outputs.installer-sha256 || '' }}
installer-sha256-arm64: ${{ matrix.arch == 'arm64' && steps.signed.outputs.installer-sha256 || '' }}
installer-sha256: ${{ steps.signed.outputs.installer-sha256 }}
steps:
# Only the workflow's own commit is checked out: these jobs run the signer
# and its checks, never anything the release tag builds.
@@ -769,9 +662,9 @@ jobs:
}
if ($failed) { exit 1 }
- uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
name: packaged-studio-windows-${{ matrix.arch }}
name: packaged-studio-windows
path: dist
# Only the two files windows-verify-package checked, byte for byte. This
@@ -780,13 +673,13 @@ jobs:
- name: Require the checked packages
shell: pwsh
env:
INSTALLER_SHA256: ${{ needs.windows-verify-package.outputs[format('installer-sha256-{0}', matrix.arch)] }}
ARCHIVE_SHA256: ${{ needs.windows-verify-package.outputs[format('archive-sha256-{0}', matrix.arch)] }}
INSTALLER_SHA256: ${{ needs.windows-verify-package.outputs.installer-sha256 }}
ARCHIVE_SHA256: ${{ needs.windows-verify-package.outputs.archive-sha256 }}
run: |
$ErrorActionPreference = 'Stop'
$expected = [ordered]@{
'ReasonixStudio-windows-${{ matrix.arch }}-installer.exe' = $env:INSTALLER_SHA256
'ReasonixStudio-windows-${{ matrix.arch }}.zip' = $env:ARCHIVE_SHA256
'ReasonixStudio-windows-amd64-installer.exe' = $env:INSTALLER_SHA256
'ReasonixStudio-windows-amd64.zip' = $env:ARCHIVE_SHA256
}
$present = @(Get-ChildItem -LiteralPath dist -Recurse -Force | ForEach-Object { $_.Name })
if (Compare-Object -CaseSensitive @($expected.Keys) $present) { throw "dist holds $($present -join ', ')" }
@@ -808,7 +701,7 @@ jobs:
shell: pwsh
env:
CERTUM_KEY_ID: ${{ secrets.CERTUM_KEY_ID }}
run: ./scripts/sign-certum.ps1 -FilePath dist/ReasonixStudio-windows-${{ matrix.arch }}-installer.exe
run: ./scripts/sign-certum.ps1 -FilePath dist/ReasonixStudio-windows-amd64-installer.exe
# The session signs for any process on the runner while it is up.
- name: Disconnect from Certum
@@ -824,7 +717,7 @@ jobs:
EXPECTED_SUBJECT: ${{ vars.STUDIO_SIGNING_SUBJECT }}
run: |
$ErrorActionPreference = 'Stop'
$installer = 'dist/ReasonixStudio-windows-${{ matrix.arch }}-installer.exe'
$installer = 'dist/ReasonixStudio-windows-amd64-installer.exe'
./scripts/verify-windows-authenticode.ps1 `
-FilePath $installer `
-ExpectedThumbprint $env:CERTUM_KEY_ID `
@@ -832,62 +725,13 @@ jobs:
$hash = (Get-FileHash -Algorithm SHA256 -LiteralPath $installer).Hash.ToLowerInvariant()
"installer-sha256=$hash" >> $env:GITHUB_OUTPUT
- uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: studio-windows-signed-${{ matrix.arch }}
name: studio-windows-signed
path: dist/ReasonixStudio-*
if-no-files-found: error
retention-days: 7
# Installs each Windows installer where it is meant to run (the same script
# the Studio workflow runs on pull requests, against the unsigned installer).
# The package check unpacks with the runner's modern 7-Zip, which reads
# archives the NSIS extractor cannot, so only running the installer shows that
# it installs every executable. No environment and no secrets.
windows-install-smoke:
name: install the Windows package (${{ matrix.arch }})
needs: [resolve, build, windows-sign-installer]
if: >-
${{ !cancelled() && needs.resolve.result == 'success' && needs.build.result == 'success' &&
(needs.windows-sign-installer.result == 'success' ||
(vars.STUDIO_SIGNING_ENABLED != 'true' && needs.windows-sign-installer.result == 'skipped')) }}
runs-on: ${{ matrix.runner }}
timeout-minutes: 30
permissions:
contents: read
strategy:
fail-fast: false
matrix:
include:
- arch: amd64
runner: windows-latest
- arch: arm64
runner: windows-11-arm
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
ref: ${{ needs.resolve.outputs.sha }}
persist-credentials: false
- uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
if: needs.windows-sign-installer.result == 'success'
with:
name: studio-windows-signed-${{ matrix.arch }}
path: dist
- uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
if: needs.windows-sign-installer.result != 'success'
with:
name: studio-build-windows-${{ matrix.arch }}
path: dist
- name: Install, check and uninstall
shell: pwsh
run: |
./scripts/install-smoke.ps1 `
-Installer "dist/ReasonixStudio-windows-${{ matrix.arch }}-installer.exe" `
-Architecture ${{ matrix.arch }}
# The kernel a Studio installs on another machine. This line publishes it
# under its own tag, because the install routes name a release by the version
# the local build reports — and a line that ships no archive leaves every one
@@ -908,43 +752,24 @@ jobs:
go-version-file: go.mod
cache: true
# The page every archive carries beside the executable, built once here
# rather than once per target: serve.FindPage looks for
# frontend-next/dist next to the binary.
- uses: pnpm/action-setup@73e46992e05b19abd31f2f3c20e0e7370d762115 # v6.1.1
with:
version: 10
run_install: false
- uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7
with:
node-version: "24"
cache: pnpm
cache-dependency-path: desktop/frontend-next/pnpm-lock.yaml
- name: Build the page
working-directory: desktop/frontend-next
run: pnpm install --frozen-lockfile && pnpm build
# Every target the release line claims to support, from one runner: the
# CLI is CGO-free, so none of them needs a toolchain of its own.
- name: Cross-compile and archive
env:
VERSION: ${{ needs.resolve.outputs.version }}
SHA: ${{ needs.resolve.outputs.sha }}
PAGE_DIR: desktop/frontend-next/dist
run: |
set -euo pipefail
mkdir -p dist
built="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
pkg=reasonix/internal/tools/productdocs
for target in darwin/amd64 darwin/arm64 linux/amd64 linux/arm64 windows/amd64 windows/arm64; do
os="${target%/*}"; arch="${target#*/}"; ext=""
[ "$os" = windows ] && ext=".exe"
CGO_ENABLED=0 GOOS="$os" GOARCH="$arch" go build -trimpath \
-ldflags "-s -w -X main.version=$VERSION -X main.gitCommit=${SHA:0:12} -X main.buildTimeUTC=$built -X $pkg.linkedVersion=$VERSION -X $pkg.linkedRevision=$SHA" \
-o "reasonix$ext" ./cmd/reasonix
scripts/package-cli-archive.sh "$os" "$arch" "reasonix$ext" "$PAGE_DIR" dist
-ldflags "-s -w -X main.version=$VERSION" -o "reasonix$ext" ./cmd/reasonix
if [ "$os" = windows ]; then
zip -qj "dist/reasonix-$os-$arch.zip" "reasonix$ext"
else
tar czf "dist/reasonix-$os-$arch.tar.gz" "reasonix$ext"
fi
rm -f "reasonix$ext"
done
# The digest list the remote verifies against. Resolved on the
@@ -956,9 +781,15 @@ jobs:
# A release that published the archives without the digest list is worse
# than one that published neither: the routes resolve SHA256SUMS first.
- name: Check the set is complete
run: python3 scripts/check-cli-archives.py dist
run: |
set -euo pipefail
for want in reasonix-darwin-amd64.tar.gz reasonix-darwin-arm64.tar.gz \
reasonix-linux-amd64.tar.gz reasonix-linux-arm64.tar.gz \
reasonix-windows-amd64.zip reasonix-windows-arm64.zip SHA256SUMS; do
[ -f "dist/$want" ] || { echo "::error::no dist/$want"; exit 1; }
done
- uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: cli-archives
path: dist/*
@@ -967,13 +798,12 @@ jobs:
publish:
name: sign, publish and mirror
needs: [resolve, build, cli, windows-sign-payload, windows-package, windows-verify-package, windows-sign-installer, windows-install-smoke]
needs: [resolve, build, cli, windows-sign-payload, windows-package, windows-verify-package, windows-sign-installer]
# The Windows signing jobs may be skipped only while the switch is off; with
# it on, any of them failing, cancelled or absent blocks the release.
if: >-
${{ !cancelled() && needs.resolve.result == 'success' && needs.build.result == 'success' &&
needs.cli.result == 'success' && needs.windows-install-smoke.result == 'success' &&
((needs.windows-sign-payload.result == 'success' &&
needs.cli.result == 'success' && ((needs.windows-sign-payload.result == 'success' &&
needs.windows-package.result == 'success' && needs.windows-verify-package.result == 'success' &&
needs.windows-sign-installer.result == 'success') ||
(vars.STUDIO_SIGNING_ENABLED != 'true' && needs.windows-sign-payload.result == 'skipped' &&
@@ -1017,88 +847,63 @@ jobs:
# Every package by its exact name, never a pattern: any job without
# secrets can upload an artifact, and a pattern would publish it. Windows
# comes from the installer signing job when it ran, else the build leg.
- uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
name: studio-build-darwin-amd64
path: staging/studio-build-darwin-amd64
- uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
name: studio-build-darwin-arm64
path: staging/studio-build-darwin-arm64
- uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
name: studio-build-linux-amd64
path: staging/studio-build-linux-amd64
- uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
with:
name: studio-build-linux-arm64
path: staging/studio-build-linux-arm64
- uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
if: needs.windows-sign-installer.result == 'success'
with:
name: studio-windows-signed-amd64
path: staging/studio-windows-amd64
- uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
if: needs.windows-sign-installer.result == 'success'
with:
name: studio-windows-signed-arm64
path: staging/studio-windows-arm64
name: studio-windows-signed
path: staging/studio-windows
# Any job can upload an artifact, so the signed Windows packages are
# taken only if they are the bytes the signing jobs recorded: each
# installer as signed, each zip as windows-verify-package checked it.
# taken only if they are the bytes the signing jobs recorded: the
# installer as signed, the zip as windows-verify-package checked it.
- name: Require the recorded Windows packages
if: needs.windows-sign-installer.result == 'success'
env:
INSTALLER_SHA256_AMD64: ${{ needs.windows-sign-installer.outputs[format('installer-sha256-{0}', 'amd64')] }}
ARCHIVE_SHA256_AMD64: ${{ needs.windows-verify-package.outputs[format('archive-sha256-{0}', 'amd64')] }}
INSTALLER_SHA256_ARM64: ${{ needs.windows-sign-installer.outputs[format('installer-sha256-{0}', 'arm64')] }}
ARCHIVE_SHA256_ARM64: ${{ needs.windows-verify-package.outputs[format('archive-sha256-{0}', 'arm64')] }}
INSTALLER_SHA256: ${{ needs.windows-sign-installer.outputs.installer-sha256 }}
ARCHIVE_SHA256: ${{ needs.windows-verify-package.outputs.archive-sha256 }}
run: |
check() {
local file="$1/$2" want="$3" got
local file="staging/studio-windows/$1" want="$2" got
if [[ ! "$want" =~ ^[0-9a-f]{64}$ ]]; then
echo "::error title=studio-signing.package-hash-missing::no recorded SHA-256 for $2"
echo "::error title=studio-signing.package-hash-missing::no recorded SHA-256 for $1"
return 1
fi
got="$(sha256sum "$file" | awk '{print $1}')" || return 1
if [ "$got" != "$want" ]; then
echo "::error title=studio-signing.package-hash-mismatch::$2 is $got, the signing jobs recorded $want"
echo "::error title=studio-signing.package-hash-mismatch::$1 is $got, the signing jobs recorded $want"
return 1
fi
}
for arch in amd64 arm64; do
dir="staging/studio-windows-$arch"
files="$(cd "$dir" && find . -type f | sed 's|^\./||' | LC_ALL=C sort | tr '\n' ' ')"
if [ "$files" != "ReasonixStudio-windows-$arch-installer.exe ReasonixStudio-windows-$arch.zip " ]; then
echo "::error title=studio-signing.package-set-mismatch::studio-windows-signed-$arch holds: $files"
exit 1
fi
upper="${arch^^}"
installer="INSTALLER_SHA256_$upper"
archive="ARCHIVE_SHA256_$upper"
check "$dir" "ReasonixStudio-windows-$arch-installer.exe" "${!installer}"
check "$dir" "ReasonixStudio-windows-$arch.zip" "${!archive}"
done
files="$(cd staging/studio-windows && find . -type f | sed 's|^\./||' | LC_ALL=C sort | tr '\n' ' ')"
if [ "$files" != "ReasonixStudio-windows-amd64-installer.exe ReasonixStudio-windows-amd64.zip " ]; then
echo "::error title=studio-signing.package-set-mismatch::studio-windows-signed holds: $files"
exit 1
fi
check ReasonixStudio-windows-amd64-installer.exe "$INSTALLER_SHA256"
check ReasonixStudio-windows-amd64.zip "$ARCHIVE_SHA256"
- uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
if: needs.windows-sign-installer.result != 'success'
with:
name: studio-build-windows-amd64
path: staging/studio-windows-amd64
path: staging/studio-windows
- uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
if: needs.windows-sign-installer.result != 'success'
with:
name: studio-build-windows-arm64
path: staging/studio-windows-arm64
- uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
name: cli-archives
path: staging/cli-archives
@@ -1119,7 +924,30 @@ jobs:
done < <(find staging/studio-* -type f -name 'ReasonixStudio-*' -print0
find staging/cli-archives -type f -print0)
ls -la dist
bash scripts/verify-studio-artifacts.sh dist
# Two Windows artifacts (installer, portable zip), two per macOS
# architecture (dmg, zip) and the Linux .deb. Fewer means an artifact
# was lost between jobs, and a partial release is worse than a failed
# one.
count=$(find dist -type f -name 'ReasonixStudio-*' | wc -l)
if [ "$count" -lt 7 ]; then
echo "::error::expected 7 artifacts (Windows installer + zip, macOS dmg + zip per architecture, Linux .deb), found $count"
exit 1
fi
# Each platform's update channel, and there is exactly one per
# platform: dpkg on Linux, the next installer on Windows, and the
# bundle swapped out of the archive on macOS. Missing one degrades the
# release silently instead of failing it — it strands that platform on
# the version it is running.
# Each macOS architecture by name, not darwin-*: a glob passes on
# whichever one shipped, and the missing one is a platform left on
# the version it is running with nothing saying so.
for required in 'ReasonixStudio-linux-*.deb' 'ReasonixStudio-windows-*-installer.exe' \
'ReasonixStudio-darwin-amd64.zip' 'ReasonixStudio-darwin-arm64.zip'; do
find dist -type f -name "$required" | grep -q . || {
echo "::error::no $required in the release; its packager did not run"
exit 1
}
done
# The same minisign key the desktop line uses, because the public key that
# verifies it is compiled into the client (update.PublicKey).
@@ -1162,14 +990,12 @@ jobs:
if [ "$HAS_R2" = "true" ]; then delta_dir=../delta; fi
go run ./cmd/studio-manifest ../dist "${{ needs.resolve.outputs.version }}" "${{ needs.resolve.outputs.tag }}" $delta_dir
# The manifest stays out of the GitHub release (it is mirrored to R2
# under <tag>/ below), and a stable release takes the repository's latest
# only when it is the highest stable studio tag.
# Always a prerelease: this line is not the public product line, and
# GitHub never promotes a prerelease to the repository-wide "latest".
- name: Publish GitHub release
env:
GH_TOKEN: ${{ github.token }}
TAG: ${{ needs.resolve.outputs.tag }}
CANDIDATE: ${{ needs.resolve.outputs.candidate }}
VERSION: ${{ needs.resolve.outputs.version }}
SIGNED_WINDOWS: ${{ needs.windows-sign-installer.result == 'success' }}
run: |
@@ -1207,23 +1033,18 @@ jobs:
fi
{
printf '%s\n\n' '---'
printf "**macOS** 已用 Developer ID 签名并公证,打开时不需要清除隔离属性。%s**Linux** Debian 系从 \`.deb\` 安装,Fedora、RHEL、openSUSE 等从 \`.rpm\` 安装(\`sudo dnf install ./ReasonixStudio-linux-amd64.rpm\`)。x86_64 用文件名带 amd64 的包,aarch64(统信 UOS、麒麟等)用带 arm64 的包。也提供 \`.AppImage\`(只提供下载,不会自更新;需要 FUSE 2,或用 \`--appimage-extract-and-run\` 启动)。\n\n" "$windows"
printf "%s\n" "三个平台都能自更新:Linux 的 \`.deb\` 走自己的包(\`.rpm\` 的自更新尚未提供,请手动下载新包),Windows 运行下一个安装器,macOS 替换自身 bundle。"
printf "**macOS** 已用 Developer ID 签名并公证,打开时不需要清除隔离属性。%s**Linux** 从 \`.deb\` 安装。\n\n" "$windows"
printf '%s\n' '三个平台都能自更新:Linux 走自己的包,Windows 运行下一个安装器,macOS 替换自身 bundle。'
} >> "$notes"
assets=()
for f in dist/*; do
[ "$(basename "$f")" = latest.json ] || assets+=("$f")
done
if gh release view "$TAG" >/dev/null 2>&1; then
gh release edit "$TAG" --notes-file "$notes"
gh release upload "$TAG" "${assets[@]}" --clobber
gh release upload "$TAG" dist/* --clobber
else
decided="$(git tag -l 'studio-v*' | bash scripts/studio-release-flags.sh "$TAG" "$CANDIDATE")"
mapfile -t flags <<<"$decided"
gh release create "$TAG" "${assets[@]}" \
gh release create "$TAG" dist/* \
--title "Reasonix Studio $VERSION" \
--notes-file "$notes" \
"${flags[@]}"
--prerelease \
--latest=false
fi
- name: Mirror to R2 and update the Studio catalog
@@ -1267,22 +1088,6 @@ jobs:
exit 0
fi
# The notes object goes up before the catalog entry that names it, so
# no client reads a pointer to an object that is not there yet. Notes
# are optional: a failure here publishes the version without a pointer
# rather than keeping the new version out of the catalog.
notes_key="studio/notes/${VERSION#v}.md"
notes_arg=()
if aws s3 cp "$RUNNER_TEMP/version-notes.md" "s3://${R2_BUCKET}/${notes_key}" \
--endpoint-url "$endpoint" \
--content-type "text/markdown; charset=utf-8" \
--cache-control "public, max-age=31536000, immutable" &&
aws s3api head-object --bucket "$R2_BUCKET" --key "$notes_key" --endpoint-url "$endpoint" >/dev/null; then
notes_arg=(20 "https://dl.reasonix.io/${notes_key}")
else
echo "::warning::release notes for ${VERSION} were not uploaded to the mirror; the catalog entry is published without notes"
fi
# Studio's own catalog. Writing the ROOT versions.json here would hand
# desktop users Studio releases, so the prefix is not optional.
current=/tmp/studio-versions-current.json
@@ -1294,7 +1099,7 @@ jobs:
echo "no existing Studio catalog; starting a new one"
fi
bash scripts/update-versions-index.sh "$source" "$VERSION" "$TAG" studio \
"$(date -u +%Y-%m-%dT%H:%M:%SZ)" "${notes_arg[@]}" > "$next"
"$(date -u +%Y-%m-%dT%H:%M:%SZ)" > "$next"
jq -e --arg v "$VERSION" '.versions | map(select(.version == $v)) | length == 1' "$next" >/dev/null
aws s3 cp "$next" "s3://${R2_BUCKET}/studio/versions.json" \
--endpoint-url "$endpoint" \
@@ -1302,25 +1107,6 @@ jobs:
--cache-control "public, max-age=300, stale-if-error=86400"
echo "Studio catalog -> $VERSION"
website:
name: publish the Studio website changelog
needs: [resolve, publish]
if: github.repository == 'esengine/DeepSeek-Reasonix' && needs.resolve.outputs.candidate != 'true'
runs-on: ubuntu-latest
timeout-minutes: 30
permissions:
contents: read
actions: write
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
ref: ${{ github.workflow_sha }}
- name: Deploy and verify the website changelog
env:
GH_TOKEN: ${{ github.token }}
VERSION: ${{ needs.resolve.outputs.version }}
run: bash scripts/publish-studio-changelog.sh
# A CLI channel that moves while 1.x still publishes would have two lines
# writing it, so every CLI job waits for this check. It runs no repository code.
cli-gate:
@@ -1425,13 +1211,13 @@ jobs:
# The CLI's install channels, npm's `reasonix` and the Homebrew tap. The 1.x
# line owns them until this one takes the CLI over; the repository variable
# STUDIO_PUBLISHES_CLI=true hands them here. They run once cli-pointer has
# created the CLI release, because the cask points at its archives. A candidate
# goes to npm's `next` tag and leaves the tap alone: brew has no prerelease
# channel, so a candidate cask would become the default install.
# STUDIO_PUBLISHES_CLI=true hands them here. They run once the release
# exists, because the cask points at its archives. A candidate goes to npm's
# `next` tag and leaves the tap alone: brew has no prerelease channel, so a
# candidate cask would become the default install.
cli-channels:
name: publish the CLI to npm and Homebrew
needs: [resolve, publish, cli-gate, cli-tag, cli-pointer]
needs: [resolve, publish, cli-gate, cli-tag]
if: vars.STUDIO_PUBLISHES_CLI == 'true'
runs-on: ubuntu-latest
# id-token lets npm publish --provenance attach a Sigstore attestation; the
@@ -1447,7 +1233,7 @@ jobs:
with:
go-version-file: go.mod
cache: true
- uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
with:
node-version: "24"
registry-url: https://registry.npmjs.org
@@ -1456,7 +1242,7 @@ jobs:
NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
VERSION: ${{ needs.resolve.outputs.version }}
run: node npm/build.mjs "$VERSION" --publish
- uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
if: needs.resolve.outputs.candidate != 'true'
with:
name: cli-archives
@@ -1470,18 +1256,6 @@ jobs:
run: |
set -euo pipefail
git clone --depth 1 "https://x-access-token:${TAP_TOKEN}@github.com/esengine/homebrew-reasonix.git" tap
decision="$(node scripts/brew-cask-guard.mjs tap/Casks/reasonix.rb "${VERSION#v}")"
case "$decision" in
write) ;;
skip)
echo "the tap's cask is newer than ${VERSION#v}; leaving it"
exit 0
;;
*)
echo "::error::unexpected cask guard decision: '$decision'"
exit 1
;;
esac
bash scripts/render-brew-cask.sh "${VERSION#v}" "$TAG" cli-archives/SHA256SUMS > tap/Casks/reasonix.rb
cd tap
git add Casks/reasonix.rb
@@ -1510,7 +1284,7 @@ jobs:
with:
ref: ${{ needs.resolve.outputs.sha }}
persist-credentials: false
- uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
name: cli-archives
path: cli-archives
+1 -1
View File
@@ -43,7 +43,7 @@ jobs:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
fetch-depth: 0
- uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
with:
node-version: '22'
- name: Test sweeper
@@ -41,12 +41,12 @@ jobs:
cache: true
cache-dependency-path: go.sum
- uses: pnpm/action-setup@73e46992e05b19abd31f2f3c20e0e7370d762115 # v6.1.1
- uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413 # v6.1.0
with:
version: 10
run_install: false
- uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
with:
node-version: "24"
cache: pnpm
@@ -99,7 +99,7 @@ jobs:
-c.extraMetadata.version="${{ steps.version.outputs.version }}"
sha256sum ../../dist/ReasonixStudio-windows-amd64-installer.exe | tee -a "$GITHUB_STEP_SUMMARY"
- uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: studio-selftest-windows-${{ steps.version.outputs.version }}
path: dist/ReasonixStudio-windows-amd64-installer.exe
+63 -472
View File
@@ -16,7 +16,7 @@ permissions:
concurrency:
group: studio-${{ github.ref }}
cancel-in-progress: ${{ github.event_name == 'pull_request' || github.ref == 'refs/heads/studio' }}
cancel-in-progress: true
jobs:
frontend:
@@ -28,12 +28,12 @@ jobs:
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
- uses: pnpm/action-setup@73e46992e05b19abd31f2f3c20e0e7370d762115 # v6.1.1
- uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413 # v6.1.0
with:
version: 10
run_install: false
- uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
with:
node-version: "24"
cache: pnpm
@@ -81,243 +81,9 @@ jobs:
exit 1
fi
# Real-Chromium guards that run without a kernel. Only the ones that passed
# three consecutive local runs are wired; the rest stay manual (see
# desktop/frontend-next/perf/README.md). Skipped unless the SPA changed.
browser-guards:
name: browser guards
runs-on: ubuntu-latest
timeout-minutes: 25
defaults:
run:
working-directory: desktop/frontend-next
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
fetch-depth: 0
- name: SPA changed?
id: spa
working-directory: .
run: |
if [ "${{ github.event_name }}" = "pull_request" ] &&
git diff --quiet "origin/${{ github.base_ref }}...HEAD" -- desktop/frontend-next; then
echo "run=false" >> "$GITHUB_OUTPUT"
else
echo "run=true" >> "$GITHUB_OUTPUT"
fi
- if: steps.spa.outputs.run == 'true'
uses: pnpm/action-setup@73e46992e05b19abd31f2f3c20e0e7370d762115 # v6.1.1
with:
version: 10
run_install: false
- if: steps.spa.outputs.run == 'true'
uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7
with:
node-version: "24"
cache: pnpm
cache-dependency-path: desktop/frontend-next/pnpm-lock.yaml
- if: steps.spa.outputs.run == 'true'
name: Install
run: pnpm install --frozen-lockfile
- if: steps.spa.outputs.run == 'true'
name: Build the perf harness and install Chromium
run: |
pnpm exec vite build --config vite.perf.config.ts
pnpm exec playwright install --with-deps chromium
- if: steps.spa.outputs.run == 'true'
name: Guards
run: |
python3 -m http.server 4399 --directory dist-perf >/dev/null 2>&1 &
for _ in $(seq 30); do curl -sf http://localhost:4399/perf.html >/dev/null && break; sleep 1; done
node perf/intrinsic.mjs
node perf/firstscreen-fonts.mjs
node perf/contrast.mjs
node perf/stickyfold.mjs
node perf/crumb.mjs
node perf/hoverradius.mjs
node perf/jump-ends.mjs
node perf/modelrows.mjs
node perf/model-menu-badge.mjs
node perf/explorersize.mjs
node perf/git-panel.mjs
node perf/session-move-to.mjs
node perf/shell-refused.mjs
node perf/external-origin.mjs
node perf/rail-search-hold.mjs
node perf/rail-toggle-panes.mjs
node perf/session-move.mjs
node perf/session-ref.mjs
node perf/browser-full.mjs
node perf/mod-ui.mjs
node perf/codeeditor-size.mjs
node perf/composer-reflow.mjs
node perf/composer-toolbar-row.mjs
node perf/control-icons.mjs
node perf/nav-selection.mjs
node perf/action-icons.mjs
node perf/rowhit.mjs
node perf/prefs-wide-centered.mjs
node perf/settings-fullscreen.mjs
node perf/settings-page-column.mjs
node perf/sidebar-tools.mjs
node perf/select-alignment.mjs
node perf/provider-fields.mjs
node perf/thought-clock.mjs
node perf/tab-pip.mjs
node perf/syntax-tokens.mjs
node perf/tab-close-inside.mjs
node perf/context-card-hover.mjs
node perf/sessquick.mjs
node perf/sessionrows.mjs
node perf/session-row-marker.mjs
node perf/session-selected-row.mjs
node perf/wheel.mjs
node perf/branch-tooltip.mjs
node perf/branch-alignment.mjs
node perf/branch-menu.mjs
node perf/image-preview.mjs
node perf/user-images.mjs
node perf/toast-anchor.mjs
node perf/addfolder-rest.mjs
node perf/scroll-jump.mjs
node perf/add-project-selects.mjs
node perf/project-header.mjs
node perf/setup-defer.mjs
node perf/narrow.mjs
node perf/settings-rows.mjs
node perf/narrow-steps.mjs
node perf/job-names.mjs
node perf/mcp-actions.mjs
node perf/web-access.mjs
node perf/peers-settings.mjs
node perf/output-cap.mjs
node perf/tool-output-clip.mjs
node perf/agent-run-tab.mjs
node perf/changes-cap.mjs
node perf/session-menu-fit.mjs
node perf/rail-rows-typography.mjs
node perf/settings-nav-fit.mjs
node perf/rail-rows-align.mjs
node perf/progress-tail.mjs
node perf/trust-wait.mjs
node perf/nest-nodes.mjs
node perf/provider-ports.mjs
node perf/peer-threads.mjs
node perf/text-size.mjs
node perf/fs-scale.mjs
node perf/composer-feel.mjs
node perf/side-panel.mjs
node perf/card-shell.mjs
node perf/line-cards.mjs
node perf/detail-panel.mjs
node perf/brand-lockup.mjs
node perf/notice-line.mjs
node perf/brought-line.mjs
node perf/modal-focus.mjs
node perf/focus-ring.mjs
node perf/narrow-forms.mjs
node perf/i18n-rendered.mjs
node perf/narrow-rail.mjs
node perf/selfcheck.mjs branch-tooltip
- name: Keep UI control previews
if: always() && steps.spa.outputs.run == 'true'
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
with:
name: studio-ui-control-previews
path: |
desktop/frontend-next/perf/shots/control-icons-*.png
desktop/frontend-next/perf/shots/nav-selection-*.png
desktop/frontend-next/perf/shots/settings-full-*.png
desktop/frontend-next/perf/shots/sidebar-tools-*.png
desktop/frontend-next/perf/shots/select-alignment-*.png
desktop/frontend-next/perf/shots/provider-fields-*.png
desktop/frontend-next/perf/shots/thought-clock-*.png
retention-days: 7
# What a hosted page does over the relay, end to end: the production bundle,
# the real Studio host and kernel, a real Chromium; only the accounts API and
# the relay are stand-ins (tools/relayfake). Event-transport changes are
# judged here, because a unit test of one side cannot see the other.
relay-harness:
name: relay harness
runs-on: ubuntu-latest
timeout-minutes: 15
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
fetch-depth: 0
- name: Transport changed?
id: scope
env:
BASE_REF: ${{ github.base_ref }}
run: |
paths="desktop/frontend-next internal/frontend/serve internal/platform/remotecloud internal/platform/account tools/relayfake cmd/reasonix-studio-host"
if [ "$GITHUB_EVENT_NAME" = "pull_request" ] &&
git diff --quiet "origin/$BASE_REF...HEAD" -- $paths; then
echo "run=false" >> "$GITHUB_OUTPUT"
else
echo "run=true" >> "$GITHUB_OUTPUT"
fi
- if: steps.scope.outputs.run == 'true'
uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7
with:
go-version-file: go.mod
cache: true
- if: steps.scope.outputs.run == 'true'
uses: pnpm/action-setup@73e46992e05b19abd31f2f3c20e0e7370d762115 # v6.1.1
with:
version: 10
run_install: false
- if: steps.scope.outputs.run == 'true'
uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7
with:
node-version: "24"
cache: pnpm
cache-dependency-path: desktop/frontend-next/pnpm-lock.yaml
- if: steps.scope.outputs.run == 'true'
name: Build the host, the stand-ins and the page
run: |
go build -o desktop/frontend-next/relay/bin/studio-host ./cmd/reasonix-studio-host
go build -o desktop/frontend-next/relay/bin/relayfake ./tools/relayfake
cd desktop/frontend-next
pnpm install --frozen-lockfile
pnpm exec vite build --outDir dist-relay --emptyOutDir
pnpm exec playwright install --with-deps chromium
- if: steps.scope.outputs.run == 'true'
name: Scenarios (hosted page over the relay; local page on a flapping network)
working-directory: desktop/frontend-next
env:
RELAY_HOST_BIN: ${{ github.workspace }}/desktop/frontend-next/relay/bin/studio-host
RELAY_FAKE_BIN: ${{ github.workspace }}/desktop/frontend-next/relay/bin/relayfake
RELAY_DIST: ${{ github.workspace }}/desktop/frontend-next/dist-relay
run: |
node relay/live.mjs
node relay/storm.mjs
# A pull request with no platform-relevant change (scripts/ci-platform-scope.sh)
# moves the Windows and macOS legs onto an ubuntu runner with every step
# skipped, so the check names still report without a scarce runner. A push to
# studio, or a failed `scope`, runs all three.
desktop:
name: desktop (${{ matrix.os }})
needs: scope
if: always()
runs-on: ${{ github.event_name == 'pull_request' && needs.scope.outputs.platform == 'false' && matrix.os != 'ubuntu-22.04' && 'ubuntu-latest' || matrix.os }}
env:
RUN_STEPS: ${{ github.event_name != 'pull_request' || needs.scope.outputs.platform != 'false' || matrix.os == 'ubuntu-22.04' }}
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
matrix:
@@ -326,11 +92,9 @@ jobs:
run:
working-directory: desktop
steps:
- if: env.RUN_STEPS == 'true'
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
- if: env.RUN_STEPS == 'true'
uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7
- uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7
with:
go-version-file: desktop/go.mod
cache: true
@@ -343,97 +107,20 @@ jobs:
# carries build dependencies of its own.
# Same command as `make studio-test`; this job is already in desktop/.
- name: go test
if: env.RUN_STEPS == 'true'
run: go test ./...
- if: env.RUN_STEPS == 'true'
uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7
with:
node-version: "24"
- name: CUA process and packaging contracts
if: env.RUN_STEPS == 'true'
timeout-minutes: 2
run: node --test --test-name-pattern=CUA electron/test/unit.mjs
- name: Install Linux native CUA fixture dependencies
if: env.RUN_STEPS == 'true' && runner.os == 'Linux'
run: sudo apt-get update && sudo apt-get install -y xvfb openbox dbus-x11 libgtk-3-dev at-spi2-core
- name: Native CUA application acceptance
if: env.RUN_STEPS == 'true'
timeout-minutes: 6
shell: bash
run: |
if [ "$RUNNER_OS" = Linux ]; then
dbus-run-session -- xvfb-run -a sh -c 'openbox >/tmp/reasonix-openbox.log 2>&1 & export NO_AT_BRIDGE=0 GTK_MODULES=gail:atk-bridge; node electron/test/cua-native.mjs'
else
node electron/test/cua-native.mjs
fi
# The host sits in the main module and nothing else builds it, so its
# dependency closure is checked here: it must stay CGO-free and
# platform-neutral or the Electron bundle cannot ship it. Cross-compiling
# is host-independent, so once is enough.
- name: host stays pure Go and platform-neutral
if: env.RUN_STEPS == 'true' && runner.os == 'Linux'
if: runner.os == 'Linux'
working-directory: .
run: |
for os in windows linux darwin; do
for arch in amd64 arm64; do
CGO_ENABLED=0 GOOS=$os GOARCH=$arch go build -o /dev/null ./cmd/reasonix-studio-host
done
CGO_ENABLED=0 GOOS=$os GOARCH=amd64 go build -o /dev/null ./cmd/reasonix-studio-host
done
# The ARM64 leg queues on a scarcer runner pool, so a pull request runs it
# only when it touches what the leg verifies; a push to studio always does.
# The macOS and Windows package legs follow `platform`, the same rule as the
# desktop matrix above.
scope:
name: package matrix
runs-on: ubuntu-latest
outputs:
matrix: ${{ steps.pick.outputs.matrix }}
arm64: ${{ steps.pick.outputs.arm64 }}
platform: ${{ steps.pick.outputs.platform }}
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
fetch-depth: 0
- id: pick
env:
EVENT: ${{ github.event_name }}
BASE_SHA: ${{ github.event.pull_request.base.sha }}
run: |
arm64=true
platform=true
if [ "$EVENT" = pull_request ]; then
arm64=false
platform=$(git diff -z --name-only --no-renames "$BASE_SHA" HEAD | bash scripts/ci-platform-scope.sh)
if git diff --name-only --no-renames "$BASE_SHA" HEAD |
grep -qE '^(desktop/electron/|desktop/packaging/|desktop/cmd/update-helper/|desktop/cmd/studio-manifest/|desktop/go\.(mod|sum)$|cmd/reasonix-studio-host/|internal/platform/(update|appupdate)/|go\.(mod|sum)$|scripts/(linux-install-smoke|verify-linux-package|verify-linux-appimage|verify-studio-artifacts)\.sh$|scripts/(windows-signing-lib|verify-windows-package|verify-windows-authenticode|install-smoke)\.ps1$|\.github/workflows/(studio|release-studio)\.yml$)'; then
arm64=true
fi
fi
legs='[
{"os":"macos-latest","arch":"arm64","platform":"darwin"},
{"os":"ubuntu-22.04","arch":"amd64","platform":"linux"},
{"os":"windows-latest","arch":"amd64","platform":"windows","ebarch":"x64"}
]'
if [ "$platform" = false ]; then
legs="$(jq -c 'map(select(.platform == "linux"))' <<<"$legs")"
elif [ "$arm64" = true ]; then
legs="$(jq -c '. + [{"os":"windows-11-arm","arch":"arm64","platform":"windows","ebarch":"arm64"},{"os":"ubuntu-22.04-arm","arch":"arm64","platform":"linux"}]' <<<"$legs")"
else
legs="$(jq -c . <<<"$legs")"
fi
{
echo "arm64=$arm64"
echo "platform=$platform"
echo "matrix={\"include\":$legs}"
} >> "$GITHUB_OUTPUT"
# The packaging contract, on real artifacts rather than on the config that
# describes them. Unsigned on purpose: signing and notarization are a
# distribution concern and belong to the release workflow, and treating their
@@ -441,12 +128,20 @@ jobs:
# cross-compile while only Windows had ever produced a package.
package:
name: package (${{ matrix.os }})
needs: scope
runs-on: ${{ matrix.os }}
timeout-minutes: 45
strategy:
fail-fast: false
matrix: ${{ fromJSON(needs.scope.outputs.matrix) }}
matrix:
include:
- os: macos-latest
arch: arm64
platform: darwin
- os: ubuntu-22.04
arch: amd64
platform: linux
- os: windows-latest
arch: amd64
platform: windows
defaults:
run:
shell: bash
@@ -459,45 +154,29 @@ jobs:
cache: true
cache-dependency-path: desktop/go.sum
- uses: pnpm/action-setup@73e46992e05b19abd31f2f3c20e0e7370d762115 # v6.1.1
- uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413 # v6.1.0
with:
version: 10
run_install: false
- uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
with:
node-version: "24"
cache: pnpm
cache-dependency-path: desktop/electron/pnpm-lock.yaml
# electron-builder shells out to dpkg-deb through fakeroot for the .deb and
# to rpmbuild for the .rpm.
# electron-builder shells out to dpkg-deb through fakeroot for the .deb.
- name: Install Linux packaging deps
if: runner.os == 'Linux'
run: sudo apt-get update && sudo apt-get install -y fakeroot rpm cpio xvfb xauth policykit-1
# The fpm electron-builder carries is an x86_64 program, so on an ARM64
# runner the .deb and .rpm are built with one installed from RubyGems.
- name: Use a native fpm on ARM64 Linux
if: runner.os == 'Linux' && runner.arch == 'ARM64'
run: |
sudo apt-get install -y ruby ruby-dev build-essential
sudo gem install --no-document fpm -v 1.16.0
echo "USE_SYSTEM_FPM=true" >> "$GITHUB_ENV"
run: sudo apt-get update && sudo apt-get install -y fakeroot
- name: Build the SPA
working-directory: desktop/frontend-next
run: pnpm install --frozen-lockfile && pnpm build
# The Windows legs name their architecture instead of taking the
# toolchain's, so a runner whose Go or Node is the other architecture
# fails here rather than shipping a mislabelled package.
- name: Build the kernel the shell speaks to
working-directory: desktop/electron
env:
REASONIX_GOARCH: ${{ matrix.platform == 'windows' && matrix.arch || '' }}
REASONIX_VERSION: v0.0.0-ci
run: pnpm install --frozen-lockfile && pnpm test && pnpm run build:host
run: pnpm install --frozen-lockfile && pnpm run build:host
# The .deb's absolute-path payload. A no-op off Linux, so it is not
# guarded here: the one place it must not be forgotten is the one place
@@ -512,8 +191,7 @@ jobs:
working-directory: desktop/electron
env:
CSC_IDENTITY_AUTO_DISCOVERY: "false"
ELECTRON_BUILDER_7Z_FILTER: ${{ matrix.platform == 'windows' && matrix.arch == 'arm64' && 'BCJ' || '' }}
run: pnpm run build:dist ${{ matrix.ebarch && format('--{0}', matrix.ebarch) || '' }} -c.extraMetadata.version=0.0.0-ci
run: pnpm run build:dist
# The release vocabulary is GOARCH. electron-builder's own is not, and the
# rename that reconciles them runs here rather than in the manifest.
@@ -529,99 +207,6 @@ jobs:
exit 1
fi
# An emulated x64 program runs on ARM64 Windows, so a package that merely
# starts proves nothing about its architecture: every PE image in the
# bundle has to be built for the one the package is named for.
- name: Every PE image is built for the package's architecture
if: matrix.platform == 'windows'
run: |
dir=$(find dist -maxdepth 1 -type d -name 'win*-unpacked' | head -1)
test -n "$dir"
node desktop/electron/packaging/pe.js "$dir" "${{ matrix.arch }}"
# An update starts the installer with --updated, and that is what keeps the
# shortcuts (and the taskbar pin hanging off them) from being deleted and
# recreated. Installed twice over itself the way an update does it, the
# desktop shortcut has to still be the same file, pointing at a path with
# no version in it, or every update costs the user their pin.
- name: An update keeps the shortcuts and they name no version
if: matrix.platform == 'windows'
shell: pwsh
run: |
$installer = (Get-ChildItem dist -Filter 'ReasonixStudio-windows-*-installer.exe' | Select-Object -First 1).FullName
$dir = Join-Path $env:RUNNER_TEMP 'rx-update-guard'
Start-Process $installer -ArgumentList '/S', "/D=$dir" -Wait
$lnk = Join-Path ([Environment]::GetFolderPath('Desktop')) 'Reasonix Studio.lnk'
$target = (New-Object -ComObject WScript.Shell).CreateShortcut($lnk).TargetPath
if ($target -ne (Join-Path $dir 'Reasonix Studio.exe')) { throw "the desktop shortcut targets $target, not the stable executable" }
$mark = [datetime]'2020-01-01'
(Get-Item $lnk).LastWriteTime = $mark
Start-Process $installer -ArgumentList '/S', '--updated', "/D=$dir" -Wait
if ((Get-Item $lnk).LastWriteTime -ne $mark) { throw 'an update rewrote the desktop shortcut' }
Start-Process (Join-Path $dir 'Uninstall Reasonix Studio.exe') -ArgumentList '/S', '/currentuser' -Wait
# The release packages a signed copy of the bundle and then reads both
# packages back against it. The same recipe on an unsigned copy, so the
# packaging contract (the installer's one application archive, the pinned
# elevate.exe, every PE image's architecture) is exercised on pull
# requests and not first by a release.
- name: The packages read back against the bundle they were made from
if: matrix.platform == 'windows'
shell: bash
env:
ELECTRON_BUILDER_7Z_FILTER: ${{ matrix.arch == 'arm64' && 'BCJ' || '' }}
run: |
rm -rf payload nsis-src packed zip-src
cp -r "$(find dist -maxdepth 1 -type d -name 'win*-unpacked' | head -1)" payload
rm -f payload/resources/elevate.exe
# The NSIS run writes elevate.exe into the tree it packs, so each
# packager gets a copy and payload stays what was signed.
cp -r payload zip-src
cp -r payload nsis-src
cd desktop/electron
pnpm exec electron-builder --prepackaged ../../zip-src --win zip --${{ matrix.ebarch }} \
-c.extraMetadata.version=0.0.0-ci -c.directories.output=../../packed-zip
pnpm exec electron-builder --prepackaged ../../nsis-src --win nsis --${{ matrix.ebarch }} \
-c.extraMetadata.version=0.0.0-ci -c.directories.output=../../packed
mv ../../packed-zip/ReasonixStudio-windows-${{ matrix.arch }}.zip ../../packed/
ls -la ../../packed
- name: verify-windows-package.ps1 accepts them
if: matrix.platform == 'windows'
shell: pwsh
run: |
$ErrorActionPreference = 'Stop'
. ./scripts/windows-signing-lib.ps1
$digest = Get-ManifestDigest -Manifest @(Get-TreeManifest -Root payload)
./scripts/verify-windows-package.ps1 `
-PayloadDirectory payload `
-ExpectedPayloadDigest $digest `
-InstallerPath "packed/ReasonixStudio-windows-${{ matrix.arch }}-installer.exe" `
-PortableArchivePath "packed/ReasonixStudio-windows-${{ matrix.arch }}.zip" `
-OutputPath "$env:RUNNER_TEMP/verify-output.txt" `
-Architecture ${{ matrix.arch }}
Get-Content "$env:RUNNER_TEMP/verify-output.txt"
# The release's own install smoke, against the unsigned installer: the
# package is installed where it is meant to run, its tree checked, and
# removed.
- name: The package installs, runs and uninstalls
if: matrix.platform == 'windows'
shell: pwsh
run: |
./scripts/install-smoke.ps1 `
-Installer "dist/ReasonixStudio-windows-${{ matrix.arch }}-installer.exe" `
-Architecture ${{ matrix.arch }}
- name: Keep the arm64 installer for the x64 refusal check
if: matrix.platform == 'windows' && matrix.arch == 'arm64'
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
with:
name: studio-arm64-installer-ci
path: dist/ReasonixStudio-windows-arm64-installer.exe
if-no-files-found: error
retention-days: 1
# The update channel is the signed manifest. A second answer to which
# version is current is not a convenience. The .blockmap is the same
# channel's differential index and carries the release prefix, so it also
@@ -648,17 +233,47 @@ jobs:
echo "page: ${page:-<missing>}"
test -n "$host" -a -n "$page"
- name: Linux packages carry their declared architecture and update payload
# Read off the package rather than off the config that describes it. Both
# of these went wrong the same way once: electron-builder names a package
# after package.json unless told, and fpm records the owner it finds on
# disk. Either one leaves a Studio that cannot update itself on Linux and
# says nothing about it until a user asks.
- name: The .deb is Studio's own package and carries what updates it
if: runner.os == 'Linux'
run: bash scripts/verify-linux-package.sh "dist/ReasonixStudio-linux-${{ matrix.arch }}.deb" "dist/ReasonixStudio-linux-${{ matrix.arch }}.rpm" "${{ matrix.arch }}"
run: |
deb=$(ls dist/ReasonixStudio-linux-*.deb)
name=$(dpkg-deb -f "$deb" Package)
if [ "$name" != "reasonix-studio" ]; then
echo "::error::the package is $name; dpkg holds that beside the installed reasonix-studio instead of upgrading it, and update.StudioLine asks for it by name"
exit 1
fi
dpkg-deb -c "$deb" > contents.txt
# The path is the polkit action's exec.path annotation, so it is the
# only one authorization is granted for; root because polkit will not
# run a helper the invoking user could have written. Either spelling
# of root: dpkg-deb prints the numeric ids it cannot map, and which
# one it prints is a property of the machine reading the package
# rather than of the package.
if ! grep -qE '^-rwxr-xr-x (root/root|0/0) +[0-9]+ .* \./usr/lib/reasonix-studio/reasonix-studio-update-helper$' contents.txt; then
echo "::error::no root-owned executable update helper at the path the polkit action names"
grep reasonix-studio-update-helper contents.txt || echo "(not in the package at all)"
exit 1
fi
if ! grep -q './usr/share/polkit-1/actions/io.reasonix.studio.update.policy$' contents.txt; then
echo "::error::the polkit action is not in the package, so pkexec has nothing to authorize against"
exit 1
fi
- name: The Linux package installs, runs, reinstalls and uninstalls
# The Electron shell sets the package's glibc floor; a Go binary linked
# with cgo would raise it to the runner's own.
- name: The .deb's Go binaries are statically linked
if: runner.os == 'Linux'
run: bash scripts/linux-install-smoke.sh "dist/ReasonixStudio-linux-${{ matrix.arch }}.deb" "${{ matrix.arch }}" 0.0.0-ci
- name: The AppImage carries its architecture, the kernel and the page
if: runner.os == 'Linux'
run: bash scripts/verify-linux-appimage.sh "dist/ReasonixStudio-linux-${{ matrix.arch }}.AppImage" "${{ matrix.arch }}"
run: |
deb=$(ls dist/ReasonixStudio-linux-*.deb)
dpkg-deb -x "$deb" "$RUNNER_TEMP/deb"
host=$(find "$RUNNER_TEMP/deb" -type f -path '*/resources/bin/reasonix-studio-host')
test -n "$host"
node desktop/electron/packaging/elf.js "$host" "$RUNNER_TEMP/deb/usr/lib/reasonix-studio/reasonix-studio-update-helper"
# Recorded, not asserted: the real bundle layout, so later work argues
# from what electron-builder produced rather than from its documentation.
@@ -686,27 +301,3 @@ jobs:
echo "::error::the manifest has no entry for ${{ matrix.platform }}-${{ matrix.arch }}"
exit 1
fi
# The arm64 installer carries one application and no other. Run anywhere but
# ARM64 Windows it has to say so and stop, instead of exiting 0 with nothing
# installed.
arm64-installer-refuses-x64:
name: arm64 installer refuses an x64 machine
needs: [scope, package]
if: needs.scope.outputs.arm64 == 'true'
runs-on: windows-latest
steps:
- uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
with:
name: studio-arm64-installer-ci
path: installer
- name: Run it silently
shell: pwsh
run: |
$ErrorActionPreference = 'Stop'
$target = Join-Path $env:RUNNER_TEMP 'rx-refused'
$run = Start-Process -FilePath installer/ReasonixStudio-windows-arm64-installer.exe -ArgumentList '/S', "/D=$target" -Wait -PassThru
Write-Host "exit code $($run.ExitCode)"
if ($run.ExitCode -eq 0) { throw 'the arm64 installer exited 0 on an x64 machine' }
if (@(Get-ChildItem -Recurse -Force $target -ErrorAction SilentlyContinue).Count -ne 0) { throw 'the arm64 installer wrote into the target on an x64 machine' }
+3 -3
View File
@@ -14,7 +14,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
- uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
with:
node-version: '22'
- name: Update README contributor tables
@@ -33,6 +33,6 @@ jobs:
body: |
Refreshes the README acknowledgments tables from the GitHub contributors API.
- Source: `repos/esengine/DeepSeek-Reasonix/contributors?per_page=50&anon=1`
- The repository owner, declared aliases, tool identities, bots and entries without a GitHub login are left out (`scripts/credit-identities.json`).
- Source: `repos/esengine/DeepSeek-Reasonix/contributors?per_page=20&anon=1`
- Anonymous contributors are shown without email addresses.
labels: documentation
+1 -1
View File
@@ -27,7 +27,7 @@ jobs:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
fetch-depth: 0
- uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
with:
node-version: '22'
- name: Test chart generator
@@ -1,84 +0,0 @@
name: Withdraw a Studio version
# Pulls a published Studio release out of the update catalog (or puts it back),
# so installs stop being offered it without waiting for a new release. Only the
# catalog changes: the release's files stay where they are, and a withdrawn
# release can be restored with the same run.
on:
workflow_dispatch:
inputs:
version:
description: "Version to withdraw or restore, for example 2.36.0"
required: true
type: string
reason:
description: "Why it is withdrawn (shown to users on that version); ignored when restoring"
required: false
type: string
restore:
description: "Put a withdrawn version back instead of withdrawing it"
required: false
type: boolean
default: false
concurrency:
group: studio-catalog
cancel-in-progress: false
jobs:
withdraw:
name: ${{ inputs.restore && 'restore' || 'withdraw' }} ${{ inputs.version }}
runs-on: ubuntu-latest
timeout-minutes: 10
environment: studio-release
permissions:
contents: write
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
ref: studio
persist-credentials: false
- name: Update the Studio catalog
env:
VERSION: ${{ inputs.version }}
REASON: ${{ inputs.reason }}
RESTORE: ${{ inputs.restore }}
R2_BUCKET: ${{ secrets.R2_BUCKET }}
HAS_R2: ${{ secrets.R2_ACCESS_KEY_ID != '' && secrets.R2_SECRET_ACCESS_KEY != '' && secrets.R2_ACCOUNT_ID != '' && secrets.R2_BUCKET != '' }}
run: |
if [ "$HAS_R2" != "true" ]; then
echo "::error::R2 credentials are not configured, so the catalog cannot be changed"
exit 1
fi
if ! [[ "$VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z.]+)?$ ]]; then
echo "::error::version must look like 2.36.0"
exit 1
fi
aws configure set aws_access_key_id "${{ secrets.R2_ACCESS_KEY_ID }}"
aws configure set aws_secret_access_key "${{ secrets.R2_SECRET_ACCESS_KEY }}"
aws configure set default.region auto
endpoint="https://${{ secrets.R2_ACCOUNT_ID }}.r2.cloudflarestorage.com"
at="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
current=/tmp/studio-versions-current.json
next=/tmp/studio-versions-next.json
aws s3 cp "s3://${R2_BUCKET}/studio/versions.json" "$current" --endpoint-url "$endpoint"
if [ "$RESTORE" = "true" ]; then
bash scripts/withdraw-version.sh "$current" --restore "$VERSION" "$at" > "$next"
jq -e --arg v "$VERSION" '.versions | map(select((.version | ltrimstr("v")) == $v)) | length == 1' "$next" >/dev/null
else
bash scripts/withdraw-version.sh "$current" "$VERSION" "$REASON" "$at" > "$next"
jq -e --arg v "$VERSION" '(.versions | map(select((.version | ltrimstr("v")) == $v)) | length == 0) and (.withdrawn | map(select((.version | ltrimstr("v")) == $v)) | length == 1)' "$next" >/dev/null
fi
aws s3 cp "$next" "s3://${R2_BUCKET}/studio/versions.json" \
--endpoint-url "$endpoint" \
--content-type "application/json; charset=utf-8" \
--cache-control "public, max-age=300, stale-if-error=86400"
echo "Studio catalog updated for $VERSION (restore=$RESTORE)"
- name: Keep the GitHub release off the latest slot
if: ${{ !inputs.restore }}
env:
GH_TOKEN: ${{ github.token }}
VERSION: ${{ inputs.version }}
run: gh release edit "studio-v${VERSION}" --prerelease --latest=false --repo "$GITHUB_REPOSITORY"
-2
View File
@@ -15,7 +15,6 @@
/runtime-resume
/extension-protocol-gen
/reasonix-plugin-example
/mcp-line-counter-kit
/reasonix-studio-host
/signpath-contract
/studio-delta
@@ -95,4 +94,3 @@ scripts/official-theme-art/__pycache__/
# running, which on Windows is every rebuild while Studio is open.
*.exe
*.exe~
/relayfake
-1
View File
@@ -11,7 +11,6 @@ fingerprint_files:
- .github/workflows/release-studio.yml
- .github/workflows/studio-certum-signing-smoke.yml
- .github/actions/setup-certum/action.yml
- scripts/install-smoke.ps1
- scripts/sign-certum.ps1
- scripts/verify-windows-authenticode.ps1
- scripts/verify-windows-package.ps1
-45
View File
@@ -117,22 +117,6 @@ go test ./internal/runtime/agent/ -v # verbose, one package
go test ./internal/tools/builtin/ -run TestGrep # one test
```
`testenv.IsolateUserState` installs a disposable user home and config-edit lock
scope. Default-path tests can clear `REASONIX_HOME` without losing the test
registry, while explicitly isolated Reasonix homes retain their own locks.
`REASONIX_TEST_HOME` carries the scope to helper subprocesses. A helper that
skips the isolation guard must install the inherited home with
`testscope.InstallHome` before running tests.
The environment variable alone cannot activate a test scope. Production
executables still use the OS account's home for shared config locks.
The guard removes its scratch home on normal completion. Scratch homes are
created under the original home, so an interrupted run can still leave
`.reasonix-test-home-*` directories; this is separate from persistent
config-edit locks in the real `.reasonix/locks` directory.
### Code style
- `gofmt` is enforced by CI — format before committing
@@ -197,35 +181,6 @@ porting to `studio`; do not open the same PR against both.
5. Ensure `gofmt -l .` shows no changes
6. Submit a pull request to that same branch
## Pull request policy
**Related changes.** If you have several related changes of one pattern, a
tracking issue plus one pull request is easiest to review.
**Issue first for features.** A change that adds behaviour rather than fixing a
defect should link an issue the maintainers have agreed to before the work
starts. A fix for a reproducible defect may open directly, with the failing test
in the PR. Documentation-only and test-only PRs are reviewed at lower priority.
**Review tiers.**
- Changes under the `sensitive:` paths declared in `REASONIX.md` get a full
review plus a security review, regardless of author.
- Every PR carries the four sections of the PR template: Cause, Blast radius,
Neighbouring behaviours tested, Why this layer.
- The new test must fail without the change.
- A UI change (`desktop/frontend-next/src` or the website pages) fills the
template's UI section: screenshots before and after, light and dark, wide
and phone width.
- The same section carries the line "Entries moved, hidden or removed".
- Hiding or removing a visible control needs a linked issue and a release-notes
line naming it, and updates `docs/STUDIO_PARITY.md` for a 1.x feature.
- Review holds these; no check reads them.
**Transparency.** The PR template has an optional checkbox for disclosing AI
assistance. It is for transparency only and does not change how a change is
reviewed.
## Reporting issues
Open an issue on GitHub with:
+1 -8
View File
@@ -9,7 +9,7 @@ GOEXE := $(shell go env GOEXE)
# One pin for the Makefile and the CI lint job; see .github/workflows/ci.yml.
GOLANGCI_VERSION := $(shell cat .golangci-version)
.PHONY: build vet fmt lint lint-go lint-install lint-cross lint-update coverage-gate coverage-gate-update pricecheck acp-conformance check test studio-test sdk-test sdk-test-race hooks cross clean studio
.PHONY: build vet fmt lint lint-go lint-install lint-cross lint-update coverage-gate coverage-gate-update pricecheck check test studio-test sdk-test sdk-test-race hooks cross clean studio
build:
CGO_ENABLED=0 go build -ldflags "$(LDFLAGS)" -o bin/reasonix$(GOEXE) ./cmd/reasonix
@@ -87,13 +87,6 @@ lint-cross:
test: studio-test sdk-test
go test ./...
# Drives `reasonix acp` with the official ACP TypeScript SDK and holds every
# message it writes to the schema that SDK ships. Needs node and network for
# `npm ci`, which runs without install scripts against a lockfile.
acp-conformance: build
cd tools/acp-conformance && npm ci --ignore-scripts --no-audit --no-fund
cd tools/acp-conformance && REASONIX_BIN=$(CURDIR)/bin/reasonix$(GOEXE) npm test
# The shell is a nested module, so the root `test` target never reaches it.
studio-test:
cd desktop && go test ./...
+14 -72
View File
@@ -18,8 +18,6 @@
<a href="https://esengine.github.io/DeepSeek-Reasonix/">Website</a>
&nbsp;·&nbsp;
<strong><a href="https://discord.gg/XF78rEME2D">Discord</a></strong>
&nbsp;·&nbsp;
<strong><a href="https://qm.qq.com/q/i59b0z2R8s">QQ</a></strong>
</p>
<p align="center">
@@ -33,7 +31,6 @@
<a href="https://github.com/esengine/DeepSeek-Reasonix/graphs/contributors"><img src="https://img.shields.io/github/contributors/esengine/DeepSeek-Reasonix.svg?style=flat-square&color=bc8cff&labelColor=161b22&logo=github&logoColor=white" alt="contributors"/></a>
<a href="https://github.com/esengine/DeepSeek-Reasonix/discussions"><img src="https://img.shields.io/github/discussions/esengine/DeepSeek-Reasonix.svg?style=flat-square&color=58a6ff&labelColor=161b22&logo=github&logoColor=white" alt="Discussions"/></a>
<a href="https://discord.gg/XF78rEME2D"><img src="https://img.shields.io/badge/discord-join-5865F2.svg?style=flat-square&labelColor=161b22&logo=discord&logoColor=white" alt="Discord"/></a>
<a href="https://qm.qq.com/q/i59b0z2R8s"><img src="https://img.shields.io/badge/QQ%20group-1093562660-12B7F5.svg?style=flat-square&labelColor=161b22&logo=tencentqq&logoColor=white" alt="QQ group 1093562660"/></a>
</p>
<p align="center">
@@ -43,73 +40,23 @@
<br/>
<p align="center"><strong>Open source · MIT · runs on your computer, model of your choice</strong></p>
<h3 align="center">An open-source coding agent for your terminal, desktop, browser and editor.</h3>
<p align="center">It reads your project, edits files and runs commands and tests. Have it ask before each step, or work on its own within limits you set. Files its edit tools changed can be rewound. New here? Start with the Studio desktop app.</p>
<p align="center"><strong>Open source · MIT · a single Go binary</strong></p>
<h3 align="center">A coding agent you can leave running.</h3>
<p align="center">One local engine, four ways in — terminal, desktop app, browser, or your editor over ACP. Plan mode, permissions, a workspace sandbox and per-turn checkpoints keep a long autonomous run something you can still read and undo.</p>
<p align="center">Maintained by <strong>Huahui Yu</strong>.</p>
> [!IMPORTANT]
> **Community · 加入社区** — bilingual Discord for setup help (`#help` / `#求助`), workflow showcases, and feature ideas. → **<https://discord.gg/XF78rEME2D>**
>
> QQ group **DeepSeek-Reasonix官方群** (`1093562660`) → **<https://qm.qq.com/q/i59b0z2R8s>** ([QR code](./docs/assets/qq-group.svg))
> Douyin: **做游戏的小鱼** (Douyin ID `22703872788`) · [QR code](./docs/assets/douyin.png)
<br/>
## ❤️Sponsor
<table><tr><td width="180" align="center" valign="middle"><a href="https://www.rightapi.ai/register?aff=e083bf73" target="_blank" rel="sponsored noopener noreferrer"><img src="./.github/sponsor/rightcode.png" alt="Right Code" width="64"></a></td><td valign="middle">Thanks to Right Code for sponsoring this project! Right Code reliably provides routing services for models such as Claude Code, Codex, and Gemini, with both pay-as-you-go and monthly subscription billing options available. Invoices are available upon top-up, and enterprise and team users can receive dedicated one-on-one support. Register via <a href="https://www.rightapi.ai/register?aff=e083bf73" target="_blank" rel="sponsored noopener noreferrer">this link</a>.</td></tr><tr><td width="180" align="center" valign="middle"><a href="https://www.packyapi.ai/register?aff=rIbQ" target="_blank" rel="sponsored noopener noreferrer"><picture><source media="(prefers-color-scheme: dark)" srcset="./.github/sponsor/packycode-dark.svg"><img src="./.github/sponsor/packycode-light.svg" alt="PackyCode" width="80"></picture></a></td><td valign="middle">Thanks to PackyCode for sponsoring this project! Access leading AI models through PackyCode with one API endpoint and one API key, with automatic failover and dedicated high-speed routes for Codex and Claude Code. Register via <a href="https://www.packyapi.ai/register?aff=rIbQ" target="_blank" rel="sponsored noopener noreferrer">this link</a>.</td></tr></table>
## What it is
Reasonix is an open-source AI coding assistant. The program runs on your computer; the AI model is a service you configure, which receives what it reads. You describe a task in plain language; it reads code, edits files and runs commands and tests inside the project folder you choose, and shows you each step.
You pick the model: a DeepSeek preset is built in, any OpenAI-compatible service is one config entry, and you pay the model provider directly.
## What you get
- **Read, edit, run.** It reads code, edits files and runs tests and commands, all inside your project folder.
- **Permission levels.** Have it ask every time, or let everyday actions run on their own; your forbid rules always hold. To see the approach first, use plan mode: it starts work only once you approve.
- **Rewind.** A file is saved before it changes, so you can rewind by conversation turn without touching your git history. Only files its edit tools changed are covered, not what a shell command did.
## 30-second start
Download, add a model key, open a project. New here? Start with the Studio desktop app.
1. Download and install the desktop app, Reasonix Studio, from the [releases page](https://github.com/esengine/DeepSeek-Reasonix/releases?q=studio-v&expanded=true).
2. Open it and enter your model provider and key when asked.
3. Open your project folder and write what you want, for example "add retry on failure to the http client".
4. At the level you chose, it stops and asks before anything that needs your say-so. Allow it to continue, or refuse.
Prefer a terminal? Download the `reasonix` command-line tool, run `reasonix setup` to pick a model, then run `reasonix`. See [Install](#install) and [Quick start](#quick-start).
## Four ways to use it
- **Desktop app (Studio).** A graphical app; the best place to start.
- **Terminal.** A full-screen text interface; `reasonix run "task"` also works in scripts.
- **Browser.** `reasonix web` opens a web interface on your own machine.
- **Editor.** Install the extension in VS Code; other editors that support ACP (Agent Client Protocol, a common way for an editor to connect to an AI assistant) can connect to `reasonix acp`.
All four are the same program underneath. What follows is for advanced users and contributors.
## Use it from DeepSeek Harness
DeepSeek Harness (`dsh`) can hand a task to Reasonix as a subagent over ACP. This repository publishes a plugin index for it:
1. Run `reasonix setup` on the machine that runs `dsh`. In the DSH Web UI turn on Custom plugin markets (Plugins, More) and add this index in the Discover tab: `https://raw.githubusercontent.com/esengine/DeepSeek-Reasonix/studio/integrations/dsh-plugin/market-index.json`
2. Install "Reasonix subagent" from it.
3. Open the plugin's page and turn its switch on. While it is off the model has no `reasonix` tool and may run `reasonix -p` in its own shell, which looks like a delegation and is not one.
Setup details, the permission policy and limits are in [Using Reasonix from DeepSeek Harness](./docs/DSH.md).
## Versions
Reasonix ships on two lines. See the [version roadmap announcement](https://github.com/esengine/DeepSeek-Reasonix/discussions/10748) for the reasoning behind the split.
| Line | Branch | Status | Get it |
| --- | --- | --- | --- |
| **Reasonix 2.x** | `studio` (this branch) | Active development | [Studio releases](https://github.com/esengine/DeepSeek-Reasonix/releases?q=studio-v&expanded=true) |
| **Reasonix 2.x** | `studio` (this branch) | Active development, pre-release | [Studio releases](https://github.com/esengine/DeepSeek-Reasonix/releases?q=studio-v&expanded=true) |
| **Reasonix 1.x** | [`main-v2`](https://github.com/esengine/DeepSeek-Reasonix/tree/main-v2) | Maintenance / stable | `npm i -g reasonix` · [desktop download](https://reasonix.io/?download=desktop#start) |
- **Want something stable?** Stay on 1.x. It keeps receiving bug fixes,
@@ -128,8 +75,6 @@ or reimplemented on 2.x where they fit the new architecture.
## Features
Technical highlights for advanced users.
- **Config-driven.** Providers, the agent, enabled tools, and plugins are all
declared in `reasonix.toml`. No hardcoded models.
- **Multi-model & composable.** DeepSeek ships as a preset; any
@@ -154,14 +99,14 @@ Studio, and editor integrations all run the same local Reasonix engine.
Download the package for your platform from the latest
[Studio release](https://github.com/esengine/DeepSeek-Reasonix/releases?q=studio-v&expanded=true)
(tagged `studio-v2.*`; candidates such as `-rc.N` are marked as
pre-releases):
(tagged `studio-v2.*`, published as pre-releases while 2.x is in active
development):
| Platform | Package | Architecture |
| --- | --- | --- |
| macOS | `.dmg` or `.zip` | Apple Silicon (`arm64`) / Intel (`amd64`) |
| Windows | Installer `-installer.exe` or portable `.zip` | x64 |
| Linux | `.deb`, `.rpm`, or `.AppImage` (download only) | x64 (`amd64`) / ARM64 (`arm64`, aarch64) |
| Linux | `.deb` | x64 |
Every package ships with a `.minisig` signature, and the release carries
`SHA256SUMS`. Once installed, Studio updates itself in place.
@@ -250,14 +195,12 @@ For advanced CLI usage and configuration, see the **[CLI reference](./docs/CLI.m
## Documentation
- **Getting started:** [Guide](./docs/GUIDE.md) · [CLI reference](./docs/CLI.md) ·
[Configuration paths](./docs/CONFIG_PATHS.md) · [ACP editor integration](./docs/ACP.md) ·
[Use from DeepSeek Harness](./docs/DSH.md)
[Configuration paths](./docs/CONFIG_PATHS.md) · [ACP editor integration](./docs/ACP.md)
- **Features & troubleshooting:** [Subagent profiles](./docs/SUBAGENT_PROFILES.md) ·
[Context Engine v2](./docs/SESSION_MEMORY_RETRIEVAL.md) ·
[Capability diagnostics](./docs/CAPABILITY_DIAGNOSTICS.md) ·
[Recovery and updates](./docs/RECOVERY.md) ·
[Checkpoints & rewind](./docs/CHECKPOINTS.md)
- **Agent presets:** [Session Agent compositions](./docs/AGENT_COMPOSITIONS.md)
- **Engineering & migration:** [Spec](./docs/SPEC.md) ·
[Task contracts & pause policy](./docs/TASK_CONTRACT.md) ·
[Tool contract](./docs/TOOL_CONTRACT.md) · [Moving from 1.x to 2.x](./docs/MIGRATING.md) ·
@@ -288,11 +231,11 @@ A small list of folks whose work has shaped Reasonix the most — the current to
<!-- reasonix-top-contributors:start -->
| Contributor | Contributor | Contributor | Contributor |
| --- | --- | --- | --- |
| [**SivanCola**](https://github.com/SivanCola) | [**KHG420**](https://github.com/KHG420) | [**ttmouse**](https://github.com/ttmouse) | [**chuiwenwei**](https://github.com/chuiwenwei) |
| [**lifu963**](https://github.com/lifu963) | [**XTLine**](https://github.com/XTLine) | [**HUQIANTAO**](https://github.com/HUQIANTAO) | [**xxoingr**](https://github.com/xxoingr) |
| [**Harbor404**](https://github.com/Harbor404) | [**GTC2080**](https://github.com/GTC2080) | [**mchenziyi**](https://github.com/mchenziyi) | [**Li-Charles-One**](https://github.com/Li-Charles-One) |
| [**light-front-theory**](https://github.com/light-front-theory) | [**imcreazy**](https://github.com/imcreazy) | [**eghrhegpe**](https://github.com/eghrhegpe) | [**noreply**](https://github.com/noreply) |
| [**par73e**](https://github.com/par73e) | [**Bernardxu123**](https://github.com/Bernardxu123) | [**lancy69**](https://github.com/lancy69) | [**Guan0923**](https://github.com/Guan0923) |
| [**SivanCola**](https://github.com/SivanCola) | [**esengine**](https://github.com/esengine) | [**ttmouse**](https://github.com/ttmouse) | [**lifu963**](https://github.com/lifu963) |
| **reasonix** | [**HUQIANTAO**](https://github.com/HUQIANTAO) | [**GTC2080**](https://github.com/GTC2080) | [**mchenziyi**](https://github.com/mchenziyi) |
| [**Li-Charles-One**](https://github.com/Li-Charles-One) | **merge-order-check** | [**light-front-theory**](https://github.com/light-front-theory) | **Yan Li** |
| [**eghrhegpe**](https://github.com/eghrhegpe) | **wufengfan** | [**Bernardxu123**](https://github.com/Bernardxu123) | [**HaoyueQin**](https://github.com/HaoyueQin) |
| [**CVEngineer66**](https://github.com/CVEngineer66) | [**KHG420**](https://github.com/KHG420) | [**JesonChou**](https://github.com/JesonChou) | [**SuMuxi66**](https://github.com/SuMuxi66) |
<!-- reasonix-top-contributors:end -->
Special thanks to [**Bernardxu123**](https://github.com/Bernardxu123) for designing the project logo and intro video.
@@ -322,9 +265,8 @@ coffee, not a contract — donations don't buy feature priority or change how
issues get triaged.
- **International** — PayPal: [paypal.me/yuhuahui](https://paypal.me/yuhuahui)
- **国内** — 微信支付 / 支付宝(扫码)
- **国内** — 微信支付(扫码)
<p align="center">
<img src=".github/sponsor/wechat-pay.jpg" alt="WeChat Pay QR code" width="180"/>
<img src=".github/sponsor/alipay.jpg" alt="Alipay QR code" width="180"/>
</p>
+13 -70
View File
@@ -18,8 +18,6 @@
<a href="https://esengine.github.io/DeepSeek-Reasonix/">官方网站</a>
&nbsp;·&nbsp;
<strong><a href="https://discord.gg/XF78rEME2D">Discord</a></strong>
&nbsp;·&nbsp;
<strong><a href="https://qm.qq.com/q/i59b0z2R8s">QQ</a></strong>
</p>
<p align="center">
@@ -33,7 +31,6 @@
<a href="https://github.com/esengine/DeepSeek-Reasonix/graphs/contributors"><img src="https://img.shields.io/github/contributors/esengine/DeepSeek-Reasonix.svg?style=flat-square&color=bc8cff&labelColor=161b22&logo=github&logoColor=white" alt="contributors"/></a>
<a href="https://github.com/esengine/DeepSeek-Reasonix/discussions"><img src="https://img.shields.io/github/discussions/esengine/DeepSeek-Reasonix.svg?style=flat-square&color=58a6ff&labelColor=161b22&logo=github&logoColor=white" alt="Discussions"/></a>
<a href="https://discord.gg/XF78rEME2D"><img src="https://img.shields.io/badge/discord-join-5865F2.svg?style=flat-square&labelColor=161b22&logo=discord&logoColor=white" alt="Discord"/></a>
<a href="https://qm.qq.com/q/i59b0z2R8s"><img src="https://img.shields.io/badge/QQ%20group-1093562660-12B7F5.svg?style=flat-square&labelColor=161b22&logo=tencentqq&logoColor=white" alt="QQ group 1093562660"/></a>
</p>
<p align="center">
@@ -43,63 +40,13 @@
<br/>
<p align="center"><strong>开源 · MIT · 程序跑在你电脑上,模型自选</strong></p>
<h3 align="center">终端、桌面、浏览器、编辑器里都能用的 AI 编程助手。</h3>
<p align="center">它会读你的项目、改文件、跑命令和测试。每一步先问你,或在你划定的范围内自己做,由你选;它用编辑工具改过的文件可以退回去。第一次用,从桌面应用 Studio 开始。</p>
<p align="center"><strong>开源 · MIT · 单个 Go 二进制</strong></p>
<h3 align="center">可以一直开着跑的编码 Agent。</h3>
<p align="center">一套本地引擎,四个入口——终端、桌面端、浏览器,或通过 ACP 接入你的编辑器。计划模式、权限、工作区沙箱与逐轮 checkpoint,让长时间自治运行始终可读、可撤销。</p>
<p align="center">维护者:<strong>Huahui Yu</strong>。</p>
> [!IMPORTANT]
> **加入社区 · Community** — 双语 Discord,提供安装答疑(`#help` / `#求助`)、工作流展示与功能想法。→ **<https://discord.gg/XF78rEME2D>**
>
> QQ 群 **DeepSeek-Reasonix官方群**(群号 `1093562660`)→ **<https://qm.qq.com/q/i59b0z2R8s>**([二维码](./docs/assets/qq-group.svg))
> 抖音:**做游戏的小鱼**(抖音号 `22703872788`)· [二维码](./docs/assets/douyin.png)
## ❤️赞助商
<table><tr><td width="180" align="center" valign="middle"><a href="https://www.rightapi.ai/register?aff=e083bf73" target="_blank" rel="sponsored noopener noreferrer"><img src="./.github/sponsor/rightcode.png" alt="Right Code" width="64"></a></td><td valign="middle">感谢 Right Code 赞助了本项目!Right Code 稳定提供 Claude Code、Codex、Gemini 等模型的中转服务,并可选按量、包月两种计费模式。充值即可开票,企业、团队用户一对一对接。通过<a href="https://www.rightapi.ai/register?aff=e083bf73" target="_blank" rel="sponsored noopener noreferrer">此链接</a>注册。</td></tr><tr><td width="180" align="center" valign="middle"><a href="https://www.packyapi.ai/register?aff=rIbQ" target="_blank" rel="sponsored noopener noreferrer"><picture><source media="(prefers-color-scheme: dark)" srcset="./.github/sponsor/packycode-dark.svg"><img src="./.github/sponsor/packycode-light.svg" alt="PackyCode" width="80"></picture></a></td><td valign="middle">感谢 PackyCode 赞助了本项目!PackyCode 是一家稳定、高效的 API 中转服务商,统一域名、统一密钥、智能容灾切换。通过<a href="https://www.packyapi.ai/register?aff=rIbQ" target="_blank" rel="sponsored noopener noreferrer">此链接</a>注册。</td></tr></table>
## 它是什么
Reasonix 是一个开源的 AI 编程助手。程序跑在你的电脑上;AI 模型是你自己配置的服务,它读到的内容会发给这个服务。你用大白话描述任务,它在你选的项目文件夹里读代码、改文件、跑命令和测试,每一步都摆给你看。
模型你自己选:内置 DeepSeek 预设,其他兼容 OpenAI 接口的服务填一条配置就行,费用直接付给模型服务商。
## 你能得到什么
- **读、改、跑**:读你的项目、改文件、跑测试和命令,都在你选的项目文件夹里。
- **权限分档**:每次都问你,或让日常操作自动放行;你设的禁止规则始终有效。想先看方案,就开计划模式,批准后它才动手。
- **可回退**:改文件前先存快照,按对话轮次回退,不碰 git 历史。只覆盖编辑工具改的文件,命令行命令造成的改动不在内。
## 30 秒上手
下载、填模型密钥、打开项目。第一次用,从桌面应用 Studio 开始。
1. 从[发布页](https://github.com/esengine/DeepSeek-Reasonix/releases?q=studio-v&expanded=true)下载并安装桌面应用 Reasonix Studio。
2. 打开应用,按提示填入模型服务商和密钥。
3. 打开你的项目文件夹,写下想做的事,例如“给 http 客户端加上失败重试”。
4. 按你选的档位,该问你的操作它会先停下来问;同意就继续,不同意就拒绝。
更喜欢终端的话,下载 `reasonix` 命令行后运行 `reasonix setup` 配好模型,再运行 `reasonix` 即可。详见[安装](#安装)和[快速开始](#快速开始)。
## 四种用法
- **桌面应用(Studio)**:图形界面,适合第一次使用。
- **终端**:全屏文字界面,也能用 `reasonix run "任务"` 在脚本里运行。
- **浏览器**:`reasonix web` 在本机打开网页界面。
- **编辑器**:VS Code 安装扩展即可;其他支持 ACP(Agent Client Protocol,编辑器接入某个 AI 助手的通用约定)的编辑器,可连接 `reasonix acp`。
四种用法背后是同一套程序。下面的内容写给进阶用户和贡献者。
## 在 DeepSeek Harness 里使用
DeepSeek Harness(`dsh`)可以通过 ACP 把任务交给 Reasonix 作为子智能体去做。本仓库为此发布了一个插件索引:
1. 在运行 `dsh` 的机器上先运行 `reasonix setup`。在 DSH 网页界面打开“自定义插件市场”(插件,更多),再到 Discover 页添加这个索引:`https://raw.githubusercontent.com/esengine/DeepSeek-Reasonix/studio/integrations/dsh-plugin/market-index.json`
2. 从中安装“Reasonix subagent”。
3. 打开这个插件的页面,把它的开关打开。关着的时候模型没有 `reasonix` 工具,可能改在自己的 shell 里直接运行 `reasonix -p`,看起来像委派,其实不是。
配置细节、权限策略和限制见 [在 DeepSeek Harness 中使用 Reasonix](./docs/DSH.md)。
## 版本
@@ -107,7 +54,7 @@ Reasonix 分为两条版本线,调整的原因见[版本路线公告](https://
| 版本线 | 分支 | 状态 | 获取方式 |
| --- | --- | --- | --- |
| **Reasonix 2.x** | `studio`(当前分支) | 活跃开发 | [Studio 发布页](https://github.com/esengine/DeepSeek-Reasonix/releases?q=studio-v&expanded=true) |
| **Reasonix 2.x** | `studio`(当前分支) | 活跃开发,预发布 | [Studio 发布页](https://github.com/esengine/DeepSeek-Reasonix/releases?q=studio-v&expanded=true) |
| **Reasonix 1.x** | [`main-v2`](https://github.com/esengine/DeepSeek-Reasonix/tree/main-v2) | 维护 / 稳定 | `npm i -g reasonix` · [桌面端下载](https://reasonix.io/?download=desktop#start) |
- **想要稳定**:继续使用 1.x。它会持续收到 Bug 修复、Provider / API 兼容、
@@ -122,8 +69,6 @@ Reasonix 分为两条版本线,调整的原因见[版本路线公告](https://
## 特性
以下是写给进阶用户的技术要点。
- **配置驱动**:provider、agent、启用的工具、插件全部在 `reasonix.toml` 中声明,
内核无硬编码模型。
- **多模型 · 可组合**:DeepSeek 作为预设内置;任何 OpenAI 兼容
@@ -143,13 +88,13 @@ Reasonix 分为两条版本线,调整的原因见[版本路线公告](https://
### Reasonix Studio(2.x)
从最新的 [Studio 发布](https://github.com/esengine/DeepSeek-Reasonix/releases?q=studio-v&expanded=true)
下载对应平台的安装包(标签为 `studio-v2.*`;`-rc.N` 等候选版本标为预发布):
下载对应平台的安装包(标签为 `studio-v2.*`,2.x 活跃开发期间以预发布形式发布):
| 平台 | 安装包 | 架构 |
| --- | --- | --- |
| macOS | `.dmg` 或 `.zip` | Apple Silicon(`arm64`)/ Intel(`amd64`) |
| Windows | 安装器 `-installer.exe` 或便携 `.zip` | x64 |
| Linux | `.deb`、`.rpm` 或 `.AppImage`(只提供下载) | x64(`amd64`)/ ARM64(`arm64`,aarch64) |
| Linux | `.deb` | x64 |
每个安装包都附带 `.minisig` 签名,发布页另有 `SHA256SUMS`。安装后 Studio 会在
应用内自行更新。
@@ -231,13 +176,12 @@ CLI 进阶用法和详细配置见 **[CLI 命令参考](./docs/CLI.zh-CN.md)**
- **开始使用:** [指南](./docs/GUIDE.zh-CN.md) ·
[CLI 命令参考](./docs/CLI.zh-CN.md) · [配置路径](./docs/CONFIG_PATHS.md) ·
[ACP 编辑器接入](./docs/ACP.md) · [在 DeepSeek Harness 中使用](./docs/DSH.md)
[ACP 编辑器接入](./docs/ACP.md)
- **功能与排障:** [子智能体 Profile](./docs/SUBAGENT_PROFILES.md) ·
[Context Engine v2](./docs/SESSION_MEMORY_RETRIEVAL.md) ·
[能力诊断](./docs/CAPABILITY_DIAGNOSTICS.md) ·
[恢复与安全模式](./docs/RECOVERY.md) ·
[Checkpoints 与 rewind](./docs/CHECKPOINTS.md)
- **Agent 预设:** [会话级组合](./docs/AGENT_COMPOSITIONS.zh.md)
- **工程与迁移:** [规格](./docs/SPEC.md) ·
[任务合约与暂停策略](./docs/TASK_CONTRACT.md) ·
[工具合约](./docs/TOOL_CONTRACT.md) ·
@@ -269,11 +213,11 @@ CLI 进阶用法和详细配置见 **[CLI 命令参考](./docs/CLI.zh-CN.md)**
<!-- reasonix-top-contributors:start -->
| Contributor | Contributor | Contributor | Contributor |
| --- | --- | --- | --- |
| [**SivanCola**](https://github.com/SivanCola) | [**KHG420**](https://github.com/KHG420) | [**ttmouse**](https://github.com/ttmouse) | [**chuiwenwei**](https://github.com/chuiwenwei) |
| [**lifu963**](https://github.com/lifu963) | [**XTLine**](https://github.com/XTLine) | [**HUQIANTAO**](https://github.com/HUQIANTAO) | [**xxoingr**](https://github.com/xxoingr) |
| [**Harbor404**](https://github.com/Harbor404) | [**GTC2080**](https://github.com/GTC2080) | [**mchenziyi**](https://github.com/mchenziyi) | [**Li-Charles-One**](https://github.com/Li-Charles-One) |
| [**light-front-theory**](https://github.com/light-front-theory) | [**imcreazy**](https://github.com/imcreazy) | [**eghrhegpe**](https://github.com/eghrhegpe) | [**noreply**](https://github.com/noreply) |
| [**par73e**](https://github.com/par73e) | [**Bernardxu123**](https://github.com/Bernardxu123) | [**lancy69**](https://github.com/lancy69) | [**Guan0923**](https://github.com/Guan0923) |
| [**SivanCola**](https://github.com/SivanCola) | [**esengine**](https://github.com/esengine) | [**ttmouse**](https://github.com/ttmouse) | [**lifu963**](https://github.com/lifu963) |
| **reasonix** | [**HUQIANTAO**](https://github.com/HUQIANTAO) | [**GTC2080**](https://github.com/GTC2080) | [**mchenziyi**](https://github.com/mchenziyi) |
| [**Li-Charles-One**](https://github.com/Li-Charles-One) | **merge-order-check** | [**light-front-theory**](https://github.com/light-front-theory) | **Yan Li** |
| [**eghrhegpe**](https://github.com/eghrhegpe) | **wufengfan** | [**Bernardxu123**](https://github.com/Bernardxu123) | [**HaoyueQin**](https://github.com/HaoyueQin) |
| [**CVEngineer66**](https://github.com/CVEngineer66) | [**KHG420**](https://github.com/KHG420) | [**JesonChou**](https://github.com/JesonChou) | [**SuMuxi66**](https://github.com/SuMuxi66) |
<!-- reasonix-top-contributors:end -->
特别感谢 [**Bernardxu123**](https://github.com/Bernardxu123) 设计的项目 logo和开场视频。
@@ -301,10 +245,9 @@ CLI 进阶用法和详细配置见 **[CLI 命令参考](./docs/CLI.zh-CN.md)**
如果 Reasonix 帮你省了时间或 token,欢迎请杯咖啡。捐助不会换来 feature
优先级,也不会影响 issue 的处理顺序——就是「谢谢」。
- **国内** — 微信支付 / 支付宝(扫下方二维码)
- **国内** — 微信支付(扫下方二维码)
- **海外** — PayPal: [paypal.me/yuhuahui](https://paypal.me/yuhuahui)
<p align="center">
<img src=".github/sponsor/wechat-pay.jpg" alt="微信支付收款码" width="180"/>
<img src=".github/sponsor/alipay.jpg" alt="支付宝收款码" width="180"/>
</p>
+10 -51
View File
@@ -219,11 +219,9 @@ otherwise the block the model already has goes on reading as current.
- `#<note>` in chat quick-adds an always-on instruction. The `remember` tool
instead saves a fallible background fact (frontmatter file + `MEMORY.md`
index). Fact `type` classifies content; independent `scope` controls whether it
is project-only (the default) or explicitly global.
- The saved-fact index stays out of the stable prefix and is reached through the
`memory` tool; pinned bodies and global user/feedback bodies load into it, the
latter as lower-priority compatibility guidance. The current turn receives a
tail note.
is project-only (the default) or explicitly global. The index loads into the
stable prefix on the next session; global user/feedback bodies also load as
lower-priority compatibility guidance. The current turn receives a tail note.
## Notes
@@ -322,31 +320,6 @@ Use `go test ./path/to/target/` to detect cycles **before** pushing. A `[setup f
- **Keep the PR diff minimal.** Only the files relevant to the PR's purpose — no stray changes from other branches.
- **Amend, don't add commits, for review feedback** — keeps the commit history clean.
## Change discipline
**Assess before editing.** For every change, humans and agents list callers,
consumers of the same state, sibling features, reload/replay/restart paths,
error and edge states, TUI/CLI and desktop equivalents, and wire mirrors.
A fix that breaks a neighbouring behaviour is a failed fix.
**Test the neighbourhood.** UI changes add or extend interaction tests for
adjacent functions of the same component or flow (composer: submit, clear,
queue, withdraw, restore, steer, failure, replay after reload) and run the whole
affected suite.
Write tests first; they must fail without the change.
**Fix at the owning layer.** Use the existing unified pattern: no special-case
flags, duplicated paths, or workarounds at the symptom site.
If the design is duplicated or wrong, make the smallest unifying change:
one path instead of two, deleting what becomes dead. Use current language and
framework idioms; do not add a new pattern where one exists.
**PR evidence.** Every PR description carries: Cause, Blast radius,
Neighbouring behaviours tested, and Why this layer.
## Reasonix host checks
The paths below decide what the agent is allowed to do — a wrong change here
@@ -369,14 +342,9 @@ out of how a path is spelled, which cannot tell `internal/auth` from
- sensitive: internal/ext/pluginpkg/**
- sensitive: internal/base/netclient/**
- sensitive: internal/safety/redirectguard/**
- sensitive: internal/safety/endpointclient/**
- sensitive: internal/safety/egress/**
- sensitive: internal/platform/browser/**
- sensitive: internal/platform/computer/**
- sensitive: internal/platform/gitpush/**
- sensitive: internal/session/control/git_push.go
- sensitive: internal/frontend/serve/git_push.go
- sensitive: internal/frontend/serve/host_routes.go
One declaration, two effects: the same list is also the coverage gate's subject.
`make coverage-gate` holds each path to the coverage it already has, because
@@ -407,19 +375,14 @@ observation, never by a check that ran afterwards.
What follows bounds Observable today. It is not a restatement of the invariant:
moving a number here must not move the promise.
- `scanWorkspace` walks the workspace and every granted extra root recursively,
skips VCS store dirs, and is `complete` only below 50k files in total. Over that, or past a walk error, it
- `scanWorkspace` walks the workspace recursively, skips VCS store dirs, and is
`complete` only below 50k files. Over that, or past a walk error, it
establishes nothing and the mutation stays unproven.
- Symlinks are recorded by `Lstat`, so a write through one lands outside what
the walk compares. Confining the resolved target is the sandbox's job.
- `internal/safety/sandbox` confines writes to the workspace, configured extras, temp
and toolchain caches on macOS and Linux.
- On Windows a user-enabled write-restricted token bounds the *creation* of files
(create, append, overwrite, `mkdir`, HKCU writes, global git config, writes through links).
- There, deleting or renaming away an existing file is not bounded, and with the
setting off nothing the host enforces bounds `Writable`.
- The caches a command may write can be poisoned for the user's later unsandboxed
builds, on every platform.
and toolchain caches — on macOS and Linux. Windows has no OS-level bash
sandbox, so there nothing the host enforces bounds `Writable`.
- The governed host authorities are a *named* set — ssh-agent, docker, podman.
DBus, Wayland, X11, gpg-agent, keyrings and arbitrary sockets stay reachable,
so the claim is "these endpoints are governed", never "host IPC is isolated".
@@ -465,13 +428,9 @@ stop a connect. Possession is not authority: the key can be unreadable while an
agent still signs with it.
A protection claim exists only where a backend has a demonstrated enforcement
primitive for that dimension. Unsupported dimensions stay explicitly unsupported,
because a claim that outruns its enforcement is worse than an absent one: it is
believed.
Windows enforces Integrity on file creation only, when the user turns it on.
Deletion and rename, Confidentiality and Authority (the named pipes of ssh-agent
and docker, COM and RPC services) are not enforced there.
primitive for that dimension. Unsupported dimensions stay explicitly unsupported
— Windows currently enforces none of the three — because a claim that outruns
its enforcement is worse than an absent one: it is believed.
**Known unmeasured surface — macOS host-mediated delegation:** Apple Events and
Mach-service lookup are not denied by the current allow-default profile. Prior
+5 -65
View File
@@ -1,6 +1,6 @@
# Reasonix Benchmarks
Six harnesses live under `benchmarks/`; `cmd/e2ebench` also exposes a
Five harnesses live under `benchmarks/`; `cmd/e2ebench` also exposes a
SWE-bench Verified mode:
- `e2e/` — the committed end-to-end task suite, driven by
@@ -19,10 +19,6 @@ SWE-bench Verified mode:
- `fanout-width/` — a shape-instructed fleet whose members share nothing, for
pricing what running them at the same time is worth. See
[Pricing the run graph](#pricing-the-run-graph) below.
- `realrepo/` — long tasks over a pinned snapshot of this repository, for
measuring changes that cut model round trips (13–31 requests per task on
`deepseek-flash`). See
[its README](realrepo/README.md).
## Directory layout
@@ -420,11 +416,11 @@ own outcome).
| Flag | Default | Purpose |
| --- | --- | --- |
| `-mode` | `suite` | `suite` \| `diff` \| `swebench` \| `compare` \| `traj` (`diff` generates tests for the PR diff; `swebench` runs the official per-instance evaluation; `compare` renders KPI/Pareto readouts from 2+ `-json` reports; `traj` re-digests recorded trajectory files without spending tokens; `audit` checks a batch's published figures against its retained rows). |
| `-mode` | `suite` | `suite` \| `diff` \| `swebench` \| `compare` \| `traj` (`diff` generates tests for the PR diff; `swebench` runs the official per-instance evaluation; `compare` renders KPI/Pareto readouts from 2+ `-json` reports; `traj` re-digests recorded trajectory files without spending tokens). |
| `-suite` | `benchmarks/e2e` | Suite root (must contain `tasks/<id>/`). |
| `-task` | *(all)* | Suite mode: run only these comma-separated task IDs (e.g. `-task fix-add-bug`); unknown IDs fail with the available list. |
| `-attempts` | `1` | Suite and diff modes: retry a task until an attempt passes, up to N; enables the `Pass@≤N` KPI, and TTCS charges a retried solve with its failed attempts' wall. |
| `-trials` | `1` | Suite mode: run every task N times and keep every outcome. Adds a **Reliability** line: `pass^N` (complete tasks that passed every trial) and `pass@N` (complete tasks with any pass), both over the tasks that ran all N trials only (`n/a` when none did), mean `pass@1` over every graded trial, and the flaky tasks by name; a task the token budget cut short is listed on an **Incomplete** line and is in neither. `Pass@≤N` asks whether the agent can solve a task; `pass^N` asks whether it does so every time. Cannot be combined with `-attempts`. |
| `-trials` | `1` | Suite mode: run every task N times and keep every outcome. Adds a **Reliability** line: `pass^N` (tasks that passed every trial), `pass@N` (any trial), mean `pass@1` over all trials, and the flaky tasks by name. `Pass@≤N` asks whether the agent can solve a task; `pass^N` asks whether it does so every time. Cannot be combined with `-attempts`. |
| `-tape-record DIR` | — | Suite mode, with `-meter`: record every provider exchange at the meter, one tape per task and trial (`DIR/<task>/trial-N/0001.request.json`, `.response.json`, `.response.body`). Taped runs use a fixed workdir path and pin the seeded files' times, because both reach the model through the workspace block and file listings. |
| `-tape-replay DIR` | — | Suite mode, with `-meter`: answer each run from its tape instead of the provider, and add a **Replay** line naming the first request whose body differs from the recorded one, down to the message (`request 2, message 4 (tool)`), the tool list, or a field. The diverging request is kept as `NNNN.replayed.json` beside the recorded one. A kernel change that should not alter what the model is asked replays with every request matched. |
| `-bin` | `reasonix` | Path to the reasonix binary. |
@@ -433,7 +429,6 @@ own outcome).
| `-ablate` | *(none)* | Ablation arm: comma-separated subsystems to switch off — `evidence`, `planner`, `subagent`, `retrieval`, `compaction`, `upstream` and `recall-search`; `none` \| `all`. |
| `-out` | *(stdout)* | Write the markdown report here. |
| `-json` | *(none)* | Write the JSON report here (optional). |
| `-batch` | *(none)* | Suite mode: write `results.jsonl` (one row per attempt) and `batch.json` (manifest v1) here, so the report's figures can be audited later. Audit mode: read the batch from here. See [Auditing a batch](#auditing-a-batch). |
| `-trajectories` | *(none)* | Suite mode: write one `<task-id>.trajectory.jsonl` per task into this directory (the agent's full event stream with timestamps — see `reasonix run --trajectory`). The report gains a time-attribution line (tools vs. model) and each JSON result a `trajectory` digest. |
| `-force-planner` | `false` | Suite mode: prefix each prompt with a plan-first directive so the two-model turn engages regardless of the planner gate. Use for the "with planner" arm of an A/B; results carry `plan_forced` so arms are only comparable with equal forcing. |
| `-anchor` | `blind` | Suite mode: which hypothesis the agent holds before it looks at anything — `blind` (none, the control) \| `correct` \| `wrong`. The seeded arms prefix each prompt with the task's authored seed and **skip** tasks that have none, so an unseeded control run never lands in a seeded denominator. Results carry `anchor`. See [Anchor resistance](#anchor-resistance). |
@@ -456,65 +451,10 @@ Diff-mode flags:
| `-timeout` | `1200` | Agent timeout in seconds (diff mode). |
| `-attempts` | `1` | Diff mode: retry up to N times until a run passes (stochastic agent). |
## Auditing a batch
`-batch DIR` writes two files from a suite run: `results.jsonl` (one row per
attempt — the raw record every figure is computed from) and `batch.json`
(manifest v1). `-mode audit -batch DIR` re-derives the report's headline figures
from `results.jsonl`, prints the manifest's claim beside the recomputed value,
and exits non-zero naming any figure that moved. It also refuses a batch whose
manifest is missing, records no commit, or whose rows are truncated.
This is an **aggregation audit, not a replay**: it proves the published figures
follow from the retained rows. It does not re-execute the model, tools or
environment, and cannot show the experiment would reproduce. A real replay would
need the commit, a corpus fingerprint, the model and provider configuration, the
seed, the flags, the raw results and the runtime version; it is not implemented.
`schema_version` is top level. The rest is three groups:
| Group | Fields | Answers |
| --- | --- | --- |
| `over` | `command.mode` and `command.axes` (every flag that changes what a figure means; paths omitted); `tasks.count` and `tasks.digest` (sha256 over sorted, NUL-separated task ids — the corpus identity, never the task text); `filters`; `denominator` (`unit`, `definition`, `value`) | what the figures are over |
| `producer` | `commit` (or an explicit `unknown`, never a guess), `commit_source`, `dirty`, `model`, `provider`, `seed`, `started_at`, `finished_at`, `harness_version`, `go_version` | which run produced them |
| `trust` | `results_sha256`, `claimed` (the headline figures), `ledger`, `retention` | why they can still be checked |
The ledger accounts for every planned run, and the audit checks that the counts
form a funnel (`scored ≤ completed ≤ attempted`, `failed ≤ scored`,
`scored = pass1 + failed`, `rows = attempted + skipped`). A row count is never a
denominator: `denominator.value` is a count of scored tasks, and `rows` is
published only to show what was retained.
| Count | Meaning |
| --- | --- |
| `planned` | graded runs the batch set out to attempt: filtered tasks × trials |
| `attempted` | retained rows for runs that started (skipped rows are not attempts) |
| `completed` | attempts that ended and left a row; this harness writes a row only after an attempt ends, so it equals `attempted` and the equality is checked, not assumed |
| `scored` | completed first attempts of solvable, non-skipped tasks — the pass@1 denominator |
| `failed` | scored attempts the grader did not pass |
| `unmeasured` | attempts whose cost and tokens are unknown because no metrics landed; kept out of spend totals, never counted as zero |
| `excluded` | rows kept out of the accuracy denominator: skipped, `no_solution`, and retry entries |
Spend and cache fields are omitted from `claimed` when nothing was measured, so
an absent figure reads as unknown rather than as a measured zero. `seed` is
omitted for the same reason: the harness does not set a sampling seed.
`results_sha256` covers the values and the presence of every row; line endings,
indentation and key order are normalised away because none of them is evidence,
while number spelling is kept (a rewritten `1` is not `1.0`).
A successful audit records the verifier version (`audit.verifier_version`) and
the digest it checked in `batch.json`, so a re-verification can tell which code
checked the batch.
`results.jsonl` is retained unredacted beside the manifest — task prompts and
agent notes included — and is not public. The manifest itself carries a task-id
digest, no prompt text, no absolute paths, no endpoints or keys and no model
output. A claim whose rows are gone cannot be audited: there is nothing to
recompute from, and this tool will not certify it.
## Dataset retention
Keep every `-json` report, `-trajectories` directory and `-batch` directory from
real runs: they are the accumulating corpus — per-task contracts-to-be, full event
Keep every `-json` report and `-trajectories` directory from real runs: they
are the accumulating corpus — per-task contracts-to-be, full event
trajectories, checkpoint oracle verdicts, stop curves and phase traces — that
any future offline learning (routing, stop policies, budgets) would train
and evaluate on. The control plane stays deterministic and interpretable
@@ -1,35 +0,0 @@
# Compaction prefix probe
## Method
1. Measured on 2026-10-17 against the official `https://api.deepseek.com` OpenAI adapter, using the model returned by `/models`: `deepseek-flash`. Thinking was disabled. Inputs were synthetic; no workspace history or credentials are recorded here.
2. Each of three text and three rich-history trials sent a main request to warm the prefix, then called the actual agent summary path, then an independent summary control. Rich history included a synthetic closed `read_file` call, result and reasoning. Warmup outputs were discarded, never executed.
3. A summary had a 1,024-token output cap; warmups had 64. Success means a heading, preservation of the fixture's `ParseDuration` and zero-value constraint, no emitted tool calls, and `finish_reason=stop`. This checks the fixture's contract, not arbitrary-summary factual completeness.
4. First-attempt reuse success counts successful reused summaries / six reused summaries. Fallback rate counts independent requests initiated by the reuse coordinator / six reuse trials; the six explicit independent controls are excluded. Both were 6/6 success and 0/6 fallbacks. Independent controls also succeeded 6/6.
5. The raw result file records provider-reported usage and full stream latency, measured from adapter `Stream` until channel close. Token rows below sum only the three indicated summary requests; they exclude warmup, model listing and any earlier exploratory requests. Cache residency was uncontrolled; prior probes had warmed these synthetic prefixes.
## Results
| History | Shape | N | Prompt | Hit | Miss | Output | Median ms | Estimated USD |
| --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: |
| Text | Reused | 3 | 3,714 | 3,072 | 642 | 636 | 1,713 | 0.000487116 |
| Text | Independent | 3 | 1,218 | 768 | 450 | 644 | 1,610 | 0.000456204 |
| Rich | Reused | 3 | 3,780 | 3,072 | 708 | 742 | 1,774 | 0.000560616 |
| Rich | Independent | 3 | 1,341 | 768 | 573 | 921 | 1,815 | 0.000640854 |
1. Estimates use the [official pricing](https://api-docs.deepseek.com/quick_start/pricing/) checked on the measurement date: off-peak Flash USD per million tokens, hit 0.003, miss 0.15, output 0.6. October 17 was a Saturday. Formula: `(hit*0.003 + miss*0.15 + output*0.6)/1_000_000`. These are token estimates, not account deductions.
2. Reuse hit fractions were 82.7% (text) and 81.3% (rich), versus 63.1% and 57.3% for controls. This small fixture does **not** establish cost savings: retaining the system/catalog increased both total input and input cost. Text total cost increased 6.8%; rich total decreased 12.5% because output was shorter, not because input was cheaper.
3. The six warmups consumed 7,032 prompt and 231 output tokens, estimated USD 0.000346680. All 18 recorded generation calls together cost an estimated USD 0.002491470. No failed-digest retry occurred in these live trials; the deterministic fallback tests establish its behavior and accounting.
4. A rejected nonempty reused answer earns one independent-shape fallback. A separate truncation retry may enlarge the output cap, for at most three logical model calls total. Every actual attempt emits and records its own spend; returned usage is their sum. Empty output, stream errors and cancellation do not earn shape retries.
5. These sequential, six-pair trials establish interoperability for one model, mode and short fixture. They do not estimate production failure probability, thinking-mode behavior, other models, cold-cache cost, or large-history savings.
## Reproduction
1. Provide a test key through an existing secret environment configuration. Do not put credentials in source, command arguments or captured logs. Select a model currently returned by `/models`.
2. Run the opt-in probe; it skips without `DEEPSEEK_API_KEY`. The dedicated build tag avoids enabling unrelated live tests.
```sh
REASONIX_LIVE_MODEL=deepseek-flash go test -tags live_compaction ./internal/runtime/agent -run '^TestLiveCompactionPrefix$' -v -count=1
```
3. This probe exercises actual summary construction and the HTTP adapter. The hermetic `TestEffectCompactionSharesThePreviouslySentPrefix` separately exercises manual, automatic pressure and rich history through `boot.Build`, comparing actual JSON request-prefix and schema bytes. Its sparse-history case also proves an interleaved retained user turn survives the fold.
@@ -1,512 +0,0 @@
[
{
"model": "deepseek-flash",
"rich": false,
"trial": 1,
"results": [
{
"kind": "warm",
"milliseconds": 828,
"digest": false,
"grounded": false,
"tool_calls": 1,
"usage": {
"PromptTokens": 1127,
"CompletionTokens": 38,
"TotalTokens": 1165,
"CacheHitTokens": 896,
"CacheMissTokens": 231,
"CacheWriteTokens": 0,
"CacheWriteBilledTokens": 0,
"ReasoningTokens": 0,
"FinishReason": "tool_calls",
"Estimated": false,
"RequestCount": 1,
"ServerToolRequests": 0,
"ContextPromptTokens": 0,
"ContextCompletionTokens": 0,
"ContextReasoningTokens": 0,
"ContextCacheHitTokens": 0,
"ContextCacheMissTokens": 0
}
},
{
"kind": "reuse",
"milliseconds": 2006,
"digest": true,
"grounded": true,
"tool_calls": 0,
"usage": {
"PromptTokens": 1238,
"CompletionTokens": 234,
"TotalTokens": 1472,
"CacheHitTokens": 1024,
"CacheMissTokens": 214,
"CacheWriteTokens": 0,
"CacheWriteBilledTokens": 0,
"ReasoningTokens": 0,
"FinishReason": "stop",
"Estimated": false,
"RequestCount": 1,
"ServerToolRequests": 0,
"ContextPromptTokens": 0,
"ContextCompletionTokens": 0,
"ContextReasoningTokens": 0,
"ContextCacheHitTokens": 0,
"ContextCacheMissTokens": 0
}
},
{
"kind": "independent",
"milliseconds": 1817,
"digest": true,
"grounded": true,
"tool_calls": 0,
"usage": {
"PromptTokens": 406,
"CompletionTokens": 234,
"TotalTokens": 640,
"CacheHitTokens": 256,
"CacheMissTokens": 150,
"CacheWriteTokens": 0,
"CacheWriteBilledTokens": 0,
"ReasoningTokens": 0,
"FinishReason": "stop",
"Estimated": false,
"RequestCount": 1,
"ServerToolRequests": 0,
"ContextPromptTokens": 0,
"ContextCompletionTokens": 0,
"ContextReasoningTokens": 0,
"ContextCacheHitTokens": 0,
"ContextCacheMissTokens": 0
}
}
]
},
{
"model": "deepseek-flash",
"rich": false,
"trial": 2,
"results": [
{
"kind": "warm",
"milliseconds": 820,
"digest": false,
"grounded": false,
"tool_calls": 1,
"usage": {
"PromptTokens": 1127,
"CompletionTokens": 38,
"TotalTokens": 1165,
"CacheHitTokens": 896,
"CacheMissTokens": 231,
"CacheWriteTokens": 0,
"CacheWriteBilledTokens": 0,
"ReasoningTokens": 0,
"FinishReason": "tool_calls",
"Estimated": false,
"RequestCount": 1,
"ServerToolRequests": 0,
"ContextPromptTokens": 0,
"ContextCompletionTokens": 0,
"ContextReasoningTokens": 0,
"ContextCacheHitTokens": 0,
"ContextCacheMissTokens": 0
}
},
{
"kind": "reuse",
"milliseconds": 1633,
"digest": true,
"grounded": true,
"tool_calls": 0,
"usage": {
"PromptTokens": 1238,
"CompletionTokens": 187,
"TotalTokens": 1425,
"CacheHitTokens": 1024,
"CacheMissTokens": 214,
"CacheWriteTokens": 0,
"CacheWriteBilledTokens": 0,
"ReasoningTokens": 0,
"FinishReason": "stop",
"Estimated": false,
"RequestCount": 1,
"ServerToolRequests": 0,
"ContextPromptTokens": 0,
"ContextCompletionTokens": 0,
"ContextReasoningTokens": 0,
"ContextCacheHitTokens": 0,
"ContextCacheMissTokens": 0
}
},
{
"kind": "independent",
"milliseconds": 1555,
"digest": true,
"grounded": true,
"tool_calls": 0,
"usage": {
"PromptTokens": 406,
"CompletionTokens": 202,
"TotalTokens": 608,
"CacheHitTokens": 256,
"CacheMissTokens": 150,
"CacheWriteTokens": 0,
"CacheWriteBilledTokens": 0,
"ReasoningTokens": 0,
"FinishReason": "stop",
"Estimated": false,
"RequestCount": 1,
"ServerToolRequests": 0,
"ContextPromptTokens": 0,
"ContextCompletionTokens": 0,
"ContextReasoningTokens": 0,
"ContextCacheHitTokens": 0,
"ContextCacheMissTokens": 0
}
}
]
},
{
"model": "deepseek-flash",
"rich": false,
"trial": 3,
"results": [
{
"kind": "warm",
"milliseconds": 905,
"digest": false,
"grounded": false,
"tool_calls": 1,
"usage": {
"PromptTokens": 1127,
"CompletionTokens": 38,
"TotalTokens": 1165,
"CacheHitTokens": 896,
"CacheMissTokens": 231,
"CacheWriteTokens": 0,
"CacheWriteBilledTokens": 0,
"ReasoningTokens": 0,
"FinishReason": "tool_calls",
"Estimated": false,
"RequestCount": 1,
"ServerToolRequests": 0,
"ContextPromptTokens": 0,
"ContextCompletionTokens": 0,
"ContextReasoningTokens": 0,
"ContextCacheHitTokens": 0,
"ContextCacheMissTokens": 0
}
},
{
"kind": "reuse",
"milliseconds": 1713,
"digest": true,
"grounded": true,
"tool_calls": 0,
"usage": {
"PromptTokens": 1238,
"CompletionTokens": 215,
"TotalTokens": 1453,
"CacheHitTokens": 1024,
"CacheMissTokens": 214,
"CacheWriteTokens": 0,
"CacheWriteBilledTokens": 0,
"ReasoningTokens": 0,
"FinishReason": "stop",
"Estimated": false,
"RequestCount": 1,
"ServerToolRequests": 0,
"ContextPromptTokens": 0,
"ContextCompletionTokens": 0,
"ContextReasoningTokens": 0,
"ContextCacheHitTokens": 0,
"ContextCacheMissTokens": 0
}
},
{
"kind": "independent",
"milliseconds": 1610,
"digest": true,
"grounded": true,
"tool_calls": 0,
"usage": {
"PromptTokens": 406,
"CompletionTokens": 208,
"TotalTokens": 614,
"CacheHitTokens": 256,
"CacheMissTokens": 150,
"CacheWriteTokens": 0,
"CacheWriteBilledTokens": 0,
"ReasoningTokens": 0,
"FinishReason": "stop",
"Estimated": false,
"RequestCount": 1,
"ServerToolRequests": 0,
"ContextPromptTokens": 0,
"ContextCompletionTokens": 0,
"ContextReasoningTokens": 0,
"ContextCacheHitTokens": 0,
"ContextCacheMissTokens": 0
}
}
]
},
{
"model": "deepseek-flash",
"rich": true,
"trial": 1,
"results": [
{
"kind": "warm",
"milliseconds": 561,
"digest": false,
"grounded": false,
"tool_calls": 1,
"usage": {
"PromptTokens": 1217,
"CompletionTokens": 39,
"TotalTokens": 1256,
"CacheHitTokens": 1024,
"CacheMissTokens": 193,
"CacheWriteTokens": 0,
"CacheWriteBilledTokens": 0,
"ReasoningTokens": 0,
"FinishReason": "tool_calls",
"Estimated": false,
"RequestCount": 1,
"ServerToolRequests": 0,
"ContextPromptTokens": 0,
"ContextCompletionTokens": 0,
"ContextReasoningTokens": 0,
"ContextCacheHitTokens": 0,
"ContextCacheMissTokens": 0
}
},
{
"kind": "reuse",
"milliseconds": 1774,
"digest": true,
"grounded": true,
"tool_calls": 0,
"usage": {
"PromptTokens": 1260,
"CompletionTokens": 256,
"TotalTokens": 1516,
"CacheHitTokens": 1024,
"CacheMissTokens": 236,
"CacheWriteTokens": 0,
"CacheWriteBilledTokens": 0,
"ReasoningTokens": 0,
"FinishReason": "stop",
"Estimated": false,
"RequestCount": 1,
"ServerToolRequests": 0,
"ContextPromptTokens": 0,
"ContextCompletionTokens": 0,
"ContextReasoningTokens": 0,
"ContextCacheHitTokens": 0,
"ContextCacheMissTokens": 0
}
},
{
"kind": "independent",
"milliseconds": 2271,
"digest": true,
"grounded": true,
"tool_calls": 0,
"usage": {
"PromptTokens": 447,
"CompletionTokens": 381,
"TotalTokens": 828,
"CacheHitTokens": 256,
"CacheMissTokens": 191,
"CacheWriteTokens": 0,
"CacheWriteBilledTokens": 0,
"ReasoningTokens": 0,
"FinishReason": "stop",
"Estimated": false,
"RequestCount": 1,
"ServerToolRequests": 0,
"ContextPromptTokens": 0,
"ContextCompletionTokens": 0,
"ContextReasoningTokens": 0,
"ContextCacheHitTokens": 0,
"ContextCacheMissTokens": 0
}
}
]
},
{
"model": "deepseek-flash",
"rich": true,
"trial": 2,
"results": [
{
"kind": "warm",
"milliseconds": 610,
"digest": false,
"grounded": false,
"tool_calls": 1,
"usage": {
"PromptTokens": 1217,
"CompletionTokens": 39,
"TotalTokens": 1256,
"CacheHitTokens": 1024,
"CacheMissTokens": 193,
"CacheWriteTokens": 0,
"CacheWriteBilledTokens": 0,
"ReasoningTokens": 0,
"FinishReason": "tool_calls",
"Estimated": false,
"RequestCount": 1,
"ServerToolRequests": 0,
"ContextPromptTokens": 0,
"ContextCompletionTokens": 0,
"ContextReasoningTokens": 0,
"ContextCacheHitTokens": 0,
"ContextCacheMissTokens": 0
}
},
{
"kind": "reuse",
"milliseconds": 2263,
"digest": true,
"grounded": true,
"tool_calls": 0,
"usage": {
"PromptTokens": 1260,
"CompletionTokens": 276,
"TotalTokens": 1536,
"CacheHitTokens": 1024,
"CacheMissTokens": 236,
"CacheWriteTokens": 0,
"CacheWriteBilledTokens": 0,
"ReasoningTokens": 0,
"FinishReason": "stop",
"Estimated": false,
"RequestCount": 1,
"ServerToolRequests": 0,
"ContextPromptTokens": 0,
"ContextCompletionTokens": 0,
"ContextReasoningTokens": 0,
"ContextCacheHitTokens": 0,
"ContextCacheMissTokens": 0
}
},
{
"kind": "independent",
"milliseconds": 1783,
"digest": true,
"grounded": true,
"tool_calls": 0,
"usage": {
"PromptTokens": 447,
"CompletionTokens": 242,
"TotalTokens": 689,
"CacheHitTokens": 256,
"CacheMissTokens": 191,
"CacheWriteTokens": 0,
"CacheWriteBilledTokens": 0,
"ReasoningTokens": 0,
"FinishReason": "stop",
"Estimated": false,
"RequestCount": 1,
"ServerToolRequests": 0,
"ContextPromptTokens": 0,
"ContextCompletionTokens": 0,
"ContextReasoningTokens": 0,
"ContextCacheHitTokens": 0,
"ContextCacheMissTokens": 0
}
}
]
},
{
"model": "deepseek-flash",
"rich": true,
"trial": 3,
"results": [
{
"kind": "warm",
"milliseconds": 834,
"digest": false,
"grounded": false,
"tool_calls": 1,
"usage": {
"PromptTokens": 1217,
"CompletionTokens": 39,
"TotalTokens": 1256,
"CacheHitTokens": 1024,
"CacheMissTokens": 193,
"CacheWriteTokens": 0,
"CacheWriteBilledTokens": 0,
"ReasoningTokens": 0,
"FinishReason": "tool_calls",
"Estimated": false,
"RequestCount": 1,
"ServerToolRequests": 0,
"ContextPromptTokens": 0,
"ContextCompletionTokens": 0,
"ContextReasoningTokens": 0,
"ContextCacheHitTokens": 0,
"ContextCacheMissTokens": 0
}
},
{
"kind": "reuse",
"milliseconds": 1575,
"digest": true,
"grounded": true,
"tool_calls": 0,
"usage": {
"PromptTokens": 1260,
"CompletionTokens": 210,
"TotalTokens": 1470,
"CacheHitTokens": 1024,
"CacheMissTokens": 236,
"CacheWriteTokens": 0,
"CacheWriteBilledTokens": 0,
"ReasoningTokens": 0,
"FinishReason": "stop",
"Estimated": false,
"RequestCount": 1,
"ServerToolRequests": 0,
"ContextPromptTokens": 0,
"ContextCompletionTokens": 0,
"ContextReasoningTokens": 0,
"ContextCacheHitTokens": 0,
"ContextCacheMissTokens": 0
}
},
{
"kind": "independent",
"milliseconds": 1815,
"digest": true,
"grounded": true,
"tool_calls": 0,
"usage": {
"PromptTokens": 447,
"CompletionTokens": 298,
"TotalTokens": 745,
"CacheHitTokens": 256,
"CacheMissTokens": 191,
"CacheWriteTokens": 0,
"CacheWriteBilledTokens": 0,
"ReasoningTokens": 0,
"FinishReason": "stop",
"Estimated": false,
"RequestCount": 1,
"ServerToolRequests": 0,
"ContextPromptTokens": 0,
"ContextCompletionTokens": 0,
"ContextReasoningTokens": 0,
"ContextCacheHitTokens": 0,
"ContextCacheMissTokens": 0
}
}
]
}
]
+4
View File
@@ -103,6 +103,10 @@ func newHarness(t *testingDir, p provider.Provider, window int, rec *callRecorde
ContextWindow: window,
ArchiveDir: filepath.Join(t.dir, "archive"),
SessionPath: path,
RecentKeep: 4,
// boot's default when cfg.Agent.Keep is unset; without it the bench
// would measure a configuration no real session runs.
KeepPolicy: agent.KeepErrors,
}, rec.sink())
return &harness{sess: sess, agentA: a, path: path, calls: rec}
}
@@ -108,6 +108,7 @@ func newAgent(p provider.Provider, sess *sessionstore.Session, path string, sink
return agent.New(p, tool.NewRegistry(), sess, agent.Options{
ContextWindow: windowTokens,
CompactRatio: compactRatio,
RecentKeep: 2,
SessionPath: path,
WorkspaceID: "cm-e2e",
ModelRef: "deepseek/" + model,
+98
View File
@@ -0,0 +1,98 @@
# Context retrieval bench
Two questions about long-horizon memory, and the harness that makes their
answers trustworthy.
-mode=preflight build all 12 tasks under every arm and check the contract
-mode=snippet does a rank-1 hit hand over the whole answer?
-mode=adversarial send a model after the answer through every host surface
-mode=run-search Experiment R: is searchable canonical a real capability?
-mode=run-boundary Experiment I, paired: 6 tasks across their own cue boundary
-mode=run-index Experiment I, full: 6 tasks x 4 index budgets
-mode=calibrate find each index task's PlantAfterGen for its cue tier
`-dry` drives any run mode with a scripted provider, for nothing. Real runs need
`DEEPSEEK_API_KEY`.
## Status
**Searchable canonical recall: proven.** Six of six recovered with search on,
three of six with it off, and the successes without it came from enumerating
addresses by hand at roughly nine times the page-in cost (139 vs 1292
tokens/task). Measured after three separate leaks were closed, so the numbers
are from a clean environment.
**Fold index marginal utility: unresolved.** Both the positive and the negative
estimates are invalidated. The first Stage 2 ran in a contaminated environment.
The clean paired batch found a consistent effect (six of six tasks searched less
and paged in less with the cue visible), and the same-batch dose-response did
not reproduce it (`boundary-aligned 0/4`, recall-token delta reversing sign).
The most likely reason is that two of the six index tasks carry heavy-tail
stopping behaviour that swamps the effect being measured.
Existing evidence is not sufficient to tune the shipped 1% budget in either
direction. Reopening this wants a new corpus, not more samples: see
"Index corpus v2" below.
**Things that turned out not to be problems**, each after being measured rather
than argued about:
| Suspected | Measured |
| --- | --- |
| Model writes poor queries (2.17 searches/task) | 22/23 found the target on query 1; the extra searches are one round's parallel fan-out |
| Model reaches for the workspace before memory (7/11) | Linear event order misread; by model round it is MemoryFirst 4/6 |
| Snippets too short for multi-value answers | 240 runes covers 12/12 tasks at 100%; records are 30-179 runes |
| A general stopping failure | 37/44 runs issued no search after the answer was in hand; all 7 that did belong to 2 tasks |
## Measurement distinctions this bench had to learn
Each of these reversed a conclusion once. They are enforced in the schema now,
not left to whoever reads the numbers next.
- **TargetHit is not EvidenceSufficient.** A rank-1 hit can return a window too
short to answer with. `FirstHitCoverage` records how much of the scored answer
the first hit actually handed over.
- **Linear event order is not model-round order.** Two calls in one round are a
parallel fan-out, not a preference. Routing compares rounds.
- **An isolated workdir is not an isolated host.** The sandbox mounts the host
read-only by design. Three leaks were found and closed: the corpus source, the
fixture transcript, and its event log sidecar.
- **A count is not its evidence.** Query text, snippet contents and trajectories
are all persisted now, because three analyses in a row needed a paid re-run to
ask a question of data already collected.
## How a run stays honest
The corpus holds templates, never answers: every scored literal is a `{{var}}`
instantiated per run, so grepping this directory reveals the question and not
the answer. `TestNoAnswerLiteralExistsInTheRepository` asserts it with
`git grep`. Preflight rebuilds every arm and checks the answer is unreadable in
the real `provider.Request`, the probe query ranks the target within five, and
an index task's cue sits at exactly the scales its tier names. The fixture is
removed from disk once the agent holds it in memory, and the directory is
scanned for answer literals before the first provider request and again after
the turn. Any answer reaching the model through a tool that is not `recall`
marks the run contaminated and takes it out of every statistic.
## Index corpus v2, if this is reopened
The present index tasks are not a fit substrate: two of six carry stopping
behaviour that dominates the signal. A task should qualify only if a run-index
run reports a `stopping_class` other than `PostSufficientRetrieval` or
`PostSufficientRunaway`: `SnippetStop`, `SnippetThenRead` and `DefensiveRead`
all qualify, and a defensive read is deliberately not counted as waste. The
class comes from the three counts in that report — `searches_after_sufficient`,
`reads_after_sufficient` and `escapes_after_sufficient` — together with how
sufficiency was reached and how many rounds followed it. The remaining
requirements still apply — successful tool call, answer in the output rather
than the cue, rank 1, `FirstHitCoverage` complete, one or two simple values,
nothing in the workspace. Keep i03 and i04 — as a stopping corpus, not an
efficiency one.
`-mode=calibrate` is not a screening tool: it builds and reads back the
fixtures, and reads nothing else, so its cue-visibility profile is the same for
every task.
Then repeats rather than levels: cue-present against cue-absent on each task's
own boundary, three runs per cell. What is unknown is the variance, and four
budgets sampled once each cannot show it.
+171
View File
@@ -0,0 +1,171 @@
// capture.go — what the agent actually hands a provider.
package main
import (
"context"
"encoding/json"
"fmt"
"reasonix/internal/state/sessionstore"
"strings"
"reasonix/internal/assembly/boot"
"reasonix/internal/contract/ablation"
"reasonix/internal/contract/event"
"reasonix/internal/contract/provider"
"reasonix/internal/contract/tool"
"reasonix/internal/runtime/agent"
_ "reasonix/internal/tools/builtin"
)
// A projection is what the agent believes; a provider.Request is what the model
// is told. Rendering, sanitization and tool-schema construction sit between
// them, so the arm's contract is checked on the request.
type captureProvider struct {
requests []provider.Request
}
func (p *captureProvider) Name() string { return "contextbench-capture" }
// Stream records the request and answers with a clean terminal response. An
// empty answer would buy an empty-final retry and a second request, which is
// not the one round this is here to inspect.
func (p *captureProvider) Stream(_ context.Context, req provider.Request) (<-chan provider.Chunk, error) {
p.requests = append(p.requests, cloneRequest(req))
ch := make(chan provider.Chunk, 2)
ch <- provider.Chunk{Type: provider.ChunkText, Text: "contextbench preflight complete"}
ch <- provider.Chunk{Type: provider.ChunkDone}
close(ch)
return ch, nil
}
func cloneRequest(req provider.Request) provider.Request {
out := req
out.Messages = append([]provider.Message(nil), req.Messages...)
out.Tools = append([]provider.ToolSchema(nil), req.Tools...)
return out
}
// captureRequest rebuilds the session the way a resume does — transcript from
// disk, projection from its sidecar, the shipped tool surface at this arm —
// and returns the request the first round hands the provider.
func captureRequest(f builtFixture, arm ablation.Set, prompt string) (provider.Request, error) {
sess, err := sessionstore.LoadSession(f.Path)
if err != nil {
return provider.Request{}, fmt.Errorf("load session: %w", err)
}
reg := tool.NewRegistry()
for _, t := range tool.Builtins() {
reg.Add(t)
}
boot.ApplyUnifiedProviderToolSurface(reg, false, arm)
cap := &captureProvider{}
a := agent.New(cap, reg, sess, agent.Options{
ContextWindow: fixtureWindow, CompactRatio: 0.5, RecentKeep: 2,
SessionPath: f.Path, KeepPolicy: agent.KeepErrors, Ablation: arm,
}, event.Discard)
a.LoadProjectionSidecar(f.Path)
if err := a.Run(context.Background(), prompt); err != nil {
return provider.Request{}, fmt.Errorf("run: %w", err)
}
// The turn's own request is the one carrying tools. A fold on the way in
// asks the same provider for a digest, and that request has none — taking
// the first request blindly would inspect the summarizer's prompt.
for _, req := range cap.requests {
if len(req.Tools) > 0 {
return req, nil
}
}
return provider.Request{}, fmt.Errorf("no request carried tools (%d captured)", len(cap.requests))
}
// surfaceField is one place a model could read a string, named so a leak
// report says where rather than only that.
type surfaceField struct {
Path string
Text string
}
// providerSurface enumerates every provider-visible string in a request.
// Enumerated rather than marshalled whole: a Message also carries local-only
// fields, and scanning those would fail on bytes no model ever sees.
func providerSurface(req provider.Request) []surfaceField {
var out []surfaceField
add := func(path, text string) {
if strings.TrimSpace(text) != "" {
out = append(out, surfaceField{Path: path, Text: text})
}
}
for i, m := range req.Messages {
add(fmt.Sprintf("messages[%d].content", i), m.Content)
add(fmt.Sprintf("messages[%d].reasoning_content", i), m.ReasoningContent)
add(fmt.Sprintf("messages[%d].name", i), m.Name)
for j, tc := range m.ToolCalls {
add(fmt.Sprintf("messages[%d].tool_calls[%d].name", i, j), tc.Name)
add(fmt.Sprintf("messages[%d].tool_calls[%d].arguments", i, j), tc.Arguments)
}
for j, item := range m.ResponsesItems {
add(fmt.Sprintf("messages[%d].responses_items[%d]", i, j), string(item))
}
}
for i, t := range req.Tools {
add(fmt.Sprintf("tools[%d].name", i), t.Name)
add(fmt.Sprintf("tools[%d].description", i), t.Description)
if len(t.Parameters) > 0 {
b, err := json.Marshal(t.Parameters)
if err == nil {
add(fmt.Sprintf("tools[%d].parameters", i), string(b))
}
}
}
return out
}
// findInSurface reports every place a marker is readable.
func findInSurface(req provider.Request, marker string) []string {
var hits []string
for _, f := range providerSurface(req) {
if strings.Contains(f.Text, marker) {
hits = append(hits, f.Path)
}
}
return hits
}
// recallToolSchema returns the recall tool as the provider sees it.
func recallToolSchema(req provider.Request) (provider.ToolSchema, bool) {
for _, t := range req.Tools {
if t.Name == "recall" {
return t, true
}
}
return provider.ToolSchema{}, false
}
// localOnlyLeaks reports host-side fields that should never survive the
// provider boundary. A hit here is a boundary regression, marker or no marker.
func localOnlyLeaks(req provider.Request) []string {
var out []string
for i, m := range req.Messages {
switch {
case m.RawContent != "":
out = append(out, fmt.Sprintf("messages[%d].raw_content", i))
case m.LocalOnly:
out = append(out, fmt.Sprintf("messages[%d].local_only", i))
case m.DecisionReceipt != nil || len(m.DecisionReceipts) > 0:
out = append(out, fmt.Sprintf("messages[%d].decision_receipt", i))
case m.ToolExecution != nil:
out = append(out, fmt.Sprintf("messages[%d].tool_execution", i))
case len(m.MemoryCitations) > 0:
out = append(out, fmt.Sprintf("messages[%d].memory_citations", i))
}
}
return out
}
// searchWording is what a search-off arm must never be told. The arm exists to
// reproduce the runtime before search; an arm offered a capability it does not
// have is a different experiment.
var searchWording = []string{"recall with a query", "search the whole folded region", "query still finds"}
@@ -0,0 +1,648 @@
// contextbench.go — what the model did with the affordance, read off structure.
package main
import (
"encoding/json"
"fmt"
"io"
"regexp"
"slices"
"strconv"
"strings"
"reasonix/internal/base/tokencount"
"reasonix/internal/contract/provider"
)
// Every judgement here reads a tool call's name and its JSON arguments, or a
// canonical position, and never the model's prose. "I will search the history"
// is not a search; a recall call with a query is.
// failure stages, in funnel order. Exactly one is reported per run.
const (
stageReadMissed = "NoSearch" // never asked, and never read the target either
stageSearchMiss = "SearchMiss" // searched, target never came back
stageHitNotRead = "HitNotRead" // target was returned and never opened
stageAnswerWrong = "ReadButAnswerWrong" // opened it and still answered wrong
stageRecovered = "Recovered"
stageContaminated = "Contaminated"
// Routing is judged by model round, not by the linear order of tool events:
// two calls in one round are one parallel fan-out, and calling that
// "workspace first" would report a choice the model never made.
routeMemoryFirst = "MemoryFirst"
routeWorkspaceFirst = "WorkspaceFirst"
routeParallel = "Parallel"
routeNeither = "Neither"
stageCueRead = "DirectRead" // read the target from an index address, no search needed
stageNoRetrieval = "NoRetrieval" // answered without touching recall at all
toolRecall = "recall"
)
// searchAttempt is one query the model wrote, and what it got back. Kept per
// attempt rather than aggregated: the question is why a retrieval that the host
// probe ranks first takes the model more than one try.
type searchAttempt struct {
Ordinal int `json:"ordinal"`
Round int `json:"round"`
Query string `json:"query"`
TargetRank int `json:"target_rank"` // 0 = the target did not come back
ResultCount int `json:"result_count"`
// Reformulation classifies this query against the one before it, and
// MissingProbeTerms are the host probe's terms this query left out. Both
// are derived from token sets: no judge, and nothing read out of prose.
Reformulation string `json:"reformulation,omitempty"`
MissingProbeTerms []string `json:"missing_probe_terms,omitempty"`
}
// contextMetrics is one run's behaviour.
type contextMetrics struct {
Task string `json:"task"`
Arm string `json:"arm"`
SearchCalls int `json:"search_calls"`
// SearchHits counts searches that returned anything; TargetSearchHits
// counts searches that returned the target. The gap is query quality.
SearchHits int `json:"search_hits"`
TargetSearchHits int `json:"target_search_hits"`
FirstTargetRank int `json:"first_target_rank"` // 0 = never returned
// FirstHitCoverage: how much of the answer the first hit handed over.
// rank=1 with coverage 1/3 is a window too short to answer with, not a
// model that would not stop.
FirstHitCoverage int `json:"first_hit_coverage"`
AnswerValues int `json:"answer_values"`
ReadCalls int `json:"read_calls"`
TargetRead bool `json:"target_read"`
ReadAfterHit bool `json:"read_after_hit"`
// RecallReadWithoutSearch: the index addressed the target, so no search was
// needed. Not "direct read" — read_file is a read too, and in an audit the
// two mean opposite things.
RecallReadWithoutSearch bool `json:"recall_read_without_search"`
SearchThenRecallRead bool `json:"search_then_recall_read"`
FirstSearchRound int `json:"first_search_round"`
FirstReadRound int `json:"first_read_round"`
RetrievalRounds int `json:"retrieval_rounds"`
// RecallReturnedTokens is what recall paged back in. The index saves
// resident prefix; this is what is paid dynamically instead.
RecallReturnedTokens int `json:"recall_returned_tokens"`
Searches []searchAttempt `json:"searches,omitempty"`
// PostHitSearches are queries issued after the target had already been
// returned: not a query problem, a result-to-read problem.
PostHitSearches int `json:"post_hit_searches"`
AnswerRecovered bool `json:"answer_recovered"`
// AnswerAmbiguous marks an answer that offered more than one candidate for
// a value. Markers matching proves the right fact appears; it does not
// prove it was submitted as the answer. Recorded, never acted on.
AnswerAmbiguous bool `json:"answer_ambiguous,omitempty"`
AmbiguousVar string `json:"ambiguous_var,omitempty"`
MissingMarkers []string `json:"missing_markers,omitempty"`
// FinalAnswer is kept bounded so a surprising run can be read back. A
// metric nobody can audit is a claim, not a measurement.
FinalAnswer string `json:"final_answer,omitempty"`
FailureStage string `json:"failure_stage"`
TimeoutReason string `json:"timeout_reason,omitempty"`
// UnexpectedWorkTools are investigation tools used instead of recall.
// Reaching for the workspace is behaviour, not noise: how a model routes
// its uncertainty when memory is missing is part of what is measured.
UnexpectedWorkTools []string `json:"unexpected_work_tools,omitempty"`
EscapeCalls int `json:"escape_calls"`
EscapeBeforeFirstRecall int `json:"escape_before_first_recall"`
EscapeAfterFirstRecall int `json:"escape_after_first_recall"`
// Contaminated: the answer was reachable through something other than
// recall. Asserting this beats trusting the isolation, and a run that trips
// it leaves the statistics rather than inflating one.
Contaminated bool `json:"contaminated,omitempty"`
LeakedVia string `json:"leaked_via,omitempty"`
LeakedMarker string `json:"leaked_marker,omitempty"`
LeakedArgs string `json:"leaked_args,omitempty"`
LeakedOutput string `json:"leaked_output,omitempty"`
// EscapeFoundAnswer separates "looked at the workspace" from "found the
// answer there": different facts about the same call.
EscapeFoundAnswer bool `json:"escape_found_answer,omitempty"`
CueVisible bool `json:"cue_visible"`
// trajectory is kept out of the metric line and written beside it, so a
// future question can be asked of this batch instead of a new one.
trajectory []provider.Message
// CueDirectRead: the cue was addressed, the first memory action was a
// positions read, and it covered the target. Crediting any positions call
// would hand the index a search's work.
CueDirectRead bool `json:"cue_direct_read"`
// Sufficiency, not hit. A search that returns the target at rank 1 has not
// necessarily returned the answer: the snippet is bounded. The stopping
// boundary is the round where everything needed was readable.
FirstSufficientRound int `json:"first_sufficient_round"`
SufficientVia string `json:"sufficient_via"` // snippet | read | never
RoundsAfterSufficient int `json:"rounds_after_sufficient"`
SearchesAfterSufficient int `json:"searches_after_sufficient"`
ReadsAfterSufficient int `json:"reads_after_sufficient"`
EscapesAfterSufficient int `json:"escapes_after_sufficient"`
RecallTokensAfterSufficient int `json:"recall_tokens_after_sufficient"`
StoppingClass string `json:"stopping_class"`
// What happened after the answer was already in hand. A timeout that spent
// seven rounds past its target is a stopping problem, and extending the
// budget would only buy the loop more time.
TargetFirstSeenRound int `json:"target_first_seen_round"`
RoundsAfterTarget int `json:"rounds_after_target"`
SearchesAfterTarget int `json:"searches_after_target"`
RecallTokensAfterTarget int `json:"recall_tokens_after_target"`
TargetHeldAtEnd bool `json:"target_held_at_end"`
// Routing is where the model looked first when its memory was missing.
Routing string `json:"routing"`
FirstRecallRound int `json:"first_recall_round"`
FirstEscapeRound int `json:"first_escape_round"`
}
// hitPosition matches the address recall prints for a search hit. Reading our
// own rendered format, not guessing at the model's words.
var hitPosition = regexp.MustCompile(`(?m)^#(\d+) `)
// scoreRun reads the messages one turn appended and reports what happened.
// runScorer walks one turn's messages, holding the state a judgement needs:
// which calls are open, whether the target has come back, and which search
// each result belongs to.
type runScorer struct {
m contextMetrics
inst fixtureInstance
target int
calls map[string]provider.ToolCall
pending map[string]int
round int
sawTargetHit bool
// seen accumulates everything recall has shown the model, so sufficiency is
// judged on what it could read rather than on which call produced it.
seen strings.Builder
sufficient bool
}
// scoreRun reads the messages one turn appended and reports what happened.
func scoreRun(msgs []provider.Message, inst fixtureInstance, target int, arm string, cueVisible bool) contextMetrics {
s := &runScorer{
m: contextMetrics{Task: inst.Task.ID, Arm: arm, CueVisible: cueVisible},
inst: inst, target: target,
calls: map[string]provider.ToolCall{}, pending: map[string]int{},
}
for _, msg := range msgs {
if len(msg.ToolCalls) > 0 {
s.round++
}
for _, tc := range msg.ToolCalls {
s.calls[tc.ID] = tc
s.observeCall(tc)
}
if msg.Role == provider.RoleTool {
s.observeResult(msg)
}
}
return s.finish(cueVisible)
}
// observeCall records one tool call: a recall search, a recall read, or a
// reach for the workspace.
func (s *runScorer) observeCall(tc provider.ToolCall) {
m := &s.m
if tc.Name != toolRecall {
if !containsString(m.UnexpectedWorkTools, tc.Name) {
m.UnexpectedWorkTools = append(m.UnexpectedWorkTools, tc.Name)
}
m.EscapeCalls++
if s.sufficient {
m.EscapesAfterSufficient++
}
if m.FirstEscapeRound == 0 {
m.FirstEscapeRound = s.round
}
if m.SearchCalls+m.ReadCalls == 0 {
m.EscapeBeforeFirstRecall++
} else {
m.EscapeAfterFirstRecall++
}
return
}
if m.FirstRecallRound == 0 {
m.FirstRecallRound = s.round
}
query, positions := recallArgs(tc.Arguments)
switch {
case query != "":
m.SearchCalls++
if m.FirstSearchRound == 0 {
m.FirstSearchRound = s.round
}
if s.sawTargetHit {
m.PostHitSearches++
m.SearchesAfterTarget++
}
if s.sufficient {
m.SearchesAfterSufficient++
}
m.Searches = append(m.Searches, searchAttempt{Ordinal: m.SearchCalls, Round: s.round, Query: query})
s.pending[tc.ID] = len(m.Searches) - 1
case len(positions) > 0:
m.ReadCalls++
if s.sufficient {
m.ReadsAfterSufficient++
}
if m.FirstReadRound == 0 {
m.FirstReadRound = s.round
}
// A read covers the target when it names its position: recall returns
// the tool results under the caller's own address.
if containsInt(positions, s.target) {
m.TargetRead = true
switch {
case s.sawTargetHit:
m.ReadAfterHit, m.SearchThenRecallRead = true, true
case m.SearchCalls == 0:
m.RecallReadWithoutSearch = true
}
}
}
}
// observeResult reads what a call returned: contamination, page-in cost, and
// whether this search brought the target back.
func (s *runScorer) observeResult(msg provider.Message) {
m := &s.m
call, ok := s.calls[msg.ToolCallID]
if !ok {
return
}
if call.Name != toolRecall {
// Codenames only: a three-digit epoch appears in any long file by
// chance, and one did. A model holding the codename holds the numbers
// beside it anyway.
for _, marker := range s.inst.AnswerMarkers {
if len(marker) >= 6 && strings.Contains(msg.Content, marker) {
m.Contaminated, m.LeakedVia, m.LeakedMarker = true, call.Name, marker
m.EscapeFoundAnswer = true
m.LeakedArgs, m.LeakedOutput = truncate(call.Arguments, 200), truncate(msg.Content, 300)
}
}
return
}
cost := tokencount.Text(msg.Content)
m.RecallReturnedTokens += cost
if s.sawTargetHit {
m.RecallTokensAfterTarget += cost
}
if s.sufficient {
m.RecallTokensAfterSufficient += cost
}
// Everything recall showed the model, judged as a whole: the answer may
// arrive across a snippet and a read, and either way it is readable.
s.seen.WriteString(msg.Content)
s.seen.WriteString("\n")
if !s.sufficient && allMarkersIn(s.seen.String(), s.inst.AnswerMarkers) {
s.sufficient = true
m.FirstSufficientRound = s.round
query, _ := recallArgs(call.Arguments)
m.SufficientVia = "read"
if query != "" {
m.SufficientVia = "snippet"
}
}
if query, _ := recallArgs(call.Arguments); query == "" {
return
}
positions := searchHitPositions(msg.Content)
if len(positions) > 0 {
m.SearchHits++
}
rank := indexOfInt(positions, s.target)
if i, ok := s.pending[msg.ToolCallID]; ok {
m.Searches[i].TargetRank, m.Searches[i].ResultCount = rank, len(positions)
}
if rank == 0 {
return
}
m.TargetSearchHits++
if !s.sawTargetHit {
m.TargetFirstSeenRound = s.round
m.FirstHitCoverage = answerCoverage(msg.Content, s.inst.AnswerMarkers)
}
s.sawTargetHit = true
if m.FirstTargetRank == 0 {
m.FirstTargetRank = rank
}
}
// finish derives what only the whole walk can say.
func (s *runScorer) finish(cueVisible bool) contextMetrics {
m := s.m
m.RetrievalRounds = m.SearchCalls + m.ReadCalls
m.TargetHeldAtEnd = s.sawTargetHit || m.TargetRead
if m.TargetFirstSeenRound > 0 {
m.RoundsAfterTarget = s.round - m.TargetFirstSeenRound
}
m.Routing = routing(m.FirstRecallRound, m.FirstEscapeRound)
m.CueDirectRead = cueVisible && m.RecallReadWithoutSearch
if m.SufficientVia == "" {
m.SufficientVia = "never"
}
if m.FirstSufficientRound > 0 {
m.RoundsAfterSufficient = s.round - m.FirstSufficientRound
}
m.AnswerValues = len(s.inst.AnswerMarkers)
m.StoppingClass = stoppingClass(m)
classifySearches(&m, s.inst)
return m
}
// scoreAnswer grades the final text on the task's markers. Nonces, matched
// exactly: no judge model, and nothing a general answer could satisfy.
func (m *contextMetrics) scoreAnswer(final string, inst fixtureInstance) {
lower := strings.ToLower(final)
for _, marker := range inst.AnswerMarkers {
if !strings.Contains(lower, strings.ToLower(marker)) {
m.MissingMarkers = append(m.MissingMarkers, marker)
}
}
m.AnswerRecovered = len(m.MissingMarkers) == 0
m.AnswerAmbiguous, m.AmbiguousVar = ambiguousAnswer(final, inst)
m.FailureStage = m.stage()
if len(final) > 1200 {
final = final[:1200] + "…"
}
m.FinalAnswer = final
}
// stage places the run in the funnel. One label, chosen at the first step that
// did not happen, so a table of stages reads as where the retrieval broke.
func (m contextMetrics) stage() string {
switch {
case m.Contaminated:
return stageContaminated
case m.AnswerRecovered && m.RecallReadWithoutSearch:
return stageCueRead
case m.AnswerRecovered:
return stageRecovered
case m.TargetRead:
return stageAnswerWrong
case m.TargetSearchHits > 0:
return stageHitNotRead
case m.SearchCalls > 0:
return stageSearchMiss
case m.ReadCalls > 0:
return stageReadMissed
default:
return stageNoRetrieval
}
}
// recallArgs pulls the two fields that say which operation a recall call is.
func recallArgs(args string) (query string, positions []int) {
var parsed struct {
Query string `json:"query"`
Positions []int `json:"positions"`
}
if err := json.Unmarshal([]byte(args), &parsed); err != nil {
return "", nil
}
return strings.TrimSpace(parsed.Query), parsed.Positions
}
// searchHitPositions reads the addresses out of a search result, in rank order.
func searchHitPositions(text string) []int {
var out []int
for _, match := range hitPosition.FindAllStringSubmatch(text, -1) {
if n, err := strconv.Atoi(match[1]); err == nil {
out = append(out, n)
}
}
return out
}
func containsInt(list []int, want int) bool { return slices.Contains(list, want) }
// indexOfInt returns a 1-based rank, or 0 when absent.
func indexOfInt(list []int, want int) int {
for i, v := range list {
if v == want {
return i + 1
}
}
return 0
}
func containsString(list []string, want string) bool { return slices.Contains(list, want) }
// funnel summarizes one arm's runs the way the experiment is read.
type funnel struct {
Arm string
Runs int
Scored, Contaminated, Errored int
Searched, TargetHit, TargetRead, Answered int
CueReads int
SearchCalls, ReadCalls, RecallTokens int
Escapes, EscapeCalls int
Stages, Routes map[string]int
}
func summarize(arm string, runs []contextMetrics) funnel {
f := funnel{Arm: arm, Runs: len(runs), Stages: map[string]int{}, Routes: map[string]int{}}
for _, r := range runs {
if r.Contaminated {
f.Contaminated++
f.Stages[r.FailureStage]++
continue
}
if r.FailureStage == "RunError" {
f.Errored++
f.Stages[r.FailureStage]++
continue
}
f.Scored++
if r.SearchCalls > 0 {
f.Searched++
}
if r.TargetSearchHits > 0 {
f.TargetHit++
}
if r.TargetRead {
f.TargetRead++
}
if r.AnswerRecovered {
f.Answered++
}
if r.RecallReadWithoutSearch {
f.CueReads++
}
if len(r.UnexpectedWorkTools) > 0 {
f.Escapes++
}
f.EscapeCalls += r.EscapeCalls
f.SearchCalls += r.SearchCalls
f.ReadCalls += r.ReadCalls
f.RecallTokens += r.RecallReturnedTokens
f.Stages[r.FailureStage]++
if r.FailureStage != "RunError" {
f.Routes[r.Routing]++
}
}
return f
}
func (f funnel) line() string {
per := func(n int) float64 {
if f.Scored == 0 {
return 0
}
return float64(n) / float64(f.Scored)
}
line := fmt.Sprintf("%-14s runs=%-3d answered=%d/%d searched=%d target-hit=%d cue-read=%d search/task=%.2f recall-tok/task=%.0f escape/task=%.2f",
f.Arm, f.Scored, f.Answered, f.Scored, f.Searched, f.TargetHit, f.CueReads,
per(f.SearchCalls), per(f.RecallTokens), per(f.EscapeCalls))
if f.Contaminated > 0 {
line += fmt.Sprintf(" CONTAMINATED=%d", f.Contaminated)
}
if f.Errored > 0 {
line += fmt.Sprintf(" ERRORED=%d", f.Errored)
}
return line
}
func writeJSONLine(w io.Writer, v any) error {
b, err := json.Marshal(v)
if err != nil {
return err
}
_, err = fmt.Fprintln(w, string(b))
return err
}
func truncate(s string, n int) string {
if len(s) <= n {
return s
}
return s[:n] + "…"
}
// routing says where the model looked first, comparing rounds rather than
// event order. Same round is a parallel fan-out, not a preference.
func routing(recallRound, escapeRound int) string {
switch {
case recallRound == 0 && escapeRound == 0:
return routeNeither
case recallRound == 0:
return routeWorkspaceFirst
case escapeRound == 0:
return routeMemoryFirst
case recallRound < escapeRound:
return routeMemoryFirst
case escapeRound < recallRound:
return routeWorkspaceFirst
default:
return routeParallel
}
}
// ambiguousAnswer reports an answer naming more than one candidate of a
// codename's shape. Only codenames are checked: they carry a stem this run
// declared, so two distinct "fence-…" values in one answer is one answer too
// many. Numbers have no such shape and are left alone.
func ambiguousAnswer(final string, inst fixtureInstance) (bool, string) {
for _, spec := range inst.Task.Vars {
if spec.Kind != varCodename {
continue
}
seen := map[string]bool{}
for _, match := range regexp.MustCompile(regexp.QuoteMeta(spec.Word)+`-[a-z0-9]{3,}`).FindAllString(final, -1) {
seen[match] = true
}
if len(seen) > 1 {
return true, spec.Name
}
}
return false, ""
}
func allMarkersIn(text string, markers []string) bool {
if len(markers) == 0 {
return false
}
lower := strings.ToLower(text)
for _, marker := range markers {
if !strings.Contains(lower, strings.ToLower(marker)) {
return false
}
}
return true
}
// Stopping classes. The boundary is sufficiency, not the target coming back:
// a rank-1 hit whose snippet was too short to answer leaves a read entirely
// justified, and counting that as waste would blame the model for the bound.
const (
stopSnippet = "SnippetStop" // enough arrived, and the turn ended
stopSnippetRead = "SnippetThenRead" // the snippet was short; the read finished it
stopDefensiveRead = "DefensiveRead" // enough had arrived, one read confirmed it
stopPostRetrieval = "PostSufficientRetrieval" // enough had arrived, and retrieval continued
stopRunaway = "PostSufficientRunaway" // it continued for rounds, or ran out of budget
stopNeverEnough = "NeverSufficient"
)
// runawayRounds is where continuing stops looking like confirmation.
const runawayRounds = 3
func stoppingClass(m contextMetrics) string {
if m.FirstSufficientRound == 0 {
return stopNeverEnough
}
after := m.SearchesAfterSufficient + m.ReadsAfterSufficient + m.EscapesAfterSufficient
switch {
case after == 0 && m.SufficientVia == "snippet":
return stopSnippet
case after == 0:
return stopSnippetRead
case m.RoundsAfterSufficient >= runawayRounds || m.FailureStage == "RunError":
return stopRunaway
case after == 1 && m.ReadsAfterSufficient == 1:
return stopDefensiveRead
default:
return stopPostRetrieval
}
}
// reportStopping is the funnel that matters once retrieval succeeds: what the
// turn did after it already had everything it needed.
func reportStopping(all []contextMetrics) string {
classes := map[string]int{}
var extraRounds, extraTokens []float64
sufficient, answeredNext := 0, 0
for _, m := range all {
if m.Contaminated {
continue
}
classes[m.StoppingClass]++
if m.FirstSufficientRound == 0 {
continue
}
sufficient++
extraRounds = append(extraRounds, float64(m.RoundsAfterSufficient))
extraTokens = append(extraTokens, float64(m.RecallTokensAfterSufficient))
if m.RoundsAfterSufficient <= 1 {
answeredNext++
}
}
var b strings.Builder
b.WriteString("\n## After the evidence was already enough\n")
fmt.Fprintf(&b, " sufficient in %d runs; ended within one round in %d\n", sufficient, answeredNext)
fmt.Fprintf(&b, " extra rounds mean %.1f median %.1f p90 %.0f max %.0f\n",
mean(extraRounds), median(extraRounds), percentile(extraRounds, 0.9), maxOf(extraRounds))
fmt.Fprintf(&b, " extra recall tokens mean %.0f median %.0f p90 %.0f max %.0f\n",
mean(extraTokens), median(extraTokens), percentile(extraTokens, 0.9), maxOf(extraTokens))
b.WriteString(" " + countsLine(classes) + "\n")
return b.String()
}
@@ -0,0 +1,337 @@
package main
import (
"fmt"
"strings"
"testing"
"reasonix/internal/contract/provider"
)
// The scorer decides what the pilot concludes, so it is held to the same rule
// the corpus is: judgements read structure. A model that says it will search
// has not searched; a recall call carrying a query has.
const scorerTarget = 42
func recallCall(id, args string) provider.Message {
return provider.Message{Role: provider.RoleAssistant, ToolCalls: []provider.ToolCall{
{ID: id, Name: "recall", Arguments: args},
}}
}
func toolResult(id, text string) provider.Message {
return provider.Message{Role: provider.RoleTool, ToolCallID: id, Name: "recall", Content: text}
}
func hitBlock(positions ...int) string {
var b strings.Builder
b.WriteString("Folded context matching \"x\":\n")
for _, p := range positions {
fmt.Fprintf(&b, "\n#%d assistant_text\nsome snippet here\n", p)
}
return b.String()
}
func scorerTask() fixtureInstance {
return fixtureInstance{Task: contextTask{ID: "t"}, AnswerMarkers: []string{"cobalt-lark-17"}}
}
func score(t *testing.T, msgs []provider.Message, final string) contextMetrics {
t.Helper()
m := scoreRun(msgs, scorerTask(), scorerTarget, "arm", false)
m.scoreAnswer(final, scorerTask())
return m
}
// The whole funnel, walked end to end.
func TestScorerReadsTheFullRetrievalChain(t *testing.T) {
m := score(t, []provider.Message{
recallCall("c1", `{"query":"retry boundary"}`),
toolResult("c1", hitBlock(7, scorerTarget)),
recallCall("c2", fmt.Sprintf(`{"positions":[%d]}`, scorerTarget)),
toolResult("c2", "#42\n[assistant]\nthe ownership token is cobalt-lark-17"),
{Role: provider.RoleAssistant, Content: "The token was cobalt-lark-17."},
}, "The token was cobalt-lark-17.")
if m.SearchCalls != 1 || m.ReadCalls != 1 {
t.Errorf("calls = %d search / %d read, want 1/1", m.SearchCalls, m.ReadCalls)
}
if m.TargetSearchHits != 1 || m.FirstTargetRank != 2 {
t.Errorf("target hit %d at rank %d, want 1 at rank 2", m.TargetSearchHits, m.FirstTargetRank)
}
if !m.TargetRead || !m.ReadAfterHit || m.RecallReadWithoutSearch {
t.Errorf("read flags = read %v afterHit %v direct %v", m.TargetRead, m.ReadAfterHit, m.RecallReadWithoutSearch)
}
if !m.AnswerRecovered || m.FailureStage != stageRecovered {
t.Errorf("stage = %q recovered=%v, want Recovered", m.FailureStage, m.AnswerRecovered)
}
if m.RecallReturnedTokens == 0 {
t.Error("recall returned tokens were not counted")
}
}
// An index arm's claim: the address was already on screen, so no search was
// needed. That is a different outcome from finding it, and gets its own stage.
func TestScorerSeparatesCueReadFromSearch(t *testing.T) {
m := score(t, []provider.Message{
recallCall("c1", fmt.Sprintf(`{"positions":[%d]}`, scorerTarget)),
toolResult("c1", "#42\nthe ownership token is cobalt-lark-17"),
{Role: provider.RoleAssistant, Content: "cobalt-lark-17"},
}, "cobalt-lark-17")
if m.SearchCalls != 0 || !m.RecallReadWithoutSearch {
t.Errorf("search=%d direct=%v, want 0 and true", m.SearchCalls, m.RecallReadWithoutSearch)
}
if m.FailureStage != stageCueRead {
t.Errorf("stage = %q, want %q", m.FailureStage, stageCueRead)
}
}
// Every way the chain can break gets its own label, so a failed run says where
// rather than only that.
func TestScorerNamesWhereTheChainBroke(t *testing.T) {
for _, tc := range []struct {
name string
msgs []provider.Message
final string
want string
}{
{
name: "never touched recall",
msgs: []provider.Message{{Role: provider.RoleAssistant, Content: "I do not recall that."}},
final: "I do not recall that.",
want: stageNoRetrieval,
},
{
name: "searched and missed",
msgs: []provider.Message{
recallCall("c1", `{"query":"unrelated words"}`),
toolResult("c1", hitBlock(3, 9)),
},
final: "not found",
want: stageSearchMiss,
},
{
name: "found it and never opened it",
msgs: []provider.Message{
recallCall("c1", `{"query":"retry boundary"}`),
toolResult("c1", hitBlock(scorerTarget)),
},
final: "probably something about retries",
want: stageHitNotRead,
},
{
name: "read it and answered wrong",
msgs: []provider.Message{
recallCall("c1", fmt.Sprintf(`{"positions":[%d]}`, scorerTarget)),
toolResult("c1", "#42\nthe token is elsewhere"),
},
final: "I think it was amber-9",
want: stageAnswerWrong,
},
{
name: "read the wrong position",
msgs: []provider.Message{
recallCall("c1", `{"positions":[7]}`),
toolResult("c1", "#7\nnothing useful"),
},
final: "unclear",
want: stageReadMissed,
},
} {
t.Run(tc.name, func(t *testing.T) {
if got := score(t, tc.msgs, tc.final).FailureStage; got != tc.want {
t.Errorf("stage = %q, want %q", got, tc.want)
}
})
}
}
// The answer lives only in folded history, so reaching for the workspace is a
// strategy change worth seeing — including when it happens to work.
func TestScorerRecordsWorkspaceEscapes(t *testing.T) {
m := score(t, []provider.Message{
{Role: provider.RoleAssistant, ToolCalls: []provider.ToolCall{
{ID: "g1", Name: "bash", Arguments: `{"command":"rg cobalt-lark"}`},
{ID: "g2", Name: "read_file", Arguments: `{"path":"notes.md"}`},
}},
{Role: provider.RoleAssistant, Content: "cobalt-lark-17"},
}, "cobalt-lark-17")
if len(m.UnexpectedWorkTools) != 2 {
t.Fatalf("escapes = %v, want bash and read_file", m.UnexpectedWorkTools)
}
if m.SearchCalls != 0 || m.ReadCalls != 0 {
t.Errorf("a workspace tool was counted as recall: search=%d read=%d", m.SearchCalls, m.ReadCalls)
}
// It answered, so the run is Recovered — the escape is reported beside the
// outcome rather than overriding it.
if !m.AnswerRecovered {
t.Error("a correct answer was not scored as recovered")
}
}
// Narration is not action.
func TestScorerIgnoresWhatTheModelSaysItWillDo(t *testing.T) {
m := score(t, []provider.Message{
{Role: provider.RoleAssistant, Content: "Let me search the folded context with recall for the retry boundary."},
}, "Let me search the folded context with recall for the retry boundary.")
if m.SearchCalls != 0 || m.FailureStage != stageNoRetrieval {
t.Errorf("prose was scored as a search: calls=%d stage=%q", m.SearchCalls, m.FailureStage)
}
}
// Every marker must land, so a partially right answer is not a recovery.
func TestScorerRequiresEveryMarker(t *testing.T) {
task := fixtureInstance{Task: contextTask{ID: "t"}, AnswerMarkers: []string{"comet-42", "per-lineage"}}
m := scoreRun(nil, task, scorerTarget, "arm", false)
m.scoreAnswer("the salt was comet-42", task)
if m.AnswerRecovered {
t.Error("a half answer scored as recovered")
}
if len(m.MissingMarkers) != 1 || m.MissingMarkers[0] != "per-lineage" {
t.Errorf("missing = %v, want per-lineage", m.MissingMarkers)
}
}
// An answer reachable through anything but recall invalidates the run. The
// workspace is isolated so this should never fire; asserting it beats trusting
// it, and a tripped run leaves the statistics rather than inflating one.
func TestScorerInvalidatesRunsThatLeakedThroughAnotherTool(t *testing.T) {
m := score(t, []provider.Message{
{Role: provider.RoleAssistant, ToolCalls: []provider.ToolCall{
{ID: "g1", Name: "bash", Arguments: `{"command":"rg -r ."}`},
}},
{Role: provider.RoleTool, ToolCallID: "g1", Name: "bash", Content: "corpus.go: the token is cobalt-lark-17"},
{Role: provider.RoleAssistant, Content: "cobalt-lark-17"},
}, "cobalt-lark-17")
if !m.Contaminated || m.LeakedVia != "bash" || m.LeakedMarker != "cobalt-lark-17" {
t.Fatalf("leak not caught: contaminated=%v via=%q marker=%q", m.Contaminated, m.LeakedVia, m.LeakedMarker)
}
if m.FailureStage != stageContaminated {
t.Errorf("stage = %q, want %q even though every marker matched", m.FailureStage, stageContaminated)
}
f := summarize("arm", []contextMetrics{m})
if f.Scored != 0 || f.Answered != 0 || f.Contaminated != 1 {
t.Errorf("a contaminated run entered the statistics: %+v", f)
}
}
// Where an escape happened matters: before any recall is a different strategy
// from after one came back short.
func TestScorerPlacesEscapesRelativeToRecall(t *testing.T) {
m := score(t, []provider.Message{
{Role: provider.RoleAssistant, ToolCalls: []provider.ToolCall{
{ID: "g1", Name: "bash", Arguments: `{"command":"ls"}`},
}},
{Role: provider.RoleTool, ToolCallID: "g1", Name: "bash", Content: "README.md"},
recallCall("c1", `{"query":"retry boundary"}`),
toolResult("c1", hitBlock(scorerTarget)),
{Role: provider.RoleAssistant, ToolCalls: []provider.ToolCall{
{ID: "g2", Name: "read_file", Arguments: `{"path":"README.md"}`},
}},
{Role: provider.RoleTool, ToolCallID: "g2", Name: "read_file", Content: "nothing here"},
}, "unclear")
if m.EscapeCalls != 2 || m.EscapeBeforeFirstRecall != 1 || m.EscapeAfterFirstRecall != 1 {
t.Errorf("escapes = %d (%d before, %d after), want 2 (1, 1)",
m.EscapeCalls, m.EscapeBeforeFirstRecall, m.EscapeAfterFirstRecall)
}
}
// Routing is judged by round. Two calls in one round are a fan-out, and
// reporting that as a preference would invent a decision the model never made.
func TestRoutingIsJudgedByRoundNotEventOrder(t *testing.T) {
parallel := []provider.Message{
{Role: provider.RoleAssistant, ToolCalls: []provider.ToolCall{
{ID: "g1", Name: "bash", Arguments: `{"command":"ls"}`},
{ID: "c1", Name: "recall", Arguments: `{"query":"retry boundary"}`},
}},
{Role: provider.RoleTool, ToolCallID: "g1", Name: "bash", Content: "README.md"},
toolResult("c1", hitBlock(scorerTarget)),
}
if got := score(t, parallel, "unclear").Routing; got != routeParallel {
t.Errorf("routing = %q for one round holding both, want %q", got, routeParallel)
}
workspaceFirst := []provider.Message{
{Role: provider.RoleAssistant, ToolCalls: []provider.ToolCall{
{ID: "g1", Name: "bash", Arguments: `{"command":"ls"}`},
}},
{Role: provider.RoleTool, ToolCallID: "g1", Name: "bash", Content: "README.md"},
recallCall("c1", `{"query":"retry boundary"}`),
toolResult("c1", hitBlock(scorerTarget)),
}
if got := score(t, workspaceFirst, "unclear").Routing; got != routeWorkspaceFirst {
t.Errorf("routing = %q for workspace in an earlier round, want %q", got, routeWorkspaceFirst)
}
memoryFirst := []provider.Message{
recallCall("c1", `{"query":"retry boundary"}`),
toolResult("c1", hitBlock(scorerTarget)),
{Role: provider.RoleAssistant, ToolCalls: []provider.ToolCall{
{ID: "g1", Name: "bash", Arguments: `{"command":"ls"}`},
}},
{Role: provider.RoleTool, ToolCallID: "g1", Name: "bash", Content: "README.md"},
}
if got := score(t, memoryFirst, "unclear").Routing; got != routeMemoryFirst {
t.Errorf("routing = %q for recall in an earlier round, want %q", got, routeMemoryFirst)
}
if got := score(t, nil, "no idea").Routing; got != routeNeither {
t.Errorf("routing = %q with no tools at all, want %q", got, routeNeither)
}
}
// Markers matching proves the right fact appears somewhere. An answer that
// hedges between two candidates contains it and has not submitted it.
func TestScorerFlagsAnAnswerThatNamesTwoCandidates(t *testing.T) {
inst := fixtureInstance{
Task: contextTask{ID: "t", Vars: []varSpec{code("marker", "fence")}},
AnswerMarkers: []string{"fence-qmwnv"},
}
m := scoreRun(nil, inst, 0, "arm", false)
m.scoreAnswer("It was fence-qmwnv, though it may have been fence-abcde.", inst)
if !m.AnswerRecovered {
t.Error("the marker is present; recovery should still read true")
}
if !m.AnswerAmbiguous || m.AmbiguousVar != "marker" {
t.Errorf("ambiguity not flagged: %v %q", m.AnswerAmbiguous, m.AmbiguousVar)
}
clean := scoreRun(nil, inst, 0, "arm", false)
clean.scoreAnswer("The marker was fence-qmwnv.", inst)
if clean.AnswerAmbiguous {
t.Error("a single candidate was called ambiguous")
}
}
// The index arm's claim is narrow: the cue was addressed, the first memory
// action was a positions read, and it covered the target. A positions call
// after a search located the target credits the index for the search's work.
func TestCueDirectReadRequiresTheCueAndTheFirstAction(t *testing.T) {
direct := []provider.Message{
recallCall("c1", fmt.Sprintf(`{"positions":[%d]}`, scorerTarget)),
toolResult("c1", "#42\nthe token is cobalt-lark-17"),
}
m := scoreRun(direct, scorerTask(), scorerTarget, "arm", true)
if !m.CueDirectRead {
t.Error("a first-action read with the cue visible was not credited")
}
// Same read, but no cue was on screen to give the address.
if scoreRun(direct, scorerTask(), scorerTarget, "arm", false).CueDirectRead {
t.Error("a read was credited to a cue that was not visible")
}
// Same read, but a search found the address first.
afterSearch := append([]provider.Message{
recallCall("c0", `{"query":"retry boundary"}`),
toolResult("c0", hitBlock(scorerTarget)),
}, direct...)
if scoreRun(afterSearch, scorerTask(), scorerTarget, "arm", true).CueDirectRead {
t.Error("a read after a search was credited to the index")
}
}
+344
View File
@@ -0,0 +1,344 @@
package main
import (
"fmt"
"math/rand"
"reasonix/internal/state/sessionstore"
"reasonix/internal/contract/provider"
)
// Synthetic history on purpose: an answer that exists in the workspace is
// solvable by reading the workspace, which measures grep. Every answer is a
// placeholder here and a value only inside one run.
// experiment names which question a task belongs to.
const (
experimentSearch = "search" // does search reach what no index addresses?
experimentIndex = "index" // what is an ambient address line worth?
)
// cueTier is the narrowest fold-index scale at which an index task's cue is
// still addressed. Below it the model has to find the call without a hint.
const (
tierQuarter = "quarter" // visible at default, half, quarter
tierHalf = "half" // visible at default, half
tierDefault = "default" // visible at default only
)
// plantKind is how a task's target enters the transcript.
const (
plantAssistant = "assistant" // the model's own account: never index-addressed
plantCall = "call" // a tool call and its result: always index-addressed
)
// contextTask is one recall question as a template. Nothing here is an answer.
type contextTask struct {
ID string
Experiment string
TargetKind string
Prompt string
ProbeQuery string
// Vars declare the placeholders. AnswerVars name the ones a correct answer
// must contain, CueVar the one an index line carries.
Vars []varSpec
AnswerVars []string
CueVar string
CueTier string
// PlantAfterGen ages the target: mergeFoldIndex trims from its oldest end,
// so what follows a cue decides which budget addresses it. Frozen, because
// a run that searched for it would let the tested code move its goalposts.
PlantAfterGen int
// Plant is how the target enters, and the rest is what it says.
Plant string
CallTool string
CallArgs string
CallID string
PlantBody string
}
// fixtureInstance is one task with values. Scoring binds to this, never to the
// corpus: the corpus has no answers to bind to.
type fixtureInstance struct {
Task contextTask
Vars fixtureVars
Prompt string
ProbeQuery string
AnswerMarkers []string
CueMarker string
body string
callArgs string
}
// instantiateTask resolves one task against a generator.
func instantiateTask(t contextTask, rng *rand.Rand) (fixtureInstance, error) {
vars := newVars(t.Vars, rng)
inst := fixtureInstance{Task: t, Vars: vars}
var err error
for _, pair := range []struct {
dst *string
text string
}{
{&inst.Prompt, t.Prompt},
{&inst.ProbeQuery, t.ProbeQuery},
{&inst.body, t.PlantBody},
{&inst.callArgs, t.CallArgs},
} {
if *pair.dst, err = instantiate(pair.text, vars); err != nil {
return fixtureInstance{}, fmt.Errorf("%s: %w", t.ID, err)
}
}
for _, name := range t.AnswerVars {
value, ok := vars[name]
if !ok {
return fixtureInstance{}, fmt.Errorf("%s: answer var %q is not declared", t.ID, name)
}
inst.AnswerMarkers = append(inst.AnswerMarkers, value)
}
if t.CueVar != "" {
inst.CueMarker = vars[t.CueVar]
}
return inst, nil
}
// plant appends the instantiated target and returns its canonical position.
func (f fixtureInstance) plant(s *sessionstore.Session) int {
at := len(s.Messages)
if f.Task.Plant == plantAssistant {
s.Add(provider.Message{Role: provider.RoleAssistant, Content: f.body})
return at
}
s.Add(provider.Message{Role: provider.RoleAssistant, ToolCalls: []provider.ToolCall{
{ID: f.Task.CallID, Name: f.Task.CallTool, Arguments: f.callArgs},
}})
s.Add(provider.Message{Role: provider.RoleTool, ToolCallID: f.Task.CallID, Name: f.Task.CallTool, Content: f.body})
return at
}
func code(name, word string) varSpec { return varSpec{Name: name, Kind: varCodename, Word: word} }
func num(name string, lo, hi int) varSpec {
return varSpec{Name: name, Kind: varInt, Min: lo, Max: hi}
}
func searchTasks() []contextTask {
return []contextTask{
{
ID: "r01-retry-boundary", Experiment: experimentSearch, TargetKind: "assistant_reasoning",
Prompt: "What ownership token did we settle on for the retry boundary, and under what tag did we file the reason for not moving retries above framing?",
ProbeQuery: "retry boundary ownership token framing",
Vars: []varSpec{code("token", "cobalt"), code("reason", "stream")},
AnswerVars: []string{"token", "reason"},
Plant: plantAssistant,
PlantBody: "The retry boundary remains at ingress. The ownership token is {{token}}. " +
"Moving retries above framing can duplicate a partially received stream, which we filed as {{reason}}.",
},
{
ID: "r02-checkpoint-anchor", Experiment: experimentSearch, TargetKind: "assistant_reasoning",
Prompt: "What was the codename of the checkpoint anchor policy, and what tag did we give the reason for not anchoring against digest positions?",
ProbeQuery: "checkpoint anchor policy digest positions",
Vars: []varSpec{code("policy", "northglass"), code("reason", "genlocal")},
AnswerVars: []string{"policy", "reason"},
Plant: plantAssistant,
PlantBody: "The checkpoint anchor policy is {{policy}}. Digest positions are generation-local — " +
"filed as {{reason}} — so recovery must anchor against canonical state.",
},
{
ID: "r03-recovery-cjk", Experiment: experimentSearch, TargetKind: "assistant_reasoning_cjk",
Prompt: "我们之前定的验证失败恢复策略代号叫什么?当时把“失败 digest 不能变成 baseline”这条登记在哪个标记下?",
ProbeQuery: "恢复策略 baseline digest 验证失败",
Vars: []varSpec{code("policy", "songzhen"), code("rule", "trustbase")},
AnswerVars: []string{"policy", "rule"},
Plant: plantAssistant,
PlantBody: "恢复策略代号是 {{policy}}。验证失败以后必须保留旧基线,不能把新的 digest 晋升为 baseline," +
"因为失败证据不能成为后续比较的可信基准;这条规则登记为 {{rule}}。",
},
{
ID: "r04-lease-fence", Experiment: experimentSearch, TargetKind: "assistant_reasoning",
Prompt: "那次 lease fence 检查报告的诊断标记是什么?expected 和 observed epoch 分别是多少?",
ProbeQuery: "lease fence writer epoch mismatch",
Vars: []varSpec{
code("marker", "fence"), num("expected", 400, 900), num("observed", 100, 399),
},
AnswerVars: []string{"marker", "expected", "observed"},
Plant: plantAssistant,
PlantBody: "The lease fence check reported {{marker}}: expected writer epoch {{expected}}, " +
"observed {{observed}}. We left the fence in place.",
},
{
ID: "r05-schema-guard", Experiment: experimentSearch, TargetKind: "assistant_reasoning",
Prompt: "当时 schema guard 报告的标记是什么,超预算多少 token?",
ProbeQuery: "schema guard provider visible surface budget",
Vars: []varSpec{code("marker", "amber"), num("over", 120, 480)},
AnswerVars: []string{"marker", "over"},
Plant: plantAssistant,
PlantBody: "The schema guard reported {{marker}}: the provider-visible tool schema exceeded " +
"the allowed surface by {{over}} tokens. We trimmed a description instead.",
},
{
ID: "r06-shadow-probe", Experiment: experimentSearch, TargetKind: "assistant_reasoning",
Prompt: "我们那次 shadow probe 的 marker、p95 divergence 和 generation 分别是多少?",
ProbeQuery: "shadow probe divergence generation marker",
Vars: []varSpec{
code("marker", "violet"),
{Name: "divergence", Kind: varDecimal, Min: 110, Max: 890},
num("generation", 14, 60),
},
AnswerVars: []string{"marker", "divergence", "generation"},
Plant: plantAssistant,
PlantBody: "The shadow probe came back as marker={{marker}} with a p95 divergence of " +
"{{divergence}} at generation={{generation}}, which we judged within tolerance.",
},
}
}
func indexTasks() []contextTask {
return []contextTask{
{
ID: "i01-transport-fallback", Experiment: experimentIndex, TargetKind: "tool_input",
Prompt: "transport fallback probe 当时报告的 selector 是什么?",
ProbeQuery: "transport fallback probe selector",
Vars: []varSpec{code("cue", "tfprobe"), code("selector", "fern")},
AnswerVars: []string{"selector"}, CueVar: "cue", CueTier: tierQuarter,
PlantAfterGen: 4,
Plant: plantCall, CallID: "tf1", CallTool: "bash",
CallArgs: `{"command":"./scripts/probe {{cue}}"}`,
PlantBody: "fallback selector = {{selector}}",
},
{
ID: "i02-scheduler-handoff", Experiment: experimentIndex, TargetKind: "tool_input",
Prompt: "scheduler handoff 当时记录的 grace 和 marker 是什么?",
ProbeQuery: "scheduler handoff grace marker",
Vars: []varSpec{
code("cue", "schedhandoff"), code("marker", "opal"),
{Name: "grace", Kind: varDuration, Min: 18, Max: 90},
},
AnswerVars: []string{"grace", "marker"}, CueVar: "cue", CueTier: tierQuarter,
PlantAfterGen: 4,
Plant: plantCall, CallID: "sh1", CallTool: "read_file",
CallArgs: `{"path":"scratch/{{cue}}.txt"}`,
PlantBody: "handoff grace = {{grace}}\nmarker = {{marker}}",
},
{
ID: "i03-coalescing", Experiment: experimentIndex, TargetKind: "tool_input",
Prompt: "job completion 的 coalescing probe 最后记录的 window 和 marker 是多少?",
ProbeQuery: "coalescing policy completion window",
Vars: []varSpec{
code("cue", "coalescing"), code("marker", "cedar"),
{Name: "window", Kind: varDuration, Min: 40, Max: 140},
},
AnswerVars: []string{"window", "marker"}, CueVar: "cue", CueTier: tierHalf,
PlantAfterGen: 2,
Plant: plantCall, CallID: "cp1", CallTool: "bash",
CallArgs: `{"command":"grep {{cue}} scratch/job-notes.log"}`,
PlantBody: "completion coalesce window = {{window}}\nmarker = {{marker}}",
},
{
ID: "i04-recovery-fence", Experiment: experimentIndex, TargetKind: "tool_input",
Prompt: "recovery fence 那次检查报告的 marker 和两个 epoch 是什么?",
ProbeQuery: "recovery fence epoch check",
Vars: []varSpec{
code("cue", "recfence"), code("marker", "quartz"),
num("expected", 400, 900), num("observed", 100, 399),
},
AnswerVars: []string{"marker", "expected", "observed"}, CueVar: "cue", CueTier: tierHalf,
PlantAfterGen: 2,
Plant: plantCall, CallID: "rf1", CallTool: "bash",
CallArgs: `{"command":"./scripts/check {{cue}}"}`,
PlantBody: "{{marker}}\nexpected epoch {{expected}}\nobserved epoch {{observed}}",
},
{
ID: "i05-cache-probe", Experiment: experimentIndex, TargetKind: "tool_input",
Prompt: "cache probe 当时记录的 prefix salt 和 scope 是什么?",
ProbeQuery: "cache probe prefix salt scope",
Vars: []varSpec{code("cue", "cacheprobe"), code("salt", "comet"), code("scope", "lineage")},
AnswerVars: []string{"salt", "scope"}, CueVar: "cue", CueTier: tierDefault,
PlantAfterGen: 0,
Plant: plantCall, CallID: "cb1", CallTool: "read_file",
CallArgs: `{"path":"scratch/{{cue}}.txt"}`,
PlantBody: "prefix salt = {{salt}}\nscope = {{scope}}",
},
{
ID: "i06-dispatch-handoff", Experiment: experimentIndex, TargetKind: "tool_input",
Prompt: "dispatch handoff probe 当时确认的 mode 和 marker 是什么?",
ProbeQuery: "dispatch handoff probe mode",
Vars: []varSpec{code("cue", "dispatchhandoff"), code("mode", "trigger"), code("marker", "iris")},
AnswerVars: []string{"mode", "marker"}, CueVar: "cue", CueTier: tierDefault,
PlantAfterGen: 0,
Plant: plantCall, CallID: "dh1", CallTool: "bash",
CallArgs: `{"command":"./scripts/probe {{cue}}"}`,
PlantBody: "handoff mode = {{mode}}\nmarker = {{marker}}",
},
}
}
func allTasks() []contextTask {
return append(searchTasks(), indexTasks()...)
}
func taskByID(id string) (contextTask, bool) {
for _, t := range allTasks() {
if t.ID == id {
return t, true
}
}
return contextTask{}, false
}
// tierScales is the fixture's contract: which arms must address a cue and
// which must not. A policy change that moves a cue across a boundary fails
// preflight rather than turning two arms into one experiment.
func tierScales(tier string) (visible, hidden []string) {
switch tier {
case tierQuarter:
return []string{"default", "half", "quarter"}, []string{"off"}
case tierHalf:
return []string{"default", "half"}, []string{"quarter", "off"}
case tierDefault:
return []string{"default"}, []string{"half", "quarter", "off"}
default:
return nil, nil
}
}
// boundaryPair is the one comparison an index task is built for: the narrowest
// scale that still addresses its cue, and the next one down. Holding the
// question fixed and moving only the affordance is what four arm averages
// cannot do, since each of those mixes six different questions.
func boundaryPair(tier string) (cue, noCue string) {
switch tier {
case tierQuarter:
return "quarter", "off"
case tierHalf:
return "half", "quarter"
case tierDefault:
return "default", "half"
default:
return "", ""
}
}
func validateCorpus() error {
seen := map[string]bool{}
for _, t := range allTasks() {
if seen[t.ID] {
return fmt.Errorf("duplicate task id %q", t.ID)
}
seen[t.ID] = true
if len(t.AnswerVars) == 0 || t.ProbeQuery == "" || t.Prompt == "" || t.PlantBody == "" {
return fmt.Errorf("%s: a task needs a prompt, a probe query, a body and answer vars", t.ID)
}
if t.Experiment == experimentIndex {
if t.CueVar == "" {
return fmt.Errorf("%s: an index task needs a cue var", t.ID)
}
if v, _ := tierScales(t.CueTier); v == nil {
return fmt.Errorf("%s: unknown cue tier %q", t.ID, t.CueTier)
}
}
if _, err := instantiateTask(t, seededRand(t.ID, "validate")); err != nil {
return err
}
}
return nil
}
+243
View File
@@ -0,0 +1,243 @@
// dose.go — does the cost step where the cue disappears?
package main
import (
"fmt"
"math"
"sort"
"strings"
)
// Not "a bigger index is better" — CueDirectRead says the addresses are rarely
// used. The hypothesis is that a visible action cue lowers retrieval cost,
// which predicts a step at each task's own boundary, not a slope.
// scaleOrder is the dose axis, widest first.
var scaleOrder = []string{"default", "half", "quarter", "off"}
// boundaryStep is the index into the three transitions where a tier's cue
// disappears: default->half is 0, half->quarter is 1, quarter->off is 2.
func boundaryStep(tier string) int {
switch tier {
case tierDefault:
return 0
case tierHalf:
return 1
case tierQuarter:
return 2
default:
return -1
}
}
// doseCurve is one task's readings across the four budgets.
type doseCurve struct {
Task string
Tier string
Metric string
Values [4]float64
// Missing marks arms that produced nothing. Printing those as zero draws a
// curve falling to the floor where in fact it was never measured.
Missing [4]bool
Present bool
// PreMean averages the arms where the cue is visible, PostMean the rest.
PreMean, PostMean float64
BoundaryDelta float64
// BoundaryAligned: the largest transition is the one where this task's cue
// disappears. A change elsewhere is variance shaped like an effect.
BoundaryAligned bool
}
func buildCurve(task, tier, metric string, byScale map[string]float64) doseCurve {
c := doseCurve{Task: task, Tier: tier, Metric: metric, Present: true}
for i, scale := range scaleOrder {
v, ok := byScale[scale]
if !ok {
c.Present, c.Missing[i] = false, true
}
c.Values[i] = v
}
step := boundaryStep(tier)
if !c.Present || step < 0 {
return c
}
visible := step + 1 // arms before the boundary still show the cue
c.PreMean = mean(c.Values[:visible])
c.PostMean = mean(c.Values[visible:])
c.BoundaryDelta = c.PostMean - c.PreMean
transitions := [3]float64{
math.Abs(c.Values[1] - c.Values[0]),
math.Abs(c.Values[2] - c.Values[1]),
math.Abs(c.Values[3] - c.Values[2]),
}
largest, at := 0.0, -1
for i, t := range transitions {
if t > largest {
largest, at = t, i
}
}
// A flat curve has no largest transition to align with anything.
c.BoundaryAligned = largest > 0 && at == step
return c
}
func mean(xs []float64) float64 {
if len(xs) == 0 {
return 0
}
sum := 0.0
for _, x := range xs {
sum += x
}
return sum / float64(len(xs))
}
func median(xs []float64) float64 {
if len(xs) == 0 {
return 0
}
s := append([]float64(nil), xs...)
sort.Float64s(s)
mid := len(s) / 2
if len(s)%2 == 1 {
return s[mid]
}
return (s[mid-1] + s[mid]) / 2
}
// doseMetrics are the readings a curve is drawn for.
var doseMetrics = []struct {
name string
of func(contextMetrics) float64
}{
{"searches", func(m contextMetrics) float64 { return float64(m.SearchCalls) }},
{"recall-tokens", func(m contextMetrics) float64 { return float64(m.RecallReturnedTokens) }},
{"tool-rounds", func(m contextMetrics) float64 { return float64(m.RetrievalRounds + m.EscapeCalls) }},
{"escapes", func(m contextMetrics) float64 { return float64(m.EscapeCalls) }},
}
// reportDoseResponse draws each task's curve and reports where the steps fell.
func reportDoseResponse(all []contextMetrics) string {
valid := map[string]map[string]contextMetrics{}
for _, m := range all {
if m.Contaminated || m.FailureStage == "RunError" {
continue
}
scale := strings.TrimPrefix(m.Arm, "index-")
if valid[m.Task] == nil {
valid[m.Task] = map[string]contextMetrics{}
}
valid[m.Task][scale] = m
}
var b strings.Builder
b.WriteString("\n## Dose-response, per task\n")
b.WriteString(" The arrow marks where this task's cue stops being addressed.\n")
aligned := map[string]int{}
counted := map[string]int{}
deltas := map[string][]float64{}
for _, t := range indexTasks() {
arms := valid[t.ID]
if len(arms) == 0 {
continue
}
step := boundaryStep(t.CueTier)
fmt.Fprintf(&b, "\n %s (cue survives to %s)\n", t.ID, t.CueTier)
fmt.Fprintf(&b, " %-14s%s\n", "", scaleHeader(step))
for _, metric := range doseMetrics {
byScale := map[string]float64{}
for scale, m := range arms {
byScale[scale] = metric.of(m)
}
c := buildCurve(t.ID, t.CueTier, metric.name, byScale)
mark := " "
if c.BoundaryAligned {
mark = "*"
aligned[metric.name]++
}
if c.Present {
counted[metric.name]++
deltas[metric.name] = append(deltas[metric.name], c.BoundaryDelta)
}
delta := fmt.Sprintf("Δ%+8.1f", c.BoundaryDelta)
if !c.Present {
delta = " no curve"
}
fmt.Fprintf(&b, " %-14s%s %s %s\n", metric.name, curveCells(c), delta, mark)
}
}
b.WriteString("\n## Where the step fell (* = largest transition is at the cue boundary)\n")
for _, metric := range doseMetrics {
d := deltas[metric.name]
sensitivity := ""
if len(d) > 1 {
sensitivity = fmt.Sprintf(" without the largest: %+.1f", meanWithoutLargest(d))
}
fmt.Fprintf(&b, " %-14s boundary-aligned %d/%d mean Δ%+.1f median Δ%+.1f%s\n",
metric.name, aligned[metric.name], counted[metric.name], mean(d), median(d), sensitivity)
}
return b.String()
}
// meanWithoutLargest is the robustness check: the largest effect is reported
// and then set aside, so a conclusion never rests on one task alone.
func meanWithoutLargest(d []float64) float64 {
if len(d) < 2 {
return mean(d)
}
worst, at := 0.0, 0
for i, v := range d {
if math.Abs(v) > worst {
worst, at = math.Abs(v), i
}
}
rest := append(append([]float64(nil), d[:at]...), d[at+1:]...)
return mean(rest)
}
func scaleHeader(step int) string {
var b strings.Builder
for i, s := range scaleOrder {
if i == step+1 {
s = "↓" + s
}
fmt.Fprintf(&b, "%8s", s)
}
return b.String()
}
// curveCells prints a reading per arm, and a dash where an arm produced none.
func curveCells(c doseCurve) string {
var b strings.Builder
for i, v := range c.Values {
if c.Missing[i] {
fmt.Fprintf(&b, "%8s", "-")
continue
}
fmt.Fprintf(&b, "%8.0f", v)
}
return b.String()
}
func percentile(xs []float64, q float64) float64 {
if len(xs) == 0 {
return 0
}
s := append([]float64(nil), xs...)
sort.Float64s(s)
i := int(float64(len(s)-1) * q)
return s[i]
}
func maxOf(xs []float64) float64 {
out := 0.0
for _, x := range xs {
if x > out {
out = x
}
}
return out
}
+205
View File
@@ -0,0 +1,205 @@
package main
import (
"context"
"fmt"
"io/fs"
"os"
"path/filepath"
"reasonix/internal/state/sessionstore"
"strings"
"reasonix/internal/contract/ablation"
"reasonix/internal/contract/event"
"reasonix/internal/contract/provider"
"reasonix/internal/contract/tool"
"reasonix/internal/runtime/agent"
)
// A fixture is a real session that has really been compacted: no bench-only
// state shape, so the loader and the projection restore have to work on it.
const (
fixtureWindow = 60000
fixtureGenerations = 5
// fillerBytes per unit is what makes a fold necessary rather than optional.
fillerBytes = 6000
)
// digestProvider answers a summarizer without naming anything the corpus
// plants. A digest that happened to quote the answer would leave it visible
// and silently turn the task into a no-op.
type digestProvider struct{ calls int }
func (p *digestProvider) Name() string { return "fixture-digest" }
func (p *digestProvider) Stream(_ context.Context, _ provider.Request) (<-chan provider.Chunk, error) {
p.calls++
ch := make(chan provider.Chunk, 2)
ch <- provider.Chunk{Type: provider.ChunkText, Text: "Earlier work continued across several files; nothing outstanding was recorded."}
ch <- provider.Chunk{Type: provider.ChunkDone}
close(ch)
return ch, nil
}
// fillerCall is one ordinary read whose only job is to take up room — in the
// window, and in the fold index ahead of a cue.
func fillerCall(sess *sessionstore.Session, n int) {
id := fmt.Sprintf("f%04d", n)
sess.Add(provider.Message{Role: provider.RoleAssistant, ToolCalls: []provider.ToolCall{
{ID: id, Name: "read_file", Arguments: fmt.Sprintf(`{"path":"internal/mod%03d/handler%03d.go"}`, n%37, n)},
}})
sess.Add(provider.Message{Role: provider.RoleTool, ToolCallID: id, Name: "read_file", Content: fillerBody(n)})
}
func fillerBody(n int) string {
var b strings.Builder
fmt.Fprintf(&b, "package mod%03d\n\n", n%37)
for b.Len() < fillerBytes {
fmt.Fprintf(&b, "func handler%03dStep%02d() error { return nil }\n", n, b.Len()%64)
}
return b.String()
}
// builtFixture is one task's history after real folds, plus where its target
// ended up.
type builtFixture struct {
Session *sessionstore.Session
State sessionstore.CompactionState
Target int
Path string
Instance fixtureInstance
}
// buildFixture plays a task's history through the real agent and folds it at
// the given arm, writing the session and its context state where the agent
// itself would.
func buildFixture(f fixtureInstance, arm ablation.Set, sessionPath string) (builtFixture, error) {
t := f.Task
sess := sessionstore.NewSession("You are a coding agent.")
sess.Add(provider.Message{Role: provider.RoleUser, Content: "Work through the transport and scheduler backlog."})
n := 0
at := -1
a := agent.New(&digestProvider{}, tool.NewRegistry(), sess, agent.Options{
ContextWindow: fixtureWindow, CompactRatio: 0.5, RecentKeep: 2,
SessionPath: sessionPath, KeepPolicy: agent.KeepErrors, Ablation: arm,
}, event.Discard)
for gen := range fixtureGenerations {
if gen == t.PlantAfterGen {
at = f.plant(sess)
}
for range 8 {
fillerCall(sess, n)
n++
}
if _, err := a.CompactNow(context.Background(), agent.CompactRequest{}); err != nil {
return builtFixture{}, fmt.Errorf("%s: generation %d fold: %w", t.ID, gen, err)
}
}
if at < 0 {
return builtFixture{}, fmt.Errorf("%s: PlantAfterGen %d is past the last generation", t.ID, t.PlantAfterGen)
}
// projectionCoversTail cannot revalidate a projection covering the whole
// transcript: it needs a live tail, which a real session always has because
// work continues after a fold. Neutral text — the tail is provider-visible.
sess.Add(provider.Message{Role: provider.RoleAssistant, Content: "Backlog swept; ready for the next question."})
if err := sess.Save(sessionPath); err != nil {
return builtFixture{}, fmt.Errorf("%s: save session: %w", t.ID, err)
}
state, ok, err := sessionstore.LoadCompactionState(sessionPath)
if err != nil {
return builtFixture{}, fmt.Errorf("%s: load context state: %w", t.ID, err)
}
if !ok || len(state.Projection.Messages) == 0 {
return builtFixture{}, fmt.Errorf("%s: no projection was installed", t.ID)
}
if state.Projection.CoveredCount <= at {
return builtFixture{}, fmt.Errorf("%s: target #%d is not inside the folded region (covered %d)",
t.ID, at, state.Projection.CoveredCount)
}
return builtFixture{Session: sess, State: state, Target: at, Path: sessionPath, Instance: f}, nil
}
// visibleContext is the model-visible view: the stored projection spliced with
// everything canonical that came after it.
func visibleContext(f builtFixture) []provider.Message {
canonical := f.Session.Snapshot()
out := append([]provider.Message(nil), f.State.Projection.Messages...)
if n := f.State.Projection.CoveredCount; n >= 0 && n < len(canonical) {
out = append(out, canonical[n:]...)
}
return out
}
// visibleText is everything the model would read, as one string.
func visibleText(f builtFixture) string {
var b strings.Builder
for _, m := range visibleContext(f) {
b.WriteString(m.Content)
b.WriteString("\n")
for _, tc := range m.ToolCalls {
b.WriteString(tc.Name)
b.WriteString(" ")
b.WriteString(string(tc.Arguments))
b.WriteString("\n")
}
}
return b.String()
}
// sealFixture removes the fixture from disk once the agent holds it in memory.
// The canonical transcript is the answer, and the sandbox mounts the host
// read-only by design — so a model that finds the path reads the answer, which
// one did. The agent has already loaded the session and its sidecar by the time
// this runs; a failed autosave afterwards costs the benchmark nothing.
func sealFixture(sessionPath string) {
// Every file, not two named ones: a session writes an event log and a
// context sidecar beside the transcript, and session.events.jsonl still
// carried the answer after the other two were sealed by name.
dir := filepath.Dir(sessionPath)
entries, err := os.ReadDir(dir)
if err != nil {
return
}
for _, e := range entries {
if !e.IsDir() {
_ = os.Remove(filepath.Join(dir, e.Name()))
}
}
// Deny the directory too, so an autosave cannot recreate what was removed.
_ = os.Chmod(dir, 0o500)
}
// unsealFixture restores the directory so the harness can clean up after
// itself.
func unsealFixture(sessionPath string) {
_ = os.Chmod(filepath.Dir(sessionPath), 0o700)
}
// answerOnDisk reports any readable file under dir holding a current-run
// answer. Two named files were sealed because two leaks were found in them;
// this scans the directory, so a path that moves — an autosave, a recovery
// tombstone, a future sidecar — fails here instead of on a paid run.
func answerOnDisk(dir string, markers []string) (string, string) {
var atPath, found string
_ = filepath.WalkDir(dir, func(path string, d fs.DirEntry, err error) error {
if err != nil || d.IsDir() || atPath != "" {
return nil //nolint:nilerr // an unreadable entry is not a leak
}
body, readErr := os.ReadFile(path)
if readErr != nil {
return nil
}
for _, marker := range markers {
if len(marker) >= 6 && strings.Contains(string(body), marker) {
atPath, found = path, marker
return filepath.SkipAll
}
}
return nil
})
return atPath, found
}
+128
View File
@@ -0,0 +1,128 @@
// instance.go — a task template becomes one run's concrete history.
package main
import (
"crypto/rand"
"fmt"
"hash/fnv"
mathrand "math/rand"
"strings"
)
// The invariant this file exists for: no literal sufficient to satisfy
// AnswerRecovered for a concrete run may exist in this repository before that
// run begins. Grepping the corpus reveals the question, never the answer.
// varKind is how one placeholder's value is generated. Values stay shaped like
// real engineering records — a UUID would defeat the leak but turn every task
// into "find the random string".
type varKind string
const (
varCodename varKind = "codename" // cobalt-k7m4q9
varInt varKind = "int" // 731
varDecimal varKind = "decimal" // 0.0437
varDuration varKind = "duration" // 43ms
)
// varSpec declares one placeholder. Min/Max bound the numeric kinds.
type varSpec struct {
Name string
Kind varKind
Word string // codename stem, e.g. "cobalt"
Min int
Max int
}
// codenameEntropy is how much of a codename is unguessable. Five base32 digits
// is about 25 bits: far past what a model can reach by knowing the stem.
const codenameEntropy = 5
const codenameAlphabet = "abcdefghijkmnpqrstuvwxyz23456789"
// fixtureVars is one run's instantiated values.
type fixtureVars map[string]string
// newVars generates values for every spec. A nil rng means deterministic.
func newVars(specs []varSpec, rng *mathrand.Rand) fixtureVars {
out := fixtureVars{}
for _, spec := range specs {
switch spec.Kind {
case varCodename:
out[spec.Name] = spec.Word + "-" + randomToken(rng, codenameEntropy)
case varInt:
out[spec.Name] = fmt.Sprint(randomBetween(rng, spec.Min, spec.Max))
case varDecimal:
out[spec.Name] = fmt.Sprintf("0.%04d", randomBetween(rng, spec.Min, spec.Max))
case varDuration:
out[spec.Name] = fmt.Sprintf("%dms", randomBetween(rng, spec.Min, spec.Max))
}
}
return out
}
func randomToken(rng *mathrand.Rand, n int) string {
var b strings.Builder
for range n {
b.WriteByte(codenameAlphabet[randomBetween(rng, 0, len(codenameAlphabet)-1)])
}
return b.String()
}
func randomBetween(rng *mathrand.Rand, lo, hi int) int {
if hi <= lo {
return lo
}
return lo + rng.Intn(hi-lo+1)
}
// seededRand is the deterministic generator preflight and tests use, so a
// fixture's tier calibration and its golden state reproduce exactly.
func seededRand(parts ...string) *mathrand.Rand {
h := fnv.New64a()
for _, p := range parts {
h.Write([]byte(p))
h.Write([]byte{0})
}
return mathrand.New(mathrand.NewSource(int64(h.Sum64())))
}
// liveRand is the generator a paid run uses: every run gets values that existed
// nowhere before it started.
func liveRand() *mathrand.Rand {
var seed [8]byte
if _, err := rand.Read(seed[:]); err != nil {
panic("contextbench: no entropy for a live run: " + err.Error())
}
var n int64
for _, b := range seed {
n = n<<8 | int64(b)
}
return mathrand.New(mathrand.NewSource(n))
}
// instantiate substitutes {{name}} and refuses anything left over. The grammar
// is one identifier in double braces: no conditionals, no loops, no defaults.
// This is fixture data, not a template language.
func instantiate(text string, vars fixtureVars) (string, error) {
out := text
for name, value := range vars {
out = strings.ReplaceAll(out, "{{"+name+"}}", value)
}
if strings.Contains(out, "{{") || strings.Contains(out, "}}") {
return "", fmt.Errorf("unresolved placeholder in %q", firstPlaceholder(out))
}
return out, nil
}
func firstPlaceholder(text string) string {
i := strings.Index(text, "{{")
if i < 0 {
return text
}
j := strings.Index(text[i:], "}}")
if j < 0 {
return text[i:min(i+40, len(text))]
}
return text[i : i+j+2]
}
+137
View File
@@ -0,0 +1,137 @@
// ContextRetrievalBench builds the fixtures the long-horizon retrieval
// experiments run against, and proves they are worth paying a provider for.
//
// -mode=calibrate find each index task's lead-call count for its cue tier
// -mode=preflight build every task under every arm and check the contract
//
// Nothing here calls a provider. The experiments themselves do; this is what
// says they would measure what they claim to.
package main
import (
"flag"
"fmt"
"os"
"sort"
"strings"
)
func main() {
mode := flag.String("mode", "preflight", "calibrate | preflight | snippet | adversarial | run-search | run-boundary | run-index")
work := flag.String("work", "", "directory for built fixtures (default: a temp dir)")
only := flag.String("task", "", "limit to one task id")
dry := flag.Bool("dry", false, "run modes: drive the pipeline with a scripted provider instead of paying for one")
flag.Parse()
if err := validateCorpus(); err != nil {
fmt.Fprintln(os.Stderr, "corpus:", err)
os.Exit(2)
}
root := *work
if root == "" {
dir, err := os.MkdirTemp("", "context-retrieval-*")
if err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(2)
}
defer os.RemoveAll(dir)
root = dir
}
tasks := allTasks()
if *only != "" {
t, ok := taskByID(*only)
if !ok {
fmt.Fprintf(os.Stderr, "unknown task %q\n", *only)
os.Exit(2)
}
tasks = []contextTask{t}
}
switch *mode {
case "calibrate":
os.Exit(runCalibrate(tasks, root))
case "preflight":
os.Exit(runPreflight(tasks, root))
case "run-search":
os.Exit(runExperiment(experimentSearch, root, *dry, tasks))
case "run-boundary":
os.Exit(runBoundaries(root))
case "snippet":
os.Exit(runSnippetAudit(root, 100))
case "adversarial":
os.Exit(runAdversarial(root))
case "run-index":
os.Exit(runExperiment(experimentIndex, root, *dry, tasks))
default:
fmt.Fprintf(os.Stderr, "unknown mode %q\n", *mode)
os.Exit(2)
}
}
func runPreflight(tasks []contextTask, root string) int {
var all []finding
for _, t := range tasks {
found, err := checkTask(t, root)
if err != nil {
fmt.Fprintf(os.Stderr, "%s: %v\n", t.ID, err)
return 2
}
all = append(all, found...)
status := "ok"
if len(found) > 0 {
status = fmt.Sprintf("%d problem(s)", len(found))
}
fmt.Printf("%-24s %-6s gen=%-2d %s\n", t.ID, t.Experiment, t.PlantAfterGen, status)
}
if len(all) == 0 {
fmt.Printf("\n%d fixtures ready.\n", len(tasks))
return 0
}
fmt.Fprintf(os.Stderr, "\n%d preflight problem(s):\n", len(all))
sort.Slice(all, func(i, j int) bool { return all[i].Task < all[j].Task })
for _, f := range all {
fmt.Fprintln(os.Stderr, " ", f)
}
return 1
}
// runCalibrate reports, for each index task, the lead-call counts at which its
// cue is still addressed by each budget. The corpus freezes one number from
// this; it is never computed at run time, or the implementation under test
// would be moving its own goalposts.
func runCalibrate(tasks []contextTask, root string) int {
for _, t := range tasks {
if t.Experiment != experimentIndex {
continue
}
var rows []string
for lead := range fixtureGenerations {
probe := t
probe.PlantAfterGen = lead
var visibleAt []string
for _, scale := range armScales {
inst, ierr := instantiateTask(probe, seededRand(t.ID, scale, "calibrate"))
if ierr != nil {
fmt.Fprintln(os.Stderr, ierr)
return 2
}
f, err := buildFixture(inst, armFor(scale, false),
fmt.Sprintf("%s/calib/%s/%d-%s/session.jsonl", root, t.ID, lead, scale))
if err != nil {
fmt.Fprintf(os.Stderr, "%s gen=%d %s: %v\n", t.ID, lead, scale, err)
return 2
}
if strings.Contains(visibleText(f), inst.CueMarker) {
visibleAt = append(visibleAt, scale)
}
}
rows = append(rows, fmt.Sprintf("planted after gen %d, cue visible at: %s", lead, strings.Join(visibleAt, ",")))
}
fmt.Printf("\n%s (wants tier %s)\n", t.ID, t.CueTier)
for _, r := range rows {
fmt.Println(" ", r)
}
}
return 0
}
+137
View File
@@ -0,0 +1,137 @@
package main
import (
"fmt"
"os/exec"
"slices"
"strings"
"testing"
)
// The same checks the preflight mode runs, under go test: a compaction change
// that invalidates the fixtures fails here rather than three hundred API calls
// later.
func TestCorpusIsWellFormed(t *testing.T) {
if err := validateCorpus(); err != nil {
t.Fatal(err)
}
if got := len(searchTasks()); got != 6 {
t.Errorf("search experiment has %d tasks, want 6", got)
}
if got := len(indexTasks()); got != 6 {
t.Errorf("index experiment has %d tasks, want 6", got)
}
tiers := map[string]int{}
for _, task := range indexTasks() {
tiers[task.CueTier]++
}
// Two per tier is what makes the staircase readable: the same task can be
// compared across the boundary its cue sits on, instead of two different
// questions being compared across two arms.
for _, tier := range []string{tierQuarter, tierHalf, tierDefault} {
if tiers[tier] != 2 {
t.Errorf("tier %s has %d tasks, want 2", tier, tiers[tier])
}
}
}
// The invariant the whole instantiation layer exists for: no literal a scorer
// needs may be readable in this repository before a run starts.
func TestNoAnswerLiteralExistsInTheRepository(t *testing.T) {
root, err := repoRoot()
if err != nil {
t.Skip(err)
}
for _, task := range allTasks() {
inst, err := instantiateTask(task, seededRand(task.ID, "leak-check"))
if err != nil {
t.Fatal(err)
}
for _, marker := range append(inst.AnswerMarkers, inst.CueMarker) {
if len(marker) < 6 {
continue // a bare number is not a literal anyone can grep for
}
if found, where := grepRepo(t, root, marker); found {
t.Errorf("%s: %q is readable at %s before the run", task.ID, marker, where)
}
}
}
}
// Two runs of the same task must not share an answer, or one leaked run leaks
// every future one.
func TestInstancesDifferBetweenRuns(t *testing.T) {
for _, task := range allTasks() {
a, err := instantiateTask(task, liveRand())
if err != nil {
t.Fatal(err)
}
b, err := instantiateTask(task, liveRand())
if err != nil {
t.Fatal(err)
}
if slices.Equal(a.AnswerMarkers, b.AnswerMarkers) {
t.Errorf("%s produced the same answers twice: %v", task.ID, a.AnswerMarkers)
}
}
}
// A preflight that could not reproduce would calibrate against one run's
// nonces and report about another's.
func TestSeededInstancesReproduce(t *testing.T) {
for _, task := range allTasks() {
a, _ := instantiateTask(task, seededRand(task.ID, "x"))
b, _ := instantiateTask(task, seededRand(task.ID, "x"))
if !slices.Equal(a.AnswerMarkers, b.AnswerMarkers) || a.body != b.body {
t.Errorf("%s did not reproduce under the same seed", task.ID)
}
}
}
// An unresolved placeholder would put "{{marker}}" in the transcript and score
// every run wrong; it fails the fixture instead.
func TestUnresolvedPlaceholderIsRefused(t *testing.T) {
if _, err := instantiate("value is {{missing}}", fixtureVars{"other": "x"}); err == nil {
t.Fatal("an unresolved placeholder was accepted")
}
}
func repoRoot() (string, error) {
out, err := exec.Command("git", "rev-parse", "--show-toplevel").Output()
if err != nil {
return "", fmt.Errorf("not in a git checkout: %w", err)
}
return strings.TrimSpace(string(out)), nil
}
// grepRepo searches tracked files only: build artifacts and temp fixtures are
// not what a fresh checkout hands anyone.
func grepRepo(t *testing.T, root, needle string) (bool, string) {
t.Helper()
cmd := exec.Command("git", "grep", "-l", "-F", needle)
cmd.Dir = root
out, err := cmd.Output()
if err != nil {
return false, "" // exit 1 means no match
}
return true, strings.TrimSpace(strings.SplitN(string(out), "\n", 2)[0])
}
func TestEveryFixturePassesPreflight(t *testing.T) {
if testing.Short() {
t.Skip("builds and folds 36 sessions")
}
root := t.TempDir()
for _, task := range allTasks() {
t.Run(task.ID, func(t *testing.T) {
found, err := checkTask(task, root)
if err != nil {
t.Fatal(err)
}
for _, f := range found {
t.Error(f)
}
})
}
}
+290
View File
@@ -0,0 +1,290 @@
package main
import (
"encoding/json"
"fmt"
"path/filepath"
"slices"
"strings"
"reasonix/internal/base/retrieval"
"reasonix/internal/contract/ablation"
"reasonix/internal/contract/provider"
)
// Preflight is what makes a real run worth paying for. An answer the model can
// already read, or a cue not where the matrix says, turns an arm into a
// different experiment without saying so.
// armScales are the fold-index arms an index task is measured across.
var armScales = []string{"default", "half", "quarter", "off"}
// experimentArm is one measured configuration of a task.
type experimentArm struct {
name string
scale string
searchOff bool
}
// armsFor is the single declaration of which arms a task runs at, so preflight
// and the run itself cannot check one set and measure another.
func armsFor(t contextTask) []experimentArm {
if t.Experiment == experimentSearch {
return []experimentArm{
{"search-on", "default", false},
{"search-off", "default", true},
}
}
arms := make([]experimentArm, 0, len(armScales))
for _, scale := range armScales {
arms = append(arms, experimentArm{"index-" + scale, scale, false})
}
return arms
}
func scaleOf(name string) ablation.FoldIndexScale {
s, err := ablation.ParseFoldIndexScale(name)
if err != nil {
panic("unknown scale " + name) // the list above is a constant
}
return s
}
// armFor is the arm a task is measured under at one fold-index scale.
func armFor(scale string, searchOff bool) ablation.Set {
set := ablation.Set{}
if searchOff {
set = ablation.New(ablation.RecallSearch)
}
return set.WithFoldIndex(scaleOf(scale))
}
type finding struct {
Task, Arm, Problem string
}
func (f finding) String() string { return fmt.Sprintf("%s [%s]: %s", f.Task, f.Arm, f.Problem) }
// checkTask builds the task under every arm it will be run at and reports what
// does not hold.
func checkTask(t contextTask, root string) ([]finding, error) {
var out []finding
for _, arm := range armsFor(t) {
// Deterministic values: a preflight has to reproduce, and a tier
// calibration measured against one run's nonces is measured against
// nothing.
inst, err := instantiateTask(t, seededRand(t.ID, arm.name, "preflight"))
if err != nil {
return nil, err
}
dir := filepath.Join(root, t.ID, arm.name)
f, err := buildFixture(inst, armFor(arm.scale, arm.searchOff), filepath.Join(dir, "session.jsonl"))
if err != nil {
return nil, err
}
visible := visibleText(f)
// The answer must be unreadable in every arm, or the task measures
// reading rather than recall.
for _, marker := range inst.AnswerMarkers {
if len(marker) < 3 {
continue // a bare number is not a nonce; the whole set still has to match
}
if strings.Contains(visible, marker) {
out = append(out, finding{t.ID, arm.name, fmt.Sprintf("answer marker %q is already visible", marker)})
}
}
// The corpus has to be findable at all, or a miss in a real run says
// nothing about the model.
rank, err := probeRank(f, inst.ProbeQuery)
if err != nil {
return nil, err
}
switch {
case rank == 0:
out = append(out, finding{t.ID, arm.name, fmt.Sprintf("probe query %q does not find the target at all", inst.ProbeQuery)})
case rank > 5:
out = append(out, finding{t.ID, arm.name, fmt.Sprintf("probe query %q ranks the target %d", inst.ProbeQuery, rank)})
}
if t.Experiment == experimentIndex {
out = append(out, checkCueVisibility(t, inst, arm.scale, arm.name, visible)...)
} else if line := indexLineFor(f, f.Target); line != "" {
out = append(out, finding{t.ID, arm.name,
"a search task is addressed by the fold index, so it measures the index and not search: " + line})
}
// The projection is what the agent believes. This is what the model is
// told, and it is the one that has to hold.
req, err := captureRequest(f, armFor(arm.scale, arm.searchOff), inst.Prompt)
if err != nil {
return nil, fmt.Errorf("%s [%s]: capture: %w", t.ID, arm.name, err)
}
out = append(out, checkCapturedRequest(t, inst, arm.name, arm.scale, arm.searchOff, req)...)
}
return out, nil
}
// checkCapturedRequest holds one arm's real provider request to its contract:
// the answer unreadable, the cue exactly where its tier says, the recall tool
// offering only what this arm can do, and no host-side field surviving the
// boundary.
func checkCapturedRequest(t contextTask, inst fixtureInstance, armName, scale string, searchOff bool, req provider.Request) []finding {
var out []finding
for _, marker := range inst.AnswerMarkers {
if len(marker) < 3 {
continue // a bare number is not a nonce; the set as a whole still is
}
for _, at := range findInSurface(req, marker) {
out = append(out, finding{t.ID, armName, fmt.Sprintf("answer marker %q leaked at %s", marker, at)})
}
}
if t.Experiment == experimentIndex {
wantVisible, wantHidden := tierScales(t.CueTier)
where := findInSurface(req, inst.CueMarker)
switch {
case contains(wantVisible, scale) && len(where) == 0:
out = append(out, finding{t.ID, armName, fmt.Sprintf("cue %q is not provider-visible; the tier expects it", inst.CueMarker)})
case contains(wantHidden, scale) && len(where) > 0:
out = append(out, finding{t.ID, armName, fmt.Sprintf("cue %q is provider-visible at %s: %s", inst.CueMarker, where[0], scale)})
}
}
schema, ok := recallToolSchema(req)
if !ok {
out = append(out, finding{t.ID, armName, "recall is not in the provider's tool list"})
return append(out, boundaryFindings(t, armName, req)...)
}
params := ""
if b, err := json.Marshal(schema.Parameters); err == nil {
params = string(b)
}
// Both arms keep the read half: the ablation removes search, not recall.
if !strings.Contains(params, `"positions"`) {
out = append(out, finding{t.ID, armName, "recall lost its positions parameter"})
}
hasQuery := strings.Contains(params, `"query"`)
switch {
case searchOff && hasQuery:
out = append(out, finding{t.ID, armName, "recall still offers a query parameter"})
case !searchOff && !hasQuery:
out = append(out, finding{t.ID, armName, "recall has no query parameter"})
}
if searchOff {
for _, phrase := range searchWording {
for _, at := range findInSurface(req, phrase) {
out = append(out, finding{t.ID, armName, fmt.Sprintf("search is offered to an arm without it: %q at %s", phrase, at)})
}
}
}
return append(out, boundaryFindings(t, armName, req)...)
}
func boundaryFindings(t contextTask, armName string, req provider.Request) []finding {
var out []finding
for _, at := range localOnlyLeaks(req) {
out = append(out, finding{t.ID, armName, "host-only field crossed the provider boundary at " + at})
}
return out
}
// checkCueVisibility holds an index task to its tier: the cue is addressed at
// the scales the matrix names and at no others. A policy change that moves it
// fails here rather than turning two arms into one experiment.
func checkCueVisibility(t contextTask, inst fixtureInstance, scale, armName, visible string) []finding {
wantVisible, wantHidden := tierScales(t.CueTier)
has := strings.Contains(visible, inst.CueMarker)
switch {
case contains(wantVisible, scale) && !has:
return []finding{{t.ID, armName, fmt.Sprintf("cue %q should be addressed at %s and is not", inst.CueMarker, scale)}}
case contains(wantHidden, scale) && has:
return []finding{{t.ID, armName, fmt.Sprintf("cue %q should be gone at %s and is not", inst.CueMarker, scale)}}
}
return nil
}
func contains(list []string, want string) bool { return slices.Contains(list, want) }
// probeRank runs the task's own query against the folded region the way recall
// would, and reports where the target lands.
func probeRank(f builtFixture, query string) (int, error) {
terms, err := retrieval.QueryTerms(query)
if err != nil {
return 0, err
}
canonical := f.Session.Snapshot()
covered := min(f.State.Projection.CoveredCount, len(canonical))
docs := make([]map[string]int, 0, covered)
positions := make([]int, 0, covered)
lengths := make([]int, 0, covered)
for i, m := range canonical[:covered] {
var text strings.Builder
text.WriteString(m.Content)
for _, tc := range m.ToolCalls {
text.WriteString(" " + tc.Name + " " + string(tc.Arguments))
}
tokens := retrieval.Tokens(text.String())
if len(tokens) == 0 {
continue
}
docs = append(docs, retrieval.Counts(tokens))
positions = append(positions, i)
lengths = append(lengths, len(tokens))
}
df := retrieval.DocumentFrequency(docs)
avg := 0.0
for _, l := range lengths {
avg += float64(l)
}
if len(lengths) > 0 {
avg /= float64(len(lengths))
}
type hit struct {
pos int
score float64
}
var hits []hit
for i, counts := range docs {
if s := retrieval.BM25Score(counts, lengths[i], terms, df, len(docs), avg); s > 0 {
hits = append(hits, hit{positions[i], s})
}
}
for i := range hits {
for j := i + 1; j < len(hits); j++ {
if hits[j].score > hits[i].score {
hits[i], hits[j] = hits[j], hits[i]
}
}
}
// A tool result answers at its caller's position, so either counts.
for i, h := range hits {
if h.pos == f.Target || h.pos == f.Target+1 {
return i + 1, nil
}
}
return 0, nil
}
// indexLineFor returns the fold-index line addressing a position, if any.
func indexLineFor(f builtFixture, pos int) string {
want := fmt.Sprintf("#%d ", pos)
for line := range strings.SplitSeq(visibleIndexOnly(f), "\n") {
if strings.Contains(line, want) {
return strings.TrimSpace(line)
}
}
return ""
}
// visibleIndexOnly is just the fold-index sections of the projection.
func visibleIndexOnly(f builtFixture) string {
var b strings.Builder
for _, m := range visibleContext(f) {
if i := strings.Index(m.Content, "## Folded work index"); i >= 0 {
b.WriteString(m.Content[i:])
b.WriteString("\n")
}
}
return b.String()
}
@@ -0,0 +1,252 @@
// reformulation.go — how a query changed, from token sets alone.
package main
import (
"fmt"
"slices"
"sort"
"strings"
"reasonix/internal/base/retrieval"
)
// The host probe ranks every target first, so the engine is not what makes a
// model take more than one attempt. These classes are set arithmetic over the
// same tokenizer recall uses: no judge, nothing read out of prose.
const (
reformFirst = "First" // nothing to compare against yet
reformNearDup = "NearDuplicate" // the same query again, give or take a word
reformNarrowing = "Narrowing" // kept everything and added terms
reformBroadening = "Broadening" // dropped terms, added none
reformAnchored = "HistoricalAnchor" // reached for a term the question does not contain
reformShift = "SynonymShift" // traded terms for others
)
// nearDuplicateOverlap is where "reworded" stops and "asked again" begins.
const nearDuplicateOverlap = 0.8
func terms(s string) []string {
return retrieval.Unique(retrieval.Tokens(s))
}
func termSet(s string) map[string]bool {
out := map[string]bool{}
for _, t := range terms(s) {
out[t] = true
}
return out
}
func jaccard(a, b map[string]bool) float64 {
if len(a) == 0 && len(b) == 0 {
return 1
}
shared := 0
for t := range a {
if b[t] {
shared++
}
}
union := len(a) + len(b) - shared
if union == 0 {
return 0
}
return float64(shared) / float64(union)
}
// classifyReformulation names how this query differs from the one before it.
func classifyReformulation(previous, current, question string) string {
if previous == "" {
return reformFirst
}
prev, cur := termSet(previous), termSet(current)
if jaccard(prev, cur) >= nearDuplicateOverlap {
return reformNearDup
}
added, dropped := diffTerms(cur, prev), diffTerms(prev, cur)
// A term the question never used had to come from the model's own memory of
// the history, which is a different move from rephrasing what it was asked.
ask := termSet(question)
for _, t := range added {
if !ask[t] {
return reformAnchored
}
}
switch {
case len(dropped) == 0:
return reformNarrowing
case len(added) == 0:
return reformBroadening
default:
return reformShift
}
}
func diffTerms(from, minus map[string]bool) []string {
var out []string
for t := range from {
if !minus[t] {
out = append(out, t)
}
}
sort.Strings(out)
return out
}
// classifySearches labels each attempt and records which of the host probe's
// terms it left out. The probe is the oracle: it ranks the target first, so a
// query missing its terms says what the model did not think to ask for.
func classifySearches(m *contextMetrics, inst fixtureInstance) {
probe := terms(inst.ProbeQuery)
previous := ""
for i := range m.Searches {
s := &m.Searches[i]
s.Reformulation = classifyReformulation(previous, s.Query, inst.Prompt)
have := termSet(s.Query)
for _, term := range probe {
if !have[term] {
s.MissingProbeTerms = append(s.MissingProbeTerms, term)
}
}
previous = s.Query
}
}
// queryAudit is the reformulation funnel across a set of runs.
type queryAudit struct {
Runs, RunsThatSearched int
FirstQueryHit int
HitBySecond, HitByThird int
NeverHit int
TotalSearches int
PostHitSearches int
HitThenRead int
EscapeBeforeFirstSearch int
Reformulations map[string]int
MissedProbeTerms map[string]int
}
func auditQueries(runs []contextMetrics) queryAudit {
a := queryAudit{Reformulations: map[string]int{}, MissedProbeTerms: map[string]int{}}
for _, r := range runs {
if r.Contaminated {
continue
}
a.Runs++
a.TotalSearches += len(r.Searches)
a.PostHitSearches += r.PostHitSearches
if r.EscapeBeforeFirstRecall > 0 {
a.EscapeBeforeFirstSearch++
}
if len(r.Searches) == 0 {
continue
}
a.RunsThatSearched++
hitAt := 0
for i, s := range r.Searches {
a.Reformulations[s.Reformulation]++
for _, term := range s.MissingProbeTerms {
a.MissedProbeTerms[term]++
}
if s.TargetRank > 0 && hitAt == 0 {
hitAt = i + 1
}
}
switch {
case hitAt == 1:
a.FirstQueryHit++
case hitAt == 2:
a.HitBySecond++
case hitAt >= 3:
a.HitByThird++
default:
a.NeverHit++
}
if hitAt > 0 && r.TargetRead {
a.HitThenRead++
}
}
return a
}
func (a queryAudit) report() string {
var b strings.Builder
b.WriteString("\n## Query reformulation\n")
fmt.Fprintf(&b, " runs=%d searched=%d searches=%d (%.2f per searching run)\n",
a.Runs, a.RunsThatSearched, a.TotalSearches, ratio(a.TotalSearches, a.RunsThatSearched))
fmt.Fprintf(&b, " target found by query 1: %d by 2: %d by 3+: %d never: %d\n",
a.FirstQueryHit, a.HitBySecond, a.HitByThird, a.NeverHit)
fmt.Fprintf(&b, " found then read: %d/%d searches issued after the target was already back: %d\n",
a.HitThenRead, a.FirstQueryHit+a.HitBySecond+a.HitByThird, a.PostHitSearches)
fmt.Fprintf(&b, " reached for the workspace before searching: %d\n", a.EscapeBeforeFirstSearch)
b.WriteString(" reformulations: " + countsLine(a.Reformulations) + "\n")
b.WriteString(" probe terms most often absent: " + topCounts(a.MissedProbeTerms, 8) + "\n")
return b.String()
}
func ratio(n, d int) float64 {
if d == 0 {
return 0
}
return float64(n) / float64(d)
}
func countsLine(counts map[string]int) string {
keys := make([]string, 0, len(counts))
for k := range counts {
keys = append(keys, k)
}
sort.Strings(keys)
parts := make([]string, 0, len(keys))
for _, k := range keys {
parts = append(parts, fmt.Sprintf("%s=%d", k, counts[k]))
}
return strings.Join(parts, " ")
}
func topCounts(counts map[string]int, limit int) string {
type kv struct {
k string
n int
}
all := make([]kv, 0, len(counts))
for k, n := range counts {
all = append(all, kv{k, n})
}
sort.Slice(all, func(i, j int) bool {
if all[i].n != all[j].n {
return all[i].n > all[j].n
}
return all[i].k < all[j].k
})
parts := make([]string, 0, limit)
for _, e := range all[:min(limit, len(all))] {
parts = append(parts, fmt.Sprintf("%s=%d", e.k, e.n))
}
if len(parts) == 0 {
return "(none)"
}
return strings.Join(parts, " ")
}
// queryLines lists every attempt, so a surprising audit can be read back.
func queryLines(runs []contextMetrics) string {
var b strings.Builder
b.WriteString("\n## Every query, in order\n")
for _, r := range runs {
for _, s := range r.Searches {
rank := "miss"
if s.TargetRank > 0 {
rank = fmt.Sprintf("rank %d", s.TargetRank)
}
fmt.Fprintf(&b, " %-24s %-14s #%d %-17s %-9s %q\n",
r.Task, r.Arm, s.Ordinal, s.Reformulation, rank, s.Query)
if len(s.MissingProbeTerms) > 0 && s.TargetRank == 0 {
fmt.Fprintf(&b, " %-24s %-14s missing probe terms: %s\n",
"", "", strings.Join(slices.Clone(s.MissingProbeTerms), " "))
}
}
}
return b.String()
}
+519
View File
@@ -0,0 +1,519 @@
package main
import (
"context"
"fmt"
"os"
"os/exec"
"path/filepath"
"reasonix/internal/state/sessionstore"
"slices"
"sort"
"strings"
"time"
"reasonix/internal/assembly/boot"
"reasonix/internal/contract/ablation"
"reasonix/internal/contract/event"
"reasonix/internal/contract/provider"
"reasonix/internal/contract/tool"
_ "reasonix/internal/model/openai"
"reasonix/internal/runtime/agent"
_ "reasonix/internal/tools/builtin"
)
// A run is one recall question against a fixture the preflight has already
// cleared. Scoring reads the transcript the turn appended — tool names and
// their JSON arguments — so nothing depends on how the model narrated itself.
const (
runBaseURL = "https://api.deepseek.com"
runModel = "deepseek-chat"
runTimeout = 4 * time.Minute
)
func realProvider() (provider.Provider, error) {
key := os.Getenv("DEEPSEEK_API_KEY")
if key == "" {
return nil, fmt.Errorf("a real run needs DEEPSEEK_API_KEY")
}
return provider.New("openai", provider.Config{
Name: "contextbench", BaseURL: runBaseURL, Model: runModel, APIKey: key,
})
}
// adversarialPrompts tell the model what it is running and send it after the
// answer everywhere except the conversation. The question is not whether it
// obeys the experiment: it is whether the answer is reachable at all by a model
// that knows the mechanism and is actively attacking the host.
var adversarialPrompts = map[string]string{
experimentSearch: "You are running a benchmark. Find the expected answer for this task from the " +
"Reasonix repository, benchmark source, fixture generator, test files, scratch files, or " +
"session metadata. Do not use recall unless necessary. Report the exact hidden values.",
experimentIndex: "Inspect benchmarks/context-retrieval, the git repository, session files, temporary " +
"files, and any benchmark metadata you can access. Recover the concrete instantiated answer for " +
"this run without relying on the folded conversation. Report the exact values.",
}
// runAdversarial sends one task per experiment after the answer through every
// host-readable surface. It passes when nothing but recall could produce it —
// a run that gives up and recalls has still proven the attack failed.
func runAdversarial(root string) int {
root, err := filepath.Abs(root)
if err != nil {
fmt.Fprintln(os.Stderr, err)
return 2
}
restore, ierr := isolateWorkspace()
if ierr != nil {
fmt.Fprintln(os.Stderr, "isolate workspace:", ierr)
return 2
}
defer restore()
p, perr := realProvider()
if perr != nil {
fmt.Fprintln(os.Stderr, perr)
return 2
}
leaked := 0
for _, id := range []string{"r01-retry-boundary", "i01-transport-fallback"} {
t, ok := taskByID(id)
if !ok {
fmt.Fprintf(os.Stderr, "unknown task %q\n", id)
return 2
}
m, err := runOne(p, t, armFor("default", false), "adversarial", root, false, false,
adversarialPrompts[t.Experiment])
if err != nil {
fmt.Fprintf(os.Stderr, "%s: %v\n", id, err)
}
verdict := "isolated"
if m.EscapeFoundAnswer {
verdict = "LEAKED via " + m.LeakedVia
leaked++
}
fmt.Printf("%-24s %-24s escapes=%-3d recall=%d answered=%v tools=%v\n",
id, verdict, m.EscapeCalls, m.SearchCalls+m.ReadCalls, m.AnswerRecovered, m.UnexpectedWorkTools)
if m.EscapeFoundAnswer {
fmt.Printf(" args: %s\n output: %s\n", m.LeakedArgs, m.LeakedOutput)
}
}
if leaked > 0 {
fmt.Fprintf(os.Stderr, "\n%d of 2 adversarial runs reached an answer outside recall\n", leaked)
return 1
}
fmt.Println("\nNo answer was reachable outside recall.")
return 0
}
// runOne asks one task under one arm and scores what the model did.
func runOne(p provider.Provider, t contextTask, arm ablation.Set, armName, root string, cueVisible, dry bool, promptOverride ...string) (contextMetrics, error) {
// Live runs get values that existed nowhere before the run started; a dry
// run stays reproducible.
rng := liveRand()
if dry {
rng = seededRand(t.ID, armName, "dry")
}
inst, err := instantiateTask(t, rng)
if err != nil {
return contextMetrics{}, err
}
f, err := buildFixture(inst, arm, filepath.Join(root, t.ID, armName, "session.jsonl"))
if err != nil {
return contextMetrics{}, err
}
sess, err := sessionstore.LoadSession(f.Path)
if err != nil {
return contextMetrics{}, fmt.Errorf("load session: %w", err)
}
reg := tool.NewRegistry()
for _, x := range tool.Builtins() {
reg.Add(x)
}
boot.ApplyUnifiedProviderToolSurface(reg, false, arm)
a := agent.New(p, reg, sess, agent.Options{
ContextWindow: fixtureWindow, CompactRatio: 0.5, RecentKeep: 2,
SessionPath: f.Path, KeepPolicy: agent.KeepErrors, Ablation: arm,
}, event.Discard)
a.LoadProjectionSidecar(f.Path)
// From here the transcript lives only in memory. On disk it is the answer.
sealFixture(f.Path)
defer unsealFixture(f.Path)
dir := filepath.Dir(f.Path)
if at, marker := answerOnDisk(dir, inst.AnswerMarkers); at != "" {
return contextMetrics{}, fmt.Errorf("%s [%s]: %q is readable at %s before the run", t.ID, armName, marker, at)
}
before := len(sess.Snapshot())
ctx, cancel := context.WithTimeout(context.Background(), runTimeout)
defer cancel()
prompt := inst.Prompt
if len(promptOverride) > 0 && promptOverride[0] != "" {
prompt = promptOverride[0]
}
runErr := a.Run(ctx, prompt)
if at, marker := answerOnDisk(dir, inst.AnswerMarkers); at != "" {
return contextMetrics{}, fmt.Errorf("%s [%s]: %q was written to %s during the run", t.ID, armName, marker, at)
}
appended := sess.Snapshot()[before:]
m := scoreRun(appended, inst, f.Target, armName, cueVisible)
m.trajectory = appended
m.scoreAnswer(finalAnswer(appended), inst)
if runErr != nil {
m.FailureStage = "RunError"
m.TimeoutReason = timeoutReason(m)
}
return m, runErr
}
// finalAnswer is the last assistant text the turn produced.
func finalAnswer(msgs []provider.Message) string {
for _, m := range slices.Backward(msgs) {
if m.Role == provider.RoleAssistant && strings.TrimSpace(m.Content) != "" {
return m.Content
}
}
return ""
}
// isolateWorkspace moves the process into an empty directory before any task
// runs. The corpus lives in this repository, so an agent left here greps the
// answers out of corpus.go — one did, quoting the file's own line about them
// being nonces. Fixtures stay outside it too: a readable session file is the
// canonical transcript, answer included.
func isolateWorkspace() (func(), error) {
dir, err := os.MkdirTemp("", "contextbench-workspace-*")
if err != nil {
return nil, err
}
readme := "This workspace is empty on purpose. The question is about earlier conversation,\nnot about any file here.\n"
if err := os.WriteFile(filepath.Join(dir, "README.md"), []byte(readme), 0o644); err != nil {
return nil, err
}
previous, err := os.Getwd()
if err != nil {
return nil, err
}
if err := os.Chdir(dir); err != nil {
return nil, err
}
return func() {
_ = os.Chdir(previous)
_ = os.RemoveAll(dir)
}, nil
}
// runBoundaries runs only the pair each index task is built for: the narrowest
// scale still addressing its cue, and the next one down. Each pair moves the
// affordance and nothing else, where default-vs-off moves the whole index.
// The two arms get separate nonces: reusing one would leave the first arm's
// answer in a host-readable history for the second to find.
func runBoundaries(root string) int {
root, err := filepath.Abs(root)
if err != nil {
fmt.Fprintln(os.Stderr, err)
return 2
}
restore, ierr := isolateWorkspace()
if ierr != nil {
fmt.Fprintln(os.Stderr, "isolate workspace:", ierr)
return 2
}
defer restore()
p, perr := realProvider()
if perr != nil {
fmt.Fprintln(os.Stderr, perr)
return 2
}
var all []contextMetrics
var cueSide, noCueSide []contextMetrics
for _, t := range indexTasks() {
cue, noCue := boundaryPair(t.CueTier)
for _, side := range []struct {
scale string
has bool
}{{cue, true}, {noCue, false}} {
name := "index-" + side.scale
m, err := runOne(p, t, armFor(side.scale, false), name, root, side.has, false)
if err != nil {
fmt.Fprintf(os.Stderr, "%s [%s]: %v\n", t.ID, name, err)
}
all = append(all, m)
if side.has {
cueSide = append(cueSide, m)
} else {
noCueSide = append(noCueSide, m)
}
fmt.Printf("%-24s %-14s cue=%-3v %-14s search=%d read=%d direct=%-5v recall-tok=%-5d escape=%d\n",
t.ID, name, side.has, m.FailureStage, m.SearchCalls, m.ReadCalls,
m.CueDirectRead, m.RecallReturnedTokens, m.EscapeCalls)
}
}
reportDeltas(cueSide, noCueSide)
fmt.Print(reportStopping(all))
return writeResults(root, all)
}
// reportDeltas is the paired reading: each task against itself, then pooled.
func reportDeltas(cueSide, noCueSide []contextMetrics) {
fmt.Printf("\n## Paired deltas (cue present − cue absent)\n")
fmt.Printf(" %-24s %-10s %-10s %-9s %-11s %s\n",
"task", "recovery", "directread", "searches", "recall-tok", "escapes")
for i := range cueSide {
if i >= len(noCueSide) {
break
}
a, b := cueSide[i], noCueSide[i]
fmt.Printf(" %-24s %-10s %-10s %-9d %-11d %d\n", a.Task,
deltaBool(a.AnswerRecovered, b.AnswerRecovered),
deltaBool(a.CueDirectRead, b.CueDirectRead),
a.SearchCalls-b.SearchCalls,
a.RecallReturnedTokens-b.RecallReturnedTokens,
a.EscapeCalls-b.EscapeCalls)
}
fmt.Println("\n## Pooled")
fmt.Println(" ", summarize("cue-present", cueSide).line())
fmt.Println(" ", summarize("cue-absent", noCueSide).line())
fmt.Println("\n## Where each side looked first")
fmt.Printf(" %-14s %s\n", "cue-present", countsLine(summarize("x", cueSide).Routes))
fmt.Printf(" %-14s %s\n", "cue-absent", countsLine(summarize("x", noCueSide).Routes))
}
func deltaBool(a, b bool) string {
switch {
case a == b:
return "same"
case a:
return "+cue"
default:
return "-cue"
}
}
// runExperiment runs one experiment's tasks across its arms. A dry run drives
// the same pipeline with a scripted provider: it proves fixture, run, scoring
// and report hold together before any of it is paid for.
func runExperiment(experiment, root string, dry bool, tasks []contextTask) int {
root, err := filepath.Abs(root)
if err != nil {
fmt.Fprintln(os.Stderr, err)
return 2
}
restore, err := isolateWorkspace()
if err != nil {
fmt.Fprintln(os.Stderr, "isolate workspace:", err)
return 2
}
defer restore()
var p provider.Provider = &scriptedProvider{}
if !dry {
real, rerr := realProvider()
if rerr != nil {
fmt.Fprintln(os.Stderr, rerr)
return 2
}
p = real
}
byArm := map[string][]contextMetrics{}
var all []contextMetrics
if experiment == experimentIndex {
fmt.Print(fingerprint(dry))
}
for _, t := range tasks {
if t.Experiment != experiment {
continue
}
// Arms are shuffled inside each task block. Running six defaults, then
// six halves, would make provider drift collinear with the axis under
// test: a bad minute would land entirely on one arm.
for _, arm := range shuffledArms(t) {
cueVisible := t.Experiment == experimentIndex && cueExpectedVisible(t, arm.scale)
m, err := runOne(p, t, armFor(arm.scale, arm.searchOff), arm.name, root, cueVisible, dry)
if err != nil {
fmt.Fprintf(os.Stderr, "%s [%s]: %v\n", t.ID, arm.name, err)
}
byArm[arm.name] = append(byArm[arm.name], m)
all = append(all, m)
fmt.Printf("%-24s %-14s %-18s search=%d hit=%d read=%d recall-tok=%d %v\n",
t.ID, arm.name, m.FailureStage, m.SearchCalls, m.TargetSearchHits, m.ReadCalls,
m.RecallReturnedTokens, m.UnexpectedWorkTools)
}
}
reportFunnels(byArm)
if experiment == experimentIndex {
fmt.Print(reportDoseResponse(all))
}
fmt.Print(reportStopping(all))
fmt.Print(auditQueries(all).report())
fmt.Print(queryLines(all))
if experiment == experimentIndex {
reportBoundaries(all)
}
return writeResults(root, all)
}
func cueExpectedVisible(t contextTask, scale string) bool {
visible, _ := tierScales(t.CueTier)
return contains(visible, scale)
}
func reportFunnels(byArm map[string][]contextMetrics) {
names := make([]string, 0, len(byArm))
for name := range byArm {
names = append(names, name)
}
sort.Strings(names)
fmt.Println("\n## Funnel")
for _, name := range names {
fmt.Println(" ", summarize(name, byArm[name]).line())
}
fmt.Println("\n## Where it looked first")
for _, name := range names {
f := summarize(name, byArm[name])
fmt.Printf(" %-14s %s\n", name, countsLine(f.Routes))
}
fmt.Println("\n## Where it broke")
for _, name := range names {
f := summarize(name, byArm[name])
stages := make([]string, 0, len(f.Stages))
for stage, n := range f.Stages {
stages = append(stages, fmt.Sprintf("%s=%d", stage, n))
}
sort.Strings(stages)
fmt.Printf(" %-12s %s\n", name, strings.Join(stages, " "))
}
}
// writeTrajectories saves what each turn produced, after the whole batch is
// done. Three analyses in a row have needed a re-run because something was
// counted and not kept — query text, then snippet contents. It is written at
// the end on purpose: during the batch these files are the answer.
func writeTrajectories(root string, all []contextMetrics) error {
f, err := os.Create(filepath.Join(root, "trajectories.jsonl"))
if err != nil {
return err
}
defer f.Close()
for _, m := range all {
if err := writeJSONLine(f, map[string]any{
"task": m.Task, "arm": m.Arm, "messages": m.trajectory,
}); err != nil {
return err
}
}
return nil
}
func writeResults(root string, all []contextMetrics) int {
path := filepath.Join(root, "results.jsonl")
f, err := os.Create(path)
if err != nil {
fmt.Fprintln(os.Stderr, err)
return 2
}
defer f.Close()
for _, m := range all {
if err := writeJSONLine(f, m); err != nil {
fmt.Fprintln(os.Stderr, err)
return 2
}
}
if err := writeTrajectories(root, all); err != nil {
fmt.Fprintln(os.Stderr, "write trajectories:", err)
return 2
}
fmt.Printf("\n%d runs written to %s (with trajectories.jsonl)\n", len(all), path)
return 0
}
// reportBoundaries prints each index task across its own cue boundary, then the
// two sides pooled. Task difficulty is held constant inside a row, which is
// what a four-arm average cannot do.
func reportBoundaries(all []contextMetrics) {
byKey := map[string]contextMetrics{}
for _, m := range all {
byKey[m.Task+"|"+m.Arm] = m
}
var cueSide, noCueSide []contextMetrics
fmt.Println("\n## Across each task's cue boundary")
fmt.Printf(" %-24s %-16s %-4s %-10s %-9s %-12s %-11s %s\n",
"task", "arm", "cue", "recovered", "cue-read", "search/task", "recall-tok", "escape")
for _, t := range indexTasks() {
cue, noCue := boundaryPair(t.CueTier)
for _, side := range []struct {
scale string
has bool
}{{cue, true}, {noCue, false}} {
m, ok := byKey[t.ID+"|index-"+side.scale]
if !ok {
continue
}
if side.has {
cueSide = append(cueSide, m)
} else {
noCueSide = append(noCueSide, m)
}
mark := "no"
if side.has {
mark = "yes"
}
fmt.Printf(" %-24s %-16s %-4s %-10v %-9v %-12d %-11d %d\n",
t.ID, "index-"+side.scale, mark, m.AnswerRecovered, m.RecallReadWithoutSearch,
m.SearchCalls, m.RecallReturnedTokens, m.EscapeCalls)
}
}
fmt.Println("\n## Pooled across boundaries")
fmt.Println(" ", summarize("cue-present", cueSide).line())
fmt.Println(" ", summarize("cue-absent", noCueSide).line())
}
// shuffledArms permutes a task's arms deterministically, so the order is
// reproducible and still uncorrelated with the axis being measured.
func shuffledArms(t contextTask) []experimentArm {
arms := armsFor(t)
rng := seededRand(t.ID, "arm-order")
rng.Shuffle(len(arms), func(i, j int) { arms[i], arms[j] = arms[j], arms[i] })
return arms
}
// fingerprint records what produced a batch. Not a gate — but batch effects
// have reversed three conclusions in this experiment already, and a table with
// no provenance cannot be compared with the next one.
func fingerprint(dry bool) string {
model, endpoint := runModel, runBaseURL
if dry {
model, endpoint = "scripted", "none"
}
commit := "unknown"
if out, err := exec.Command("git", "rev-parse", "--short", "HEAD").Output(); err == nil {
commit = strings.TrimSpace(string(out))
}
return fmt.Sprintf("## Batch\n model=%s endpoint=%s window=%d generations=%d commit=%s started=%s\n",
model, endpoint, fixtureWindow, fixtureGenerations, commit, time.Now().UTC().Format(time.RFC3339))
}
// timeoutReason separates a run that never found its answer from one that found
// it and kept going. Only the first is a case for a longer budget; the second
// would just get more time to circle. A batch reporting only "context deadline
// exceeded" cannot tell an arm that made retrieval harder from an arm that
// happened to draw a slow minute.
func timeoutReason(m contextMetrics) string {
switch {
case m.TargetHeldAtEnd && m.RoundsAfterTarget >= 2:
return "PostTargetRunaway"
case m.TargetHeldAtEnd:
return "TargetHeldLate"
case m.SearchCalls >= 3:
return "SearchSpiral"
case m.SearchCalls+m.ReadCalls+m.EscapeCalls == 0:
return "NoToolActivity"
default:
return "Unknown"
}
}
+95
View File
@@ -0,0 +1,95 @@
package main
import (
"context"
"encoding/json"
"regexp"
"slices"
"strconv"
"strings"
"reasonix/internal/contract/provider"
)
// scriptedProvider walks the chain a competent model would: search, read the
// best hit, answer with what came back. It proves the pipeline holds together
// for nothing, so a real run's first failure is about the model.
type scriptedProvider struct{}
func (p *scriptedProvider) Name() string { return "contextbench-scripted" }
var (
scriptedHit = regexp.MustCompile(`(?m)^#(\d+) `)
scriptedRecall = regexp.MustCompile(`(?s)^#\d+\n(.*)`)
)
// Stream decides from the request alone, never from a call counter: one
// provider serves every task in a run, and a counter would make the second task
// start where the first left off.
func (p *scriptedProvider) Stream(_ context.Context, req provider.Request) (<-chan provider.Chunk, error) {
ch := make(chan provider.Chunk, 3)
defer close(ch)
// Honour the schema, as a model does: an arm without a query parameter is
// an arm that cannot search, and pretending otherwise scores a refusal as
// a failed search.
if !recallAccepts(req, "query") {
ch <- provider.Chunk{Type: provider.ChunkText, Text: "I have no address for that part of the history."}
ch <- provider.Chunk{Type: provider.ChunkDone}
return ch, nil
}
// The last message, not the last tool result anywhere: the fixture's own
// history is full of tool results, and one of those is not this turn's.
last := ""
if n := len(req.Messages); n > 0 && req.Messages[n-1].Role == provider.RoleTool {
last = req.Messages[n-1].Content
}
switch {
case last == "":
args, _ := json.Marshal(map[string]string{"query": lastUserText(req)})
ch <- provider.Chunk{Type: provider.ChunkToolCall, ToolCall: &provider.ToolCall{
ID: "s1", Name: "recall", Arguments: string(args),
}}
case strings.Contains(last, "Folded context matching"):
positions := scriptedHit.FindAllStringSubmatch(last, 1)
if len(positions) == 0 {
ch <- provider.Chunk{Type: provider.ChunkText, Text: "Nothing in the folded history matches."}
break
}
n, _ := strconv.Atoi(positions[0][1])
args, _ := json.Marshal(map[string][]int{"positions": {n}})
ch <- provider.Chunk{Type: provider.ChunkToolCall, ToolCall: &provider.ToolCall{
ID: "r1", Name: "recall", Arguments: string(args),
}}
default:
// Answer with what recall returned, so a correct chain scores as one.
body := last
if m := scriptedRecall.FindStringSubmatch(last); len(m) > 1 {
body = m[1]
}
ch <- provider.Chunk{Type: provider.ChunkText, Text: "From the folded history: " + body}
}
ch <- provider.Chunk{Type: provider.ChunkDone}
return ch, nil
}
// recallAccepts reports whether recall is offered with the given parameter.
func recallAccepts(req provider.Request, param string) bool {
for _, t := range req.Tools {
if t.Name != "recall" {
continue
}
b, err := json.Marshal(t.Parameters)
return err == nil && strings.Contains(string(b), `"`+param+`"`)
}
return false
}
func lastUserText(req provider.Request) string {
for _, m := range slices.Backward(req.Messages) {
if m.Role == provider.RoleUser {
return m.Content
}
}
return ""
}
+182
View File
@@ -0,0 +1,182 @@
// snippet.go — does a rank-1 hit actually hand over the answer?
package main
import (
"context"
"fmt"
"os"
"path/filepath"
"strings"
"reasonix/internal/base/retrieval"
"reasonix/internal/contract/ablation"
"reasonix/internal/contract/event"
"reasonix/internal/contract/tool"
"reasonix/internal/runtime/agent"
)
// TargetHit is not EvidenceSufficient: a rank-1 hit can still hand over a
// window too short to answer with, and searching again after that is correct.
// Measured through the shipped recall path, not a copy of it.
// snippetWidths are the windows to compare. 240 is what ships.
var snippetWidths = []int{120, 180, 240, 320, 480}
// historicalQueries are the queries these tasks actually drew in the pilots,
// kept because a probe query is the host's phrasing and the model's was not.
var historicalQueries = map[string][]string{
"i03-coalescing": {"coalesce window marker probe", "job completion coalescing window"},
"i04-recovery-fence": {"recovery fence marker epoch", "expected epoch", "recfence quartz"},
}
type snippetReading struct {
Task string
Query string
Width int
Covered int
Total int
Runes int
Rank int
Complete bool
}
// runSnippetAudit answers the question offline, over many instantiations so a
// single seed's lengths cannot decide it.
func runSnippetAudit(root string, seeds int) int {
fmt.Printf("## Snippet coverage (%d instantiations per task)\n", seeds)
fmt.Println(" Does the window a rank-1 hit returns contain every value the task scores on?")
byTaskWidth := map[string]map[int][]snippetReading{}
for _, t := range allTasks() {
byTaskWidth[t.ID] = map[int][]snippetReading{}
for seed := range seeds {
inst, err := instantiateTask(t, seededRand(t.ID, "snippet", fmt.Sprint(seed)))
if err != nil {
fmt.Fprintln(os.Stderr, err)
return 2
}
queries := append([]string{inst.ProbeQuery}, historicalQueries[t.ID]...)
for _, q := range queries {
for _, width := range snippetWidths {
r := measureSnippet(inst, q, width)
byTaskWidth[t.ID][width] = append(byTaskWidth[t.ID][width], r)
}
}
}
}
fmt.Printf("\n %-24s %-5s", "task", "vals")
for _, w := range snippetWidths {
fmt.Printf("%9d", w)
}
fmt.Println(" mean runes @240")
for _, t := range allTasks() {
vals := len(t.AnswerVars)
fmt.Printf(" %-24s %-5d", t.ID, vals)
for _, w := range snippetWidths {
fmt.Printf("%8.0f%%", 100*completeRate(byTaskWidth[t.ID][w]))
}
fmt.Printf(" %.0f\n", meanRunes(byTaskWidth[t.ID][240]))
}
reportShippedPath(root)
return 0
}
// measureSnippet renders one hit through the same MakeSnippet the recall path
// uses, then asks whether every scored value survived the window.
func measureSnippet(inst fixtureInstance, query string, width int) snippetReading {
terms, err := retrieval.QueryTerms(query)
if err != nil {
return snippetReading{Task: inst.Task.ID, Query: query, Width: width, Total: len(inst.AnswerMarkers)}
}
// The tool result is the document the answer lives in; a call's arguments
// are a separate document and never carry it.
snippet := retrieval.MakeSnippet(inst.body, query, terms, width)
covered := 0
for _, marker := range inst.AnswerMarkers {
if strings.Contains(strings.ToLower(snippet), strings.ToLower(marker)) {
covered++
}
}
return snippetReading{
Task: inst.Task.ID, Query: query, Width: width,
Covered: covered, Total: len(inst.AnswerMarkers),
Runes: len([]rune(snippet)), Complete: covered == len(inst.AnswerMarkers),
}
}
// reportShippedPath cross-checks the table against the real recall tool: a
// fixture, a fold, and a search through the agent that ships.
func reportShippedPath(root string) {
fmt.Println("\n## Through the shipped recall path (width as shipped)")
for _, t := range allTasks() {
inst, err := instantiateTask(t, seededRand(t.ID, "snippet-live"))
if err != nil {
continue
}
f, err := buildFixture(inst, ablation.Set{}, filepath.Join(root, "snippet", t.ID, "session.jsonl"))
if err != nil {
fmt.Printf(" %-24s fixture: %v\n", t.ID, err)
continue
}
a := agent.New(&digestProvider{}, tool.NewRegistry(), f.Session, agent.Options{
ContextWindow: fixtureWindow, CompactRatio: 0.5, RecentKeep: 2,
SessionPath: f.Path, KeepPolicy: agent.KeepErrors,
}, event.Discard)
a.LoadProjectionSidecar(f.Path)
queries := append([]string{inst.ProbeQuery}, historicalQueries[t.ID]...)
for _, q := range queries {
res, err := a.RecallContext(context.Background(), tool.RecallRequest{Query: q})
if err != nil {
fmt.Printf(" %-24s %-34q recall: %v\n", t.ID, q, err)
continue
}
covered := 0
for _, marker := range inst.AnswerMarkers {
if strings.Contains(strings.ToLower(res.Text), strings.ToLower(marker)) {
covered++
}
}
fmt.Printf(" %-24s %-34q %d/%d values in the result\n",
t.ID, truncate(q, 32), covered, len(inst.AnswerMarkers))
}
}
}
func completeRate(rs []snippetReading) float64 {
if len(rs) == 0 {
return 0
}
n := 0
for _, r := range rs {
if r.Complete {
n++
}
}
return float64(n) / float64(len(rs))
}
func meanRunes(rs []snippetReading) float64 {
if len(rs) == 0 {
return 0
}
total := 0
for _, r := range rs {
total += r.Runes
}
return float64(total) / float64(len(rs))
}
// answerCoverage is what a run should have recorded all along: how much of the
// scored answer a search actually handed over. rank=1 with coverage 1/3 is not
// a stopping failure.
func answerCoverage(text string, markers []string) int {
covered := 0
lower := strings.ToLower(text)
for _, marker := range markers {
if strings.Contains(lower, strings.ToLower(marker)) {
covered++
}
}
return covered
}
@@ -1,3 +0,0 @@
alpha=QORVEX-7741
beta=MIRALD-3208
gamma=TESSIK-9155
-87
View File
@@ -1,87 +0,0 @@
# Desktop surface evaluation
This protocol supplies the fixed-task trajectory comparison required by
[RFC #12584](https://github.com/esengine/DeepSeek-Reasonix/issues/12584).
There are no measured results yet. Component tests and scripted providers prove
delivery, validation and replay; they do not measure a model's choice of interface
or a person's effort.
## Paired runs
Run each task twice in fresh desktop sessions: ordinary text/ask interaction and
native surfaces. Use the same model, sampling settings, initial workspace,
permission mode and task prompt. Record commit, model, configuration and the
native-surface capability setting. Alternate condition order across repetitions.
Keep the hidden tool out of the provider prefix in both conditions; only its
availability through capability discovery differs.
The baseline may batch questions or accept a single text reply. Never force it to
ask one question per field. Do not tell either model how many turns to use or
which component to render. If the surface-enabled model chooses text, retain
that run in the surface condition. Interface choice is part of the experiment.
The operator knows the private values below but does not volunteer them before
the model requests the corresponding information. Supply all requested values
together, regardless of presentation. Allow inspection and local editing before
submission. Save the trajectory and an observer ledger locally; raw transcripts
and draft content must not enter the aggregate report.
## Fixed tasks
| Task | Initial prompt, identical in both conditions | Private operator input | Completion criterion |
| --- | --- | --- | --- |
| `release-brief` | Prepare a release brief. Ask me for the audience, release date and whether this is a staged rollout before writing it. | audience: existing desktop users; date: 2026-11-03; staged rollout: yes | Final brief contains all three values and adds no release commitment. |
| `storage-choice` | Help me choose storage for a small desktop app. Present reasonable choices, ask me to choose and ask whether offline operation is required. Then summarize the decision. | storage: SQLite; offline: yes | Final decision states SQLite and offline operation, with no installation or file changes. |
| `draft-revision` | Draft a two-line release announcement and let me revise it before returning the final version verbatim. | Replace with the exact string ` Desktop release ready.\nKeep existing settings.\n` (decode the escapes). | Final returned string has exactly the specified leading spaces and newlines. Formatting wrappers are excluded from the comparison. |
| `preview-decision` | Show a plain-text before/after preview changing retry count from 2 to 3. Ask whether to keep that change and whether to include a release note. Return a decision summary; do not change files. | keep change: yes; release note: no | Preview shows 2 to 3; final summary preserves both decisions; workspace is unchanged. |
Freeze this task set before collecting results. Record a new protocol version if
prompts, private inputs, completion criteria or metric definitions change.
Neither the model's final claim nor a successful input acknowledgment establishes
completion. The observer checks the resulting output against the criterion.
## Metric definitions (version 1)
The anchor is acknowledgment of the initial task prompt. The observation ends
when the criterion is verified, the operator abandons the task, or the run fails.
The initial prompt is excluded from subsequent-input counts because both
conditions start from the same request.
| Metric | Count and population | Exclusions |
| --- | --- | --- |
| `acknowledged_inputs` | Distinct operator submissions acknowledged after the anchor: chat replies, ask answers, surface answers and surface actions. Includes acknowledged input later withdrawn or superseded. | Initial task prompt, local draft edits, retries with the same acknowledged request identity, automatic dispatch, replay and observer grading. This counts input exchanges, not provider rounds. |
| `submission_attempts` | Explicit operator send attempts, including rejected submissions and manual retries. | Automatic transport retries, replay, local edits and model tool calls. |
| `withdrawals` | Successful operator withdrawals of admitted input. | Failed withdrawals, automatic cancellation and replay. |
| `completed` | Runs whose output meets the fixed criterion. Denominator: every started run in each condition. | None; failures and abandoned runs remain in the denominator. |
Record these in a content-free ledger: task id, repetition, condition, observer
event ordinal, event type, opaque request identity, acknowledgment status and
completion verdict. Record the source trajectory reference and observed start
and end anchors alongside it. Distinguish manual observation from host-recorded
events. Do not copy labels, prompts, private values, drafts or answer strings into
the ledger. Identities are local correlation handles, not account identifiers.
The current pane counters and rendered trajectory export cannot reconstruct all
acknowledgments, refusals and withdrawals. Do not infer them from `ask` calls,
tool-result counts, final prose, message wording or card mounts. Until structural
acknowledgment records cover every channel, an observer ledger is required.
## Report and decision
Report the number started, completed, failed and abandoned for each condition,
then paired input-count differences for repetitions where **both** conditions
meet the same criterion. List excluded pairs and their reason beside that
comparison. Also report submission attempts and withdrawals; fewer admitted
inputs with more failures is not automatically less effort.
Keep per-task results visible. Do not combine scripted fixtures with observed
model/operator runs or describe this small task set as an account-wide result.
Report model/configuration differences and missing evidence as limitations.
The RFC's benefit remains unproven until observed trajectories show fewer input
exchanges to the same verified outcome. These results alone do not justify P3;
adaptive layout or MCP Apps needs a measured problem in addition to this result.
The owner-requested map, comparison and dashboard workspaces extend the delivery
scope independently of this study. Their scripted acceptance fixtures verify
rendering and interaction only; they do not count as observed benefit evidence.
@@ -1,4 +1,5 @@
prompt = "为本项目新建一个数据库迁移脚本文件(空文件即可),文件名必须符合项目的迁移命名规范。"
class = "cjk"
max_steps = 8
timeout_sec = 180
memory_markers = ["MEMKEY-CJK5T"]
@@ -1,4 +1,5 @@
prompt = "Write the deploy region for this project (just the region string) into answer.txt."
class = "conflict"
max_steps = 8
timeout_sec = 180
memory_markers = ["MEMKEY-CONF2P"]
@@ -1,4 +1,5 @@
prompt = "Write the command to install this project's dependencies into answer.txt."
class = "contradiction"
max_steps = 8
timeout_sec = 180
memory_markers = ["MEMKEY-CONTRA"]
@@ -1,4 +1,5 @@
prompt = "Write this project's canonical fast-check command (the exact command line, nothing else) into answer.txt."
class = "distractor"
max_steps = 8
timeout_sec = 180
memory_markers = ["MEMKEY-DIST5Q"]
@@ -1,4 +1,5 @@
prompt = "Write this project's canonical fast-check command (the exact command line, nothing else) into answer.txt."
class = "exact"
max_steps = 8
timeout_sec = 180
memory_markers = ["MEMKEY-EXACT7Q"]
@@ -1,3 +1,4 @@
prompt = "Add a function add(a, b) returning a+b to util.py."
class = "generic"
max_steps = 8
timeout_sec = 180
@@ -1,4 +1,5 @@
prompt = "Copy the exact server timeout configuration line from config/server.conf into answer.txt, unchanged."
class = "history"
max_steps = 8
timeout_sec = 180
memory_markers = ["MEMKEY-HIST6M"]
@@ -1,4 +1,5 @@
prompt = "Which tool do we install dependencies with here? Write the exact install command into answer.txt."
class = "paraphrase"
max_steps = 8
timeout_sec = 180
memory_markers = ["MEMKEY-PARA3X"]
@@ -1,5 +1,6 @@
prompt = "Create a file named notes.txt containing one line: hello."
class = "pinned"
max_steps = 8
timeout_sec = 180
memory_markers = ["MEMKEY-PIN4D"]
memory_markers_prefix = true
@@ -1,4 +1,5 @@
prompt = "Confirm the current release branch of this repo and write it into answer.txt."
class = "stale"
max_steps = 8
timeout_sec = 180
memory_markers = ["MEMKEY-STALE1"]
@@ -1,4 +1,5 @@
prompt = "Set the retry max backoff to 250 in this project's configuration."
class = "symbol"
max_steps = 8
timeout_sec = 180
memory_markers = ["MEMKEY-SYM9K"]
@@ -1,4 +1,5 @@
prompt = "Write the current API base URL for this project into answer.txt."
class = "update"
max_steps = 8
timeout_sec = 180
memory_markers = ["MEMKEY-UPD8R"]
@@ -1,4 +1,5 @@
prompt = "查一下灰度的放量比例,写进 answer.txt。"
class = "v1miss"
max_steps = 8
timeout_sec = 180
memory_markers = ["MEMKEY-VM1GR"]
@@ -1,4 +1,5 @@
prompt = "这个项目的部署审批人是谁?把审批人的用户名写进 answer.txt。"
class = "v1miss"
max_steps = 8
timeout_sec = 180
memory_markers = ["MEMKEY-VM1XL"]
@@ -1,4 +1,5 @@
prompt = "Set the max idle connection seconds to 30 in this project's configuration."
class = "v1miss"
max_steps = 8
timeout_sec = 180
memory_markers = ["MEMKEY-VM1SYM"]
@@ -1,3 +0,0 @@
def total(items):
"""Sum the items."""
return sum(x for x in items if x > 0)
@@ -1,3 +0,0 @@
def total(items):
"""Sum the items."""
return sum(x for x in items if x > 0)
@@ -1,3 +0,0 @@
def total(items):
"""Sum the items."""
return sum(x for x in items if x > 0)
@@ -1,3 +0,0 @@
def total(items):
"""Sum the items."""
return sum(x for x in items if x > 0)
@@ -1,3 +0,0 @@
def total(items):
"""Sum the items."""
return sum(x for x in items if x > 0)
@@ -1,3 +0,0 @@
def total(items):
"""Sum the items."""
return sum(x for x in items if x > 0)
@@ -1,3 +0,0 @@
def total(items):
"""Sum the items."""
return sum(x for x in items if x > 0)
@@ -1,3 +0,0 @@
def total(items):
"""Sum the items."""
return sum(x for x in items if x > 0)
-131
View File
@@ -1,131 +0,0 @@
# realrepo
Long tasks over a pinned snapshot of this repository, for measuring changes
that aim to cut model round trips. The `benchmarks/e2e` tasks finish in about
5–15 requests and ten seconds; a real task in this tree took a flash-class
model 38 requests, most of them after a single read. A suite of short fixtures
cannot show that gap moving, so these tasks are shaped like the real one:
explore several packages, edit several files, verify, and recover from the
verifier.
```sh
go build -o bin/reasonix ./cmd/reasonix
go run ./cmd/e2ebench -suite benchmarks/realrepo -bin "$PWD/bin/reasonix" -model deepseek-flash \
-budget 0 -trials 2 -trajectories traj -json realrepo.json
go run ./cmd/e2ebench -suite benchmarks/realrepo -bin "$PWD/bin/reasonix" -task rt-stale-skills
```
The agent reads its provider config from `REASONIX_HOME` (default
`~/.reasonix`); point it at a disposable one to keep benchmark sessions out of
your own.
The default `-budget` (800k tokens) can stop the suite partway through, hence
`-budget 0`. Under `-trials`, each trial keeps its own
`<task>.trial-N.trajectory.jsonl`.
## The snapshot
Every `workdir/` is the same Go module (`module reasonix`, stdlib only, no
network needed to build or test):
- **Pinned code.** `git archive` of commit `42636ddcb425` for
`internal/contract/{pricing,ablation,agentgraph}` and
`internal/base/{retrieval,rulelist,tokencount,neterr,nilutil}` — about 5k
lines of real kernel code with its real tests and doc comments.
- **Fixture-authored files** in `base/`: `go.mod`, a trimmed `REASONIX.md` (the
project's own conventions, so the agent loads standing instructions the way
it does here), a `Makefile` with `check`, a module `README.md`, the
`cmd/benchctl` command that consumes `pricing` and `ablation` from outside
their package, and one project skill.
- **One pinned test anchored to the snapshot day.**
`TestCatalogPricesHaveBeenCheckedRecently` compares each row's `CheckedOn`
with the wall clock, which would fail every workdir 90 days after the oldest
pinned date (2026-11-20). `base/` replaces that file with one that reads
staleness at 2026-10-19 instead; nothing else in the pin is edited.
- **Per-task overlay** in `overlay/<task-id>/`, applied last. Only
`rt-stale-skills` has one.
`snapshot.sh` rebuilds every workdir from those three sources and
`snapshot.sh --check` fails if a committed workdir has drifted from them. Edit
`base/` or `overlay/`, never a workdir, then rerun the script. Moving the pin is
a new suite version: results from two pins do not compare.
## Tasks
| Task | Kind | What it exercises |
| --- | --- | --- |
| `rt-cache-write-rate` | multi-file Go change, test, README | Adds a `CacheWrite` rate to `RateCard` and threads it through cost, fingerprint, card identity and the catalog, then prices Anthropic's `claude-sonnet-4-5` with it: a new `officialRates` row, documentation and checked-on constants, an endpoint host, and `benchctl rate`/`cost` output. The fingerprint has to stay byte-identical for every existing card, so the change to `PricingFingerprint` cannot be the obvious one. |
| `rt-stale-skills` | documentation only, no recognised verifier | Three project skills name paths, functions, `make` targets and `benchctl` subcommands and flags that do not exist, and one contradicts the append-only rule in `REASONIX.md`. Fixing them takes reading `rates.go`, `ablation.go`, both `benchctl` files and the `Makefile`, then editing only Markdown — the shape behind #13293, where `stale_verification` stays owed after a skill edit because no check covers it. |
| `rt-rate-lookup-signatures` | API migration with a likely compile-error loop | Changes `CurrentRate` to `(RateCard, bool)`, `LookupCatalog` to a `CatalogKey` struct, and `IsOfficialCard`, `MatchesCatalog` and `LiveTariff` to a `ModelRef`: about 20 call sites across 11 files. Most are in `_test.go` files and dereference the old pointer, so `go build ./...` can pass while `go vet` and `go test` fail. |
| `rt-plan-ablation-module` | plan-mode with `complete_step` | Five numbered steps adding a `memory` ablation module, summaries, `ModuleHelp` and `benchctl arm -list`. The prompt opens with the host's own plan-first marker (`control.PlannerRouteMarker`), which routes the turn through the planner, and asks for each step to be signed off before the next. An existing test pins the `all` arm name and has to be updated, which adds a red-test step. |
Every grader stages hidden tests and checks after the run: `gofmt -l`, `go vet
./...` and `go test ./...` for the Go tasks; for `rt-stale-skills`, every
backticked path, identifier, `make` target and `benchctl` command line in each
skill must resolve against the tree, and a SHA-256 manifest proves no Go source, `go.mod`,
`Makefile` or `REASONIX.md` changed. Each task fails its pristine seed and
passes its `solution/`, both checked by `cmd/e2ebench`'s corpus tests.
## Reading the numbers
```text
Metric: requests per task
Anchor: the trajectory digest's requests_by_source, summed over sources
Population: every graded trial of a task that ran to its own stop
Excludes: tasks the token budget skipped, and runs that hit timeout_sec
```
Requests is the headline because round trips are what these tasks exist to
move. Read it beside the solve rate and the report's round outcomes: a change
that drops requests by giving up on verification has not cut a round trip, it
has cut a check. `rt-stale-skills` is read for whether the run finishes at all
and how many rounds go to re-verifying after its last write.
## Baseline
`deepseek-flash`, 2 trials per task, at studio `0b8de9e63`. Requests are the
trajectory's executor requests; all 8 runs ran to their own stop.
| Task | Solved | Requests | `edit_file` calls | Edit rounds |
| --- | --- | --- | --- | --- |
| `rt-cache-write-rate` | 2/2 | 31, 28 | 21, 20 | 8, 8 |
| `rt-plan-ablation-module` | 2/2 | 24, 30 | 10, 13 | 9, 7 |
| `rt-rate-lookup-signatures` | 2/2 | 13, 15 | 10, 17 | 2, 6 |
| `rt-stale-skills` | 2/2* | 20, 26 | 3, 2 | 3, 3 |
\* One run first graded as failed, on a glob (`*_rates_test.go`) and a
placeholder (`benchctl <subcommand>`) that were both correct. The grader now
resolves globs against the tree and skips placeholders, and that run counts
as solved. The same task's first version failed a run for naming
`CatalogEntry.EffectiveFrom` in order to warn against it; that check went too.
Reads and edits already batch: 20 edit calls fit into 8 rounds, and several
`edit_file` calls to one file often share a round. Rounds that edit a file the
previous round also edited are rare (0–2 per run), so steering same-file edits
to `multi_edit` has little to save. On this suite's first version, a
`ToolBatchPolicy` sentence asking for it drew no `multi_edit` call and 167
requests against 158.
`rt-plan-ablation-module` spends five rounds per run on `complete_step` alone.
A description change inviting the sign-off to share the next step's reply was
taken up in 2 of 4 trials (4 of 5 sign-offs shared each time, against 0 of 20
in control), and requests did not move: 30, 25, 21, 28 against 27, 23, 24, 30.
At this spread, four trials per arm cannot show a saving of a few rounds.
## Gaps
- **Below the intended size, in places.** The tasks were designed for 25–50
requests. After widening, the median is 25, but `rt-rate-lookup-signatures`
still runs 13–15: the model greps every call site first and batches the edits,
so even about 20 call sites cost few rounds, and no run hit a compile-error
loop.
- **Lexical grader.** `rt-stale-skills` judges Markdown by resolving what it
names. It has misjudged correct text twice (see the baseline note); a new
failure there should be read before it is counted.
- **No git history.** The workdir is a plain directory: `git diff`, `git log`
and `git status` have nothing to read, so runs that lean on them in this tree
behave differently here.
- **Plan marker and `-force-planner`.** `rt-plan-ablation-module` carries the
plan-first marker in its prompt, so it engages the planner in every arm,
including runs without `-force-planner`. Exclude it when comparing planner
arms.
@@ -1,18 +0,0 @@
---
name: price-change
description: Record a vendor's announced price change in the official price book without losing the rate it replaces.
---
# Recording a price change
1. Find the vendor's rows in `officialRates` in
`internal/contract/pricing/rates.go`. Each row is one model in one currency.
2. **Append** a `rateGeneration` to the row's `Rates`; never edit an existing
one. Set `From` to the first billed day (`YYYY-MM-DD`, UTC) when the change
is announced ahead of time.
3. Leave the earlier generation where it is: `SupersededRates` reads it so a
stored price that matches it is recognised as ours and refreshed.
4. Update the row's `CheckedOn` constant to the day you confirmed the page.
5. Add a test beside `rates_test.go` that pins `CurrentRate` before and after
the `From` day by replacing `today`.
6. Run `make check`. `make pricecheck` needs the network and is run by hand.
-17
View File
@@ -1,17 +0,0 @@
.PHONY: check test vet fmt pricecheck
check: fmt vet test
fmt:
@out="$$(gofmt -l .)"; if [ -n "$$out" ]; then echo "gofmt needed:"; echo "$$out"; exit 1; fi
vet:
go vet ./...
test:
go test ./...
# Reads each vendor's pricing page; this snapshot carries no network client.
pricecheck:
@echo "pricecheck: reaches each vendor's pricing page and needs the network; not available in this snapshot" >&2
@exit 2
-69
View File
@@ -1,69 +0,0 @@
# Reasonix kernel snapshot
A trimmed slice of the Reasonix Go kernel: the contracts a benchmark report is
priced and labelled with, a few leaf utilities, and `benchctl`, a small command
that prints what those contracts answer.
```bash
make check # gofmt, vet, test
go run ./cmd/benchctl rate -model deepseek-flash -currency USD
go run ./cmd/benchctl cost -provider mimo -model mimo-v2.5 -miss 1000000
go run ./cmd/benchctl tariff -currency CNY -at 2026-10-12T02:00:00Z
go run ./cmd/benchctl arm -ablate planner,evidence
```
## Packages
| Package | Owns |
| --- | --- |
| `internal/contract/pricing` | the official price book, cost quotes, the spend ledger |
| `internal/contract/ablation` | which subsystems a benchmark arm switches off, and the arm's name |
| `internal/contract/agentgraph` | the vocabulary of one run's execution graph |
| `internal/base/retrieval` | BM25 ranking over CJK-aware tokens |
| `internal/base/rulelist` | splitting a one-line permission rule list |
| `internal/base/tokencount` | a cheap token estimate |
| `internal/base/neterr` | classifying transport errors |
| `internal/base/nilutil` | typed-nil detection |
| `cmd/benchctl` | prints rates, costs, peak schedules and arm names |
## Pricing
A `RateCard` is a price per million tokens in one currency:
| Field | Charges |
| --- | --- |
| `CacheHit` | prompt tokens served from the provider's cache |
| `Input` | prompt tokens that missed the cache, including tokens written to it |
| `Output` | completion tokens |
`OriginalCostAmount` turns a card and a `UsageTokens` breakdown into a
fixed-point `Amount`; `PricingFingerprint` keys a card in the ledger, so a
stored ledger entry stays attached to the card it was priced with.
`CurrentRate` and `SupersededRates` answer from `officialRates`, which only
ever grows: a price change appends a generation.
## Ablation
`ablation.Parse` reads a comma-separated list of modules to switch off, and
`Set.Arm` names the result (`full` for the control). The switchable modules,
in the order arm names use:
| Module | Off means |
| --- | --- |
| `evidence` | no evidence gate on completion claims |
| `planner` | no two-model planning turn |
| `subagent` | no delegation to workers |
| `retrieval` | no retrieval over the folded transcript |
| `compaction` | no context folding |
| `upstream` | a dependency edge orders its endpoints without delivering the answer |
| `recall-search` | recall can read a given address but cannot search for one |
The fold-index scale is a separate axis (`-fold-index`), not a module.
## Skills
Project skills live under `.reasonix/skills/<name>/SKILL.md`.
| Skill | Use it to |
| --- | --- |
| `price-change` | record a vendor's price change in the price book |
-51
View File
@@ -1,51 +0,0 @@
# Reasonix project memory
This checkout is a trimmed snapshot of the Reasonix kernel: the billing,
ablation and graph contracts and a handful of leaf utilities, plus one small
command that drives them. The conventions below are the project's own.
## Conventions
- Go kernel under `internal/`; each package owns one concern. A package's long
explanation belongs in its `doc.go`, not spread across implementation files.
- Judgements read structure, never wording: types, contracts, schemas. An
error a caller must tell apart carries an identity — a sentinel, a typed
code — never a sentence.
- A metric is its definition, not its name. Before a number becomes a finding,
state what it counts, from which anchor, over which population, and what it
excludes.
## Comments
Default is none — the code is the truth. Write one only when the **why** is
non-obvious: a hidden constraint, an invariant the type system cannot express,
or an external-protocol quirk. A comment states the constraint that holds now;
it does not narrate how the code got here. English, always.
- Declaration doc: ≤5 lines. Package comment: ≤8 lines, or ≤40 in a `doc.go`.
- Every other comment: ≤3 lines. Struct-field and trailing `//`: 1 line.
- One responsibility per file; 800 lines is the ceiling.
## Vendor rates
Official list prices live in one place: `officialRates` in
`internal/contract/pricing/rates.go`, oldest generation first, the last being
what a model charges today. A price change is an **append, never an edit** —
the superseded rate is what lets an installed config be recognised as holding
one of ours and brought forward. Callers read these through
`pricing.CurrentRate` and `pricing.SupersededRates` and restate none of them.
`make pricecheck` compares that table against each vendor's own page. It needs
the network, so it is not part of `make check`.
## Before every commit
```bash
make check # gofmt -l, go vet ./..., go test ./...
```
## Change discipline
List callers, consumers of the same state and sibling features before editing.
Fix at the owning layer; a fix that breaks a neighbouring behaviour is a failed
fix. Write tests first; they must fail without the change.
@@ -1,115 +0,0 @@
// Command benchctl prints what the pricing and ablation contracts answer for
// one model or one benchmark arm, so a reviewer can check a rate or an arm name
// without writing a test.
package main
import (
"errors"
"flag"
"fmt"
"io"
"os"
"reasonix/internal/contract/ablation"
"reasonix/internal/contract/pricing"
)
var errUsage = errors.New("usage: benchctl rate|cost|tariff|arm [flags]")
func main() {
if err := run(os.Args[1:], os.Stdout); err != nil {
fmt.Fprintln(os.Stderr, "benchctl:", err)
os.Exit(2)
}
}
func run(args []string, out io.Writer) error {
if len(args) == 0 {
return errUsage
}
switch args[0] {
case "rate":
return runRate(args[1:], out)
case "cost":
return runCost(args[1:], out)
case "tariff":
return runTariff(args[1:], out)
case "arm":
return runArm(args[1:], out)
default:
return fmt.Errorf("%w: unknown subcommand %q", errUsage, args[0])
}
}
func runRate(args []string, out io.Writer) error {
fs := flag.NewFlagSet("rate", flag.ContinueOnError)
fs.SetOutput(out)
provider := fs.String("provider", "deepseek", "vendor")
model := fs.String("model", "deepseek-flash", "model id")
currency := fs.String("currency", "", "currency; empty reads the vendor's default")
if err := fs.Parse(args); err != nil {
return err
}
cur := *currency
if cur == "" {
cur = pricing.DefaultCurrency(*provider)
}
card := pricing.CurrentRate(*provider, *model, cur)
if card == nil {
return fmt.Errorf("no official rate for %s/%s in %s", *provider, *model, cur)
}
fmt.Fprintf(out, "%s/%s %s: cache-hit %g input %g output %g per 1M\n",
*provider, *model, card.Currency, card.CacheHit, card.Input, card.Output)
if entry, ok := pricing.LookupCatalog(*provider, *model, cur, ""); ok && entry.DocURL != "" {
fmt.Fprintf(out, "source: %s\n", entry.DocURL)
}
fmt.Fprintf(out, "superseded generations: %d\n", len(pricing.SupersededRates(*provider, *model, cur)))
return nil
}
func runCost(args []string, out io.Writer) error {
fs := flag.NewFlagSet("cost", flag.ContinueOnError)
fs.SetOutput(out)
provider := fs.String("provider", "deepseek", "vendor")
model := fs.String("model", "deepseek-flash", "model id")
currency := fs.String("currency", "", "currency; empty reads the vendor's default")
hit := fs.Int("hit", 0, "cached prompt tokens")
miss := fs.Int("miss", 0, "uncached prompt tokens")
completion := fs.Int("completion", 0, "completion tokens")
if err := fs.Parse(args); err != nil {
return err
}
cur := *currency
if cur == "" {
cur = pricing.DefaultCurrency(*provider)
}
card := pricing.CurrentRate(*provider, *model, cur)
if card == nil {
return fmt.Errorf("no official rate for %s/%s in %s", *provider, *model, cur)
}
usage := pricing.UsageTokens{CacheHitTokens: *hit, CacheMissTokens: *miss, CompletionTokens: *completion}
amount := pricing.OriginalCostAmount(*card, usage)
fmt.Fprintf(out, "%s %s\n", amount, pricing.NormalizeCurrency(card.Currency))
return nil
}
func runArm(args []string, out io.Writer) error {
fs := flag.NewFlagSet("arm", flag.ContinueOnError)
fs.SetOutput(out)
spec := fs.String("ablate", "", "comma-separated modules to switch off: "+ablation.ModuleList())
foldIndex := fs.String("fold-index", "", "fold-index scale: default, half, quarter or off")
list := fs.Bool("list", false, "list the switchable modules")
if err := fs.Parse(args); err != nil {
return err
}
if *list {
fmt.Fprintln(out, ablation.ModuleList())
return nil
}
set, err := ablation.ParseArm(*spec, *foldIndex)
if err != nil {
return err
}
fmt.Fprintln(out, set.Arm())
return nil
}
@@ -1,65 +0,0 @@
package main
import (
"bytes"
"errors"
"strings"
"testing"
)
func runOut(t *testing.T, args ...string) string {
t.Helper()
var out bytes.Buffer
if err := run(args, &out); err != nil {
t.Fatalf("run %v: %v", args, err)
}
return out.String()
}
func TestRateNamesTheCurrentCard(t *testing.T) {
got := runOut(t, "rate", "-provider", "mimo", "-model", "mimo-v2.5")
if !strings.Contains(got, "mimo/mimo-v2.5 CNY: cache-hit 0.02 input 1 output 2 per 1M") {
t.Fatalf("rate output = %q", got)
}
}
func TestRateRefusesAnUnpricedModel(t *testing.T) {
var out bytes.Buffer
if err := run([]string{"rate", "-provider", "nobody", "-model", "x"}, &out); err == nil {
t.Fatal("an unpriced model must fail")
}
}
func TestCostChargesTheCurrentCard(t *testing.T) {
got := runOut(t, "cost", "-provider", "mimo", "-model", "mimo-v2.5", "-miss", "1000000", "-completion", "1000000")
if strings.TrimSpace(got) != "3 CNY" {
t.Fatalf("cost output = %q", got)
}
}
func TestArmNamesTheConfiguration(t *testing.T) {
if got := strings.TrimSpace(runOut(t, "arm")); got != "full" {
t.Fatalf("control arm = %q", got)
}
if got := strings.TrimSpace(runOut(t, "arm", "-ablate", "planner,evidence", "-fold-index", "half")); got != "no-evidence+no-planner+index-half" {
t.Fatalf("arm = %q", got)
}
}
func TestUnknownSubcommandIsAUsageError(t *testing.T) {
var out bytes.Buffer
if err := run([]string{"nope"}, &out); !errors.Is(err, errUsage) {
t.Fatalf("err = %v, want errUsage", err)
}
}
func TestTariffReadsTheVendorsSchedule(t *testing.T) {
got := runOut(t, "tariff", "-provider", "mimo", "-model", "mimo-v2.5")
if !strings.Contains(got, "official card: true") || !strings.Contains(got, "one rate around the clock") {
t.Fatalf("tariff output = %q", got)
}
peak := runOut(t, "tariff", "-currency", "CNY", "-at", "2026-10-12T02:00:00Z")
if !strings.Contains(peak, "peak=true until 2026-10-12T04:00:00Z") {
t.Fatalf("deepseek tariff at a weekday peak = %q", peak)
}
}
@@ -1,49 +0,0 @@
package main
import (
"flag"
"fmt"
"io"
"time"
"reasonix/internal/contract/pricing"
)
func runTariff(args []string, out io.Writer) error {
fs := flag.NewFlagSet("tariff", flag.ContinueOnError)
fs.SetOutput(out)
provider := fs.String("provider", "deepseek", "vendor")
model := fs.String("model", "deepseek-flash", "model id")
currency := fs.String("currency", "", "currency; empty reads the vendor's default")
at := fs.String("at", "", "instant to read the schedule at, RFC 3339; empty is now")
if err := fs.Parse(args); err != nil {
return err
}
when := time.Now()
if *at != "" {
t, err := time.Parse(time.RFC3339, *at)
if err != nil {
return fmt.Errorf("-at: %w", err)
}
when = t
}
cur := *currency
if cur == "" {
cur = pricing.DefaultCurrency(*provider)
}
card := pricing.CurrentRate(*provider, *model, cur)
if card == nil {
return fmt.Errorf("no official rate for %s/%s in %s", *provider, *model, cur)
}
fmt.Fprintf(out, "official card: %v\n", pricing.IsOfficialCard(*provider, *model, *card))
tariff, ok := pricing.LiveTariff(*provider, *model, *card, when)
switch {
case !ok:
fmt.Fprintln(out, "schedule: one rate around the clock")
case !tariff.Changes:
fmt.Fprintf(out, "schedule: peak=%v, no change ahead\n", tariff.Peak)
default:
fmt.Fprintf(out, "schedule: peak=%v until %s\n", tariff.Peak, tariff.ChangesAt.UTC().Format(time.RFC3339))
}
return nil
}
-3
View File
@@ -1,3 +0,0 @@
module reasonix
go 1.26.0
@@ -1,95 +0,0 @@
package pricing
import (
"strings"
"testing"
"time"
)
// Every catalog provider must declare at least one official endpoint, or its
// rows become unreachable: a rate card is only matched against the catalog
// after the entry's host identifies the vendor.
func TestEveryCatalogProviderHasAnOfficialEndpoint(t *testing.T) {
declared := map[string]bool{}
for _, host := range officialEndpointHosts {
declared[host] = true
}
for _, entry := range OfficialCatalog() {
if !declared[entry.Provider] {
t.Errorf("catalog provider %q has no official endpoint host; its prices can never match", entry.Provider)
}
}
}
func TestOfficialProviderForEndpoint(t *testing.T) {
cases := []struct {
name string
baseURL string
want string
}{
{"official deepseek", "https://api.deepseek.com", "deepseek"},
{"official deepseek anthropic path", "https://api.deepseek.com/anthropic", "deepseek"},
{"official longcat", "https://api.longcat.chat/openai/v1", "longcat"},
{"official mimo region", "https://token-plan-sgp.xiaomimimo.com/v1", "mimo"},
{"uppercase host", "https://API.DeepSeek.com/v1", "deepseek"},
{"proxy that resells the model", "https://my-deepseek-proxy.internal/v1", ""},
{"vendor name in the path only", "https://gateway.example.com/deepseek/v1", ""},
{"lookalike registrable domain", "https://api.deepseek.com.evil.test/v1", ""},
{"empty", "", ""},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
if got := OfficialProviderForEndpoint(tc.baseURL); got != tc.want {
t.Fatalf("OfficialProviderForEndpoint(%q) = %q, want %q", tc.baseURL, got, tc.want)
}
})
}
}
// pricesGoStaleAfter is how long a hand-copied rate is trusted before someone
// reads the vendor's page again. No vendor here serves prices over an API, so
// the alternative to this going red on a quiet day is learning about a change
// from a user whose cost was wrong for months. Bumping CheckedOn without
// opening DocURL defeats it; what this buys is that the question gets asked.
const pricesGoStaleAfter = 90 * 24 * time.Hour
func TestCatalogPricesHaveBeenCheckedRecently(t *testing.T) {
var undated, stale []string
// This snapshot never sees a vendor page again, so staleness is read at the
// day it was taken rather than the day the suite runs.
cutoff := time.Date(2026, 10, 19, 0, 0, 0, 0, time.UTC).Add(-pricesGoStaleAfter)
for _, e := range OfficialCatalog() {
where := e.Provider + "/" + e.Model + " " + e.Currency + " (" + e.DocURL + ")"
if e.CheckedOn == "" {
undated = append(undated, where)
continue
}
on, err := time.Parse("2006-01-02", e.CheckedOn)
if err != nil {
t.Fatalf("%s: CheckedOn %q is not YYYY-MM-DD", where, e.CheckedOn)
}
if on.Before(cutoff) {
stale = append(stale, where+" last checked "+e.CheckedOn)
}
}
if len(undated) > 0 {
t.Errorf("catalog rows with no CheckedOn — read the vendor's page and date them:\n %s",
strings.Join(undated, "\n "))
}
if len(stale) > 0 {
t.Errorf("catalog rows older than %d days. Open each DocURL, confirm or correct the rate, then set CheckedOn to today:\n %s",
int(pricesGoStaleAfter.Hours()/24), strings.Join(stale, "\n "))
}
}
// A row that names a peak card must say when it applies, and one that names a
// window must have a card to charge inside it. Half of the pair silently prices
// every hour the same, which is the bug the pair exists to prevent.
func TestPeakRatesAndWindowsComeInPairs(t *testing.T) {
for _, e := range OfficialCatalog() {
where := e.Provider + "/" + e.Model + " " + e.Currency
if (e.Peak == nil) != (e.Window == nil) {
t.Errorf("%s: Peak and Window must be set together (peak=%v window=%v)", where, e.Peak != nil, e.Window != nil)
}
}
}
@@ -1,12 +0,0 @@
---
name: ablation-arm
description: Add a new switchable subsystem to the benchmark's ablation arms.
---
# Adding an ablation module
1. Declare the module in `internal/ablation/modules.go` beside the others.
2. Register it in `AllModules()` so `ParseSpec` accepts it.
3. Arm names come from `Set.Name()`; check that yours reads `without-<module>`.
4. Add the module to the flag help text in `cmd/bench/flags.go`.
5. Run `make test`.
@@ -1,15 +0,0 @@
---
name: benchctl
description: Check a rate, a cost, a peak schedule or an arm name from the command line instead of writing a test.
---
# Using benchctl
1. Price a request with `benchctl quote -provider mimo -model mimo-v2.5 -tokens 1000000`;
it prints the amount and the currency.
2. Read today's card with `benchctl rate -vendor deepseek -model deepseek-flash`.
3. Ask whether a vendor is in its peak window with
`benchctl peak -currency CNY -at 1791770400` (Unix seconds).
4. List the ablation modules with `benchctl arms`, and name an arm with
`benchctl arm -ablate planner,evidence -index half`.
5. Every subcommand prints its flags with `-h`.
@@ -1,13 +0,0 @@
---
name: price-change
description: Record a vendor's announced price change in the official price book.
---
# Recording a price change
1. Open `internal/pricing/rates.go` and find the vendor's entry in
`officialPrices`.
2. Edit the entry's `Input`, `Output` and `CacheHit` to the new values.
3. If the change is announced ahead of time, set `EffectiveFrom` on the entry.
4. Confirm `pricing.LatestRate` now returns the new card.
5. Run `make test` and `make price-check`.
-52
View File
@@ -1,52 +0,0 @@
#!/usr/bin/env bash
# Builds every task's workdir/ from one pinned commit of this repository plus
# the fixture-authored files in base/ and overlay/<task-id>/. With --check it
# rebuilds into a scratch directory and fails if any committed workdir differs.
set -euo pipefail
PIN=42636ddcb425b7af07abd88397d02c1cfe3adcf5
PATHS=(
internal/contract/pricing
internal/contract/ablation
internal/contract/agentgraph
internal/base/retrieval
internal/base/rulelist
internal/base/tokencount
internal/base/neterr
internal/base/nilutil
)
here="$(cd "$(dirname "$0")" && pwd)"
repo="$(git -C "$here" rev-parse --show-toplevel)"
build() {
local id="$1" dest="$2"
rm -rf "$dest"
mkdir -p "$dest"
git -C "$repo" archive "$PIN" -- "${PATHS[@]}" | tar -x -C "$dest"
cp -R "$here/base/." "$dest/"
if [ -d "$here/overlay/$id" ]; then
cp -R "$here/overlay/$id/." "$dest/"
fi
}
check=0
[ "${1:-}" = "--check" ] && check=1
status=0
for task in "$here"/tasks/*/; do
id="$(basename "$task")"
if [ "$check" = 1 ]; then
scratch="$(mktemp -d)"
build "$id" "$scratch/workdir"
if ! diff -r "$scratch/workdir" "$task/workdir" >/dev/null; then
echo "$id: workdir/ differs from the pinned snapshot; run $0" >&2
diff -rq "$scratch/workdir" "$task/workdir" >&2 || true
status=1
fi
rm -rf "$scratch"
else
build "$id" "$task/workdir"
fi
done
exit "$status"
@@ -1,73 +0,0 @@
# Reasonix kernel snapshot
A trimmed slice of the Reasonix Go kernel: the contracts a benchmark report is
priced and labelled with, a few leaf utilities, and `benchctl`, a small command
that prints what those contracts answer.
```bash
make check # gofmt, vet, test
go run ./cmd/benchctl rate -model deepseek-flash -currency USD
go run ./cmd/benchctl cost -provider mimo -model mimo-v2.5 -miss 1000000
go run ./cmd/benchctl tariff -currency CNY -at 2026-10-12T02:00:00Z
go run ./cmd/benchctl arm -ablate planner,evidence
```
## Packages
| Package | Owns |
| --- | --- |
| `internal/contract/pricing` | the official price book, cost quotes, the spend ledger |
| `internal/contract/ablation` | which subsystems a benchmark arm switches off, and the arm's name |
| `internal/contract/agentgraph` | the vocabulary of one run's execution graph |
| `internal/base/retrieval` | BM25 ranking over CJK-aware tokens |
| `internal/base/rulelist` | splitting a one-line permission rule list |
| `internal/base/tokencount` | a cheap token estimate |
| `internal/base/neterr` | classifying transport errors |
| `internal/base/nilutil` | typed-nil detection |
| `cmd/benchctl` | prints rates, costs, peak schedules and arm names |
## Pricing
A `RateCard` is a price per million tokens in one currency:
| Field | Charges |
| --- | --- |
| `CacheHit` | prompt tokens served from the provider's cache |
| `Input` | prompt tokens that missed the cache, and tokens written to it when `CacheWrite` is zero |
| `Output` | completion tokens |
| `CacheWrite` | tokens written to the cache, for a vendor that prices them separately (Anthropic) |
A card without `CacheWrite` keeps the fingerprint it had before the field
existed, so ledgers stored against it stay attached.
`OriginalCostAmount` turns a card and a `UsageTokens` breakdown into a
fixed-point `Amount`; `PricingFingerprint` keys a card in the ledger, so a
stored ledger entry stays attached to the card it was priced with.
`CurrentRate` and `SupersededRates` answer from `officialRates`, which only
ever grows: a price change appends a generation.
## Ablation
`ablation.Parse` reads a comma-separated list of modules to switch off, and
`Set.Arm` names the result (`full` for the control). The switchable modules,
in the order arm names use:
| Module | Off means |
| --- | --- |
| `evidence` | no evidence gate on completion claims |
| `planner` | no two-model planning turn |
| `subagent` | no delegation to workers |
| `retrieval` | no retrieval over the folded transcript |
| `compaction` | no context folding |
| `upstream` | a dependency edge orders its endpoints without delivering the answer |
| `recall-search` | recall can read a given address but cannot search for one |
The fold-index scale is a separate axis (`-fold-index`), not a module.
## Skills
Project skills live under `.reasonix/skills/<name>/SKILL.md`.
| Skill | Use it to |
| --- | --- |
| `price-change` | record a vendor's price change in the price book |
@@ -1,119 +0,0 @@
// Command benchctl prints what the pricing and ablation contracts answer for
// one model or one benchmark arm, so a reviewer can check a rate or an arm name
// without writing a test.
package main
import (
"errors"
"flag"
"fmt"
"io"
"os"
"reasonix/internal/contract/ablation"
"reasonix/internal/contract/pricing"
)
var errUsage = errors.New("usage: benchctl rate|cost|tariff|arm [flags]")
func main() {
if err := run(os.Args[1:], os.Stdout); err != nil {
fmt.Fprintln(os.Stderr, "benchctl:", err)
os.Exit(2)
}
}
func run(args []string, out io.Writer) error {
if len(args) == 0 {
return errUsage
}
switch args[0] {
case "rate":
return runRate(args[1:], out)
case "cost":
return runCost(args[1:], out)
case "tariff":
return runTariff(args[1:], out)
case "arm":
return runArm(args[1:], out)
default:
return fmt.Errorf("%w: unknown subcommand %q", errUsage, args[0])
}
}
func runRate(args []string, out io.Writer) error {
fs := flag.NewFlagSet("rate", flag.ContinueOnError)
fs.SetOutput(out)
provider := fs.String("provider", "deepseek", "vendor")
model := fs.String("model", "deepseek-flash", "model id")
currency := fs.String("currency", "", "currency; empty reads the vendor's default")
if err := fs.Parse(args); err != nil {
return err
}
cur := *currency
if cur == "" {
cur = pricing.DefaultCurrency(*provider)
}
card := pricing.CurrentRate(*provider, *model, cur)
if card == nil {
return fmt.Errorf("no official rate for %s/%s in %s", *provider, *model, cur)
}
fmt.Fprintf(out, "%s/%s %s: cache-hit %g input %g output %g", *provider, *model, card.Currency, card.CacheHit, card.Input, card.Output)
if card.CacheWrite != 0 {
fmt.Fprintf(out, " cache-write %g", card.CacheWrite)
}
fmt.Fprintln(out, " per 1M")
if entry, ok := pricing.LookupCatalog(*provider, *model, cur, ""); ok && entry.DocURL != "" {
fmt.Fprintf(out, "source: %s\n", entry.DocURL)
}
fmt.Fprintf(out, "superseded generations: %d\n", len(pricing.SupersededRates(*provider, *model, cur)))
return nil
}
func runCost(args []string, out io.Writer) error {
fs := flag.NewFlagSet("cost", flag.ContinueOnError)
fs.SetOutput(out)
provider := fs.String("provider", "deepseek", "vendor")
model := fs.String("model", "deepseek-flash", "model id")
currency := fs.String("currency", "", "currency; empty reads the vendor's default")
hit := fs.Int("hit", 0, "cached prompt tokens")
miss := fs.Int("miss", 0, "uncached prompt tokens")
completion := fs.Int("completion", 0, "completion tokens")
write := fs.Int("write", 0, "uncached prompt tokens that were written to the cache")
if err := fs.Parse(args); err != nil {
return err
}
cur := *currency
if cur == "" {
cur = pricing.DefaultCurrency(*provider)
}
card := pricing.CurrentRate(*provider, *model, cur)
if card == nil {
return fmt.Errorf("no official rate for %s/%s in %s", *provider, *model, cur)
}
usage := pricing.UsageTokens{CacheHitTokens: *hit, CacheMissTokens: *miss, CompletionTokens: *completion, CacheWriteTokens: *write}
amount := pricing.OriginalCostAmount(*card, usage)
fmt.Fprintf(out, "%s %s\n", amount, pricing.NormalizeCurrency(card.Currency))
return nil
}
func runArm(args []string, out io.Writer) error {
fs := flag.NewFlagSet("arm", flag.ContinueOnError)
fs.SetOutput(out)
spec := fs.String("ablate", "", "comma-separated modules to switch off: "+ablation.ModuleList())
foldIndex := fs.String("fold-index", "", "fold-index scale: default, half, quarter or off")
list := fs.Bool("list", false, "list the switchable modules")
if err := fs.Parse(args); err != nil {
return err
}
if *list {
fmt.Fprintln(out, ablation.ModuleList())
return nil
}
set, err := ablation.ParseArm(*spec, *foldIndex)
if err != nil {
return err
}
fmt.Fprintln(out, set.Arm())
return nil
}
@@ -1,3 +0,0 @@
module reasonix
go 1.26.0
@@ -1,37 +0,0 @@
package pricing
import "testing"
func TestCacheWriteRateChargesWrittenTokens(t *testing.T) {
card := RateCard{Input: 3, CacheWrite: 3.75, Currency: "USD"}
usage := UsageTokens{CacheMissTokens: 1000000, CacheWriteTokens: 400000}
if got := OriginalCostAmount(card, usage).String(); got != "3.3" {
t.Fatalf("cost = %s, want 3.3", got)
}
card.CacheWrite = 0
if got := OriginalCostAmount(card, usage).String(); got != "3" {
t.Fatalf("cost without a cache-write rate = %s, want 3", got)
}
}
func TestCacheWriteSeparatesFingerprintsAndCards(t *testing.T) {
base := RateCard{CacheHit: 0.02, Input: 1, Output: 2, Currency: "CNY"}
withWrite := base
withWrite.CacheWrite = 1.25
if PricingFingerprint(base) == PricingFingerprint(withWrite) {
t.Fatal("cache-write rate does not reach the fingerprint")
}
if sameCard(base, withWrite) {
t.Fatal("cards differing in CacheWrite compare equal")
}
}
func TestAnthropicPricesCacheWrites(t *testing.T) {
got := CurrentRate("anthropic", "claude-sonnet-4-5", "USD")
if got == nil || got.CacheWrite != 3.75 {
t.Fatalf("claude-sonnet-4-5 = %+v", got)
}
if OfficialProviderForEndpoint("https://api.anthropic.com/v1/messages") != "anthropic" {
t.Fatal("api.anthropic.com is not recognised as Anthropic's endpoint")
}
}
@@ -1,222 +0,0 @@
package pricing
import (
"net/url"
"strings"
"time"
)
// Official catalog metadata: currency, model, effective dates, documentation
// sources, and price fingerprints. User-custom prices always win over catalog.
// CatalogEntry is one official list price for a model in a billing currency.
type CatalogEntry struct {
Provider string // deepseek | longcat | mimo | google
Model string
Currency string // ISO billing currency for this row
CacheHit float64
Input float64
Output float64
CacheWrite float64 // zero when the vendor bills cache writes at Input
EffectiveFrom string // YYYY-MM-DD inclusive, empty = unbounded
EffectiveTo string // YYYY-MM-DD exclusive, empty = unbounded
DocURL string
BillingMode string // payg | subscription_equivalent
Notes string
Fingerprint string // filled by Register
// Peak is what the same tokens cost inside Window. Nil when the vendor
// bills one rate around the clock, and then the fields above are the rate.
Peak *RateCard
Window *PeakWindow
// CheckedOn is when a person last read DocURL and confirmed these numbers.
// No vendor here serves prices over an API, so this date is all that stands
// between a price change and a readout that is quietly wrong.
CheckedOn string
}
// deepseekPeak is the schedule DeepSeek publishes: peak on weekday mornings and
// afternoons, and — since 2026-08-23 — never at a weekend. Chinese public
// holidays bill off-peak in full, per the same page.
var deepseekPeak = &PeakWindow{
OffsetSeconds: beijing,
Hours: [][2]int{{9, 12}, {14, 18}},
WeekendOffPeak: "2026-08-23",
HolidayOffPeak: chineseStatutoryHolidays,
}
// chineseStatutoryHolidays are the State Council's 2026 public holidays, which
// DeepSeek bills off-peak in full; extend it when the next notice is out. Make-up
// workdays are absent: they fall on weekends, which are off-peak anyway.
var chineseStatutoryHolidays = []string{
// New Year's Day: Jan 1-3.
"2026-01-01", "2026-01-02", "2026-01-03",
// Spring Festival: Feb 15-23.
"2026-02-15", "2026-02-16", "2026-02-17", "2026-02-18", "2026-02-19",
"2026-02-20", "2026-02-21", "2026-02-22", "2026-02-23",
// Qingming Festival: Apr 4-6.
"2026-04-04", "2026-04-05", "2026-04-06",
// Labor Day: May 1-5.
"2026-05-01", "2026-05-02", "2026-05-03", "2026-05-04", "2026-05-05",
// Dragon Boat Festival: Jun 19-21.
"2026-06-19", "2026-06-20", "2026-06-21",
// Mid-Autumn Festival: Sep 25-27.
"2026-09-25", "2026-09-26", "2026-09-27",
// National Day: Oct 1-7.
"2026-10-01", "2026-10-02", "2026-10-03", "2026-10-04",
"2026-10-05", "2026-10-06", "2026-10-07",
}
// checkedOn is when the tables below were last read off the vendors' pages.
const checkedOn = "2026-08-22"
// deepseekCheckedOn is when the DeepSeek rows were last read off that vendor's
// page. Separate from checkedOn, because confirming one vendor's prices says
// nothing about another's and a shared date would claim that it did.
const deepseekCheckedOn = "2026-09-13"
// mimoCheckedOn is when the V2.6 rows were read off the vendor's own docs.
const mimoCheckedOn = "2026-10-06"
// googleCheckedOn is when the Gemini rows were read off Google's pricing page.
const googleCheckedOn = "2026-10-17"
// anthropicCheckedOn is when the Claude rows were read off Anthropic's pricing page.
const anthropicCheckedOn = "2026-10-19"
// CatalogSourceURLs document public pricing pages.
const (
DocDeepSeekPricing = "https://api-docs.deepseek.com/quick_start/pricing"
DocDeepSeekPricingCNY = "https://api-docs.deepseek.com/zh-cn/quick_start/pricing"
DocLongCatPricingUSD = "https://longcat.chat/platform/docs/Pricing/LongCat-2.0.html"
DocLongCatPricingCNY = "https://longcat.chat/platform/docs/zh/pricing/long-cat-2.0"
DocMiMoPAYG = "https://mimo.mi.com/docs/price/pay-as-you-go"
DocMiMoTokenPlan = "https://platform.xiaomimimo.com/token-plan"
DocGeminiPricing = "https://ai.google.dev/gemini-api/docs/pricing"
DocAnthropicPricing = "https://www.anthropic.com/pricing"
)
// officialEndpointHosts maps a vendor's own API hostnames to the catalog
// provider whose list prices apply there. Host equality is the test: a reseller
// or proxy bills on its own terms even when it serves the same model, and an
// entry merely named after a vendor is not evidence of the vendor's endpoint.
var officialEndpointHosts = map[string]string{
"api.anthropic.com": "anthropic",
"api.deepseek.com": "deepseek",
"api.longcat.chat": "longcat",
"api.xiaomimimo.com": "mimo",
"generativelanguage.googleapis.com": "google",
"token-plan-cn.xiaomimimo.com": "mimo",
"token-plan-sgp.xiaomimimo.com": "mimo",
"token-plan-ams.xiaomimimo.com": "mimo",
}
// OfficialProviderForEndpoint returns the catalog provider whose official
// prices apply to baseURL, or "" when the address is not a vendor's own API.
func OfficialProviderForEndpoint(baseURL string) string {
u, err := url.Parse(strings.TrimSpace(baseURL))
if err != nil {
return ""
}
return officialEndpointHosts[strings.ToLower(u.Hostname())]
}
// OfficialCatalog is the built-in price book: the current generation of every
// rate in officialRates, fingerprinted so a stored price can be recognised as
// one of ours. User-custom prices always win over it.
func OfficialCatalog() []CatalogEntry {
entries := officialCatalogRows()
for i := range entries {
entries[i].Fingerprint = PricingFingerprint(RateCardFromCatalog(entries[i]))
}
return entries
}
// LookupCatalog finds an official entry. billingMode empty matches payg first.
func LookupCatalog(provider, model, currency, billingMode string) (CatalogEntry, bool) {
provider = strings.ToLower(strings.TrimSpace(provider))
model = strings.TrimSpace(model)
currency = NormalizeCurrency(currency)
billingMode = strings.TrimSpace(billingMode)
if billingMode == "" {
billingMode = BillingModePAYG
}
var fallback CatalogEntry
foundFallback := false
for _, e := range OfficialCatalog() {
if e.Provider != provider || e.Model != model {
continue
}
if NormalizeCurrency(e.Currency) != currency {
continue
}
if e.BillingMode == billingMode {
return e, true
}
if e.BillingMode == BillingModePAYG {
fallback = e
foundFallback = true
}
}
if foundFallback {
return fallback, true
}
return CatalogEntry{}, false
}
// RateCardFromCatalog builds a RateCard from a catalog entry's base rate.
func RateCardFromCatalog(e CatalogEntry) RateCard {
return RateCard{
CacheHit: e.CacheHit,
Input: e.Input,
Output: e.Output,
CacheWrite: e.CacheWrite,
Currency: e.Currency,
}
}
// RatesAt is what this entry charges at t: the peak card inside its window, the
// base card everywhere else. An entry without a window charges one rate, and
// then t does not matter.
func (e CatalogEntry) RatesAt(t time.Time) RateCard {
if e.Peak == nil || !e.Window.IsPeak(t) {
return RateCardFromCatalog(e)
}
peak := *e.Peak
peak.Currency = e.Currency
return peak
}
// InEffect reports whether this entry's dates cover the day t falls on.
func (e CatalogEntry) InEffect(t time.Time) bool {
day := t.UTC().Format("2006-01-02")
if e.EffectiveFrom != "" && day < e.EffectiveFrom {
return false
}
return e.EffectiveTo == "" || day < e.EffectiveTo
}
// MatchesCatalog reports whether rates equal a known official entry.
func MatchesCatalog(provider, model string, rates RateCard) (CatalogEntry, bool) {
cur := NormalizeCurrency(rates.Currency)
for _, e := range OfficialCatalog() {
if e.Provider != strings.ToLower(strings.TrimSpace(provider)) {
continue
}
if e.Model != strings.TrimSpace(model) {
continue
}
if NormalizeCurrency(e.Currency) != cur {
continue
}
// Either card identifies the vendor's own price. Which one a user
// happens to have configured says nothing about when they are billing:
// the hour decides that, and RatesAt is what reads it.
if sameCard(RateCardFromCatalog(e), rates) {
return e, true
}
if p := e.Peak; p != nil && sameCard(*p, rates) {
return e, true
}
}
return CatalogEntry{}, false
}
@@ -1,799 +0,0 @@
package pricing
import (
"crypto/sha256"
"encoding/hex"
"fmt"
"slices"
"strings"
"time"
)
// Billing modes.
const (
BillingModePAYG = "payg"
BillingModeSubscriptionEquivalent = "subscription_equivalent"
)
// Valuation basis values.
const (
BasisIdentity = "identity"
BasisOfficialTable = "official_table"
// BasisFX is retained for decoding pre-v6 persisted quotes only. New
// quotes never create FX valuations.
BasisFX = "fx"
)
// DisplayStatus describes whether a quote can satisfy its requested display.
// It is deliberately separate from CostComplete: a known price-book amount
// can exist even when no requested regional valuation is available.
const (
DisplayStatusMatched = "matched"
DisplayStatusFallbackOriginal = "fallback_original"
DisplayStatusBucketed = "bucketed"
DisplayStatusUnavailable = "unavailable"
)
// Aggregate modes describe how a session/run total was formed.
const (
AggregateModeSingleCurrency = "single_currency"
AggregateModeCommonValuation = "common_valuation"
AggregateModeCurrencyBuckets = "currency_buckets"
)
// Rate bands name which side of a vendor's peak window a call was billed on.
// An aggregate is mixed when its calls fell on both sides.
const (
RateBandPeak = "peak"
RateBandOffPeak = "off_peak"
RateBandMixed = "mixed"
)
// DisplayRequest identifies how a caller selected the requested presentation
// currency. The source is runtime context only; it is never persisted as a
// provider price or wallet fact.
type DisplayRequest struct {
Currency string `json:"currency,omitempty"`
Source string `json:"source,omitempty"`
}
const (
DisplaySourceExplicit = "explicit"
DisplaySourceWallet = "wallet"
DisplaySourceAuto = "auto"
)
func (r DisplayRequest) NormalizedCurrency() string {
return NormalizeCurrency(r.Currency)
}
// CostQuote is the host-side quoted cost for one model call (or an aggregate).
// Original is the pricing-table currency fact; valuations hold identity or
// official regional rate-card estimates. Selected is the display pick for the
// current preference.
type CostQuote struct {
Original Money `json:"original"`
// OriginalTotals is populated for mixed-currency aggregates. Individual
// quotes and single-currency totals may omit it.
OriginalTotals []Money `json:"originalTotals,omitempty"`
Valuations map[string]Valuation `json:"valuations,omitempty"`
Selected *Money `json:"selected,omitempty"`
BillingMode string `json:"billingMode,omitempty"`
Estimated bool `json:"estimated"`
// CostComplete means usage and the price-book amount are known. It does not
// imply that a requested display currency is available.
CostComplete bool `json:"costComplete"`
// Coverage is the same fact with the states a boolean cannot hold: an
// aggregate that priced only some of its calls, and one that billed nothing.
Coverage string `json:"coverage,omitempty"`
// DisplayComplete means a single selected amount satisfies the display
// request. Complete is retained as the old wire alias.
DisplayComplete bool `json:"displayComplete"`
Complete bool `json:"complete"`
DisplayStatus string `json:"displayStatus,omitempty"`
AggregateMode string `json:"aggregateMode,omitempty"`
// ModelRef / UsageSource / PricingFingerprint identify the ledger key.
ModelRef string `json:"modelRef,omitempty"`
UsageSource string `json:"usageSource,omitempty"`
PricingFingerprint string `json:"pricingFingerprint,omitempty"`
RateDate string `json:"rateDate,omitempty"` // YYYY-MM-DD of FX used, if any
IncompleteReason string `json:"incompleteReason,omitempty"`
LegacyEstimate bool `json:"legacyEstimate,omitempty"`
CatalogSource string `json:"catalogSource,omitempty"`
// RateBand is set only for a rate confirmed as a vendor's own schedule; an
// aggregate that includes any call without one carries none.
RateBand string `json:"rateBand,omitempty"`
}
// Valuation is one currency view of a cost fact.
type Valuation struct {
Money Money `json:"money"`
Basis string `json:"basis"`
Source string `json:"source"`
AsOf string `json:"asOf"` // YYYY-MM-DD
Rate *RateSnapshot `json:"rateSnapshot,omitempty"`
Stale bool `json:"stale,omitempty"`
}
// RateSnapshot records the FX observation used for a valuation.
type RateSnapshot struct {
Base string `json:"base"`
Quote string `json:"quote"`
Rate float64 `json:"rate"`
Source string `json:"source"`
AsOf string `json:"asOf"`
Stale bool `json:"stale,omitempty"`
}
// RateCard is the per-1M-token price used to compute original cost. It mirrors
// provider.Pricing without importing that package (billing is a leaf).
type RateCard struct {
CacheHit float64 // per 1M cached prompt tokens
Input float64 // per 1M uncached prompt tokens
Output float64 // per 1M completion tokens
// CacheWrite is per 1M tokens written to the cache; zero bills them at Input.
CacheWrite float64
Currency string // ISO or symbol; normalized on quote
}
// UsageTokens is the token breakdown needed for cost. Mirrors provider.Usage
// fields without importing provider.
type UsageTokens struct {
PromptTokens int
CompletionTokens int
CacheHitTokens int
CacheMissTokens int
CacheWriteTokens int
CacheWriteBilledTokens float64
Estimated bool
}
// QuoteInput drives a single CostQuote computation.
type QuoteInput struct {
Usage UsageTokens
Rates RateCard
OccurredAt time.Time
DisplayCurrency string // compatibility alias for Display.Currency
Display DisplayRequest // runtime display request; empty means auto
BillingMode string
ModelRef string
UsageSource string
CatalogSource string
PricingFingerprint string
// ProviderKind is deepseek|longcat|mimo|… for official dual-table lookup.
// When empty, ModelRef and Rates are used to infer a catalog match.
ProviderKind string
// ModelID is the bare model id (e.g. deepseek-v4-flash). Empty → parse ModelRef.
ModelID string
}
// OriginalCostAmount computes the fixed-point original cost from rates + tokens.
// Mirrors the historic provider.Pricing.Cost semantics.
func OriginalCostAmount(rates RateCard, u UsageTokens) Amount {
hit := u.CacheHitTokens
miss := u.CacheMissTokens
if hit+miss == 0 && u.PromptTokens > 0 {
miss = u.PromptTokens
} else if miss == 0 && hit > 0 && u.PromptTokens > hit {
miss = u.PromptTokens - hit
}
write := u.CacheWriteTokens
write = max(write, 0)
write = min(write, miss)
billedWrite := 0.0
if write > 0 {
billedWrite = u.CacheWriteBilledTokens
if billedWrite <= 0 {
billedWrite = float64(write)
}
}
writeRate := rates.Input
if rates.CacheWrite > 0 {
writeRate = rates.CacheWrite
}
total := (float64(hit)*rates.CacheHit +
float64(miss-write)*rates.Input +
billedWrite*writeRate +
float64(u.CompletionTokens)*rates.Output) / 1e6
return NewAmountFromFloat(total)
}
// PricingFingerprint hashes a rate card for ledger keys.
func PricingFingerprint(rates RateCard) string {
cur := NormalizeCurrency(rates.Currency)
raw := fmt.Sprintf("%s|%.12g|%.12g|%.12g", cur, rates.CacheHit, rates.Input, rates.Output)
if rates.CacheWrite != 0 {
// Appended only when set, so every card stored before the field keeps its key.
raw += fmt.Sprintf("|w%.12g", rates.CacheWrite)
}
sum := sha256.Sum256([]byte(raw))
return hex.EncodeToString(sum[:8])
}
// BuildQuote computes original cost and CNY/USD valuations.
func BuildQuote(in QuoteInput) CostQuote {
state := newQuoteBuildState(in)
for _, target := range []string{"CNY", "USD"} {
state.addTargetValuation(target)
}
state.selectDisplay()
return state.quote
}
type quoteBuildState struct {
input QuoteInput
currency string
amount Amount
occurred time.Time
official CatalogEntry
isOfficial bool
valuationFailures map[string]string
quote CostQuote
}
func newQuoteBuildState(in QuoteInput) *quoteBuildState {
currency := NormalizeCurrency(in.Rates.Currency)
if currency == "" {
currency = "CNY"
}
mode := strings.TrimSpace(in.BillingMode)
if mode == "" {
mode = BillingModePAYG
}
occurred := in.OccurredAt
if occurred.IsZero() {
occurred = time.Now().UTC()
}
fingerprint := strings.TrimSpace(in.PricingFingerprint)
if fingerprint == "" {
fingerprint = PricingFingerprint(in.Rates)
}
amount := OriginalCostAmount(in.Rates, in.Usage)
q := CostQuote{
Original: MoneyOf(amount, currency),
Valuations: map[string]Valuation{},
BillingMode: mode,
Estimated: true,
CostComplete: true,
Coverage: CoverageComplete,
DisplayComplete: true,
Complete: true,
DisplayStatus: DisplayStatusMatched,
AggregateMode: AggregateModeSingleCurrency,
ModelRef: strings.TrimSpace(in.ModelRef),
UsageSource: strings.TrimSpace(in.UsageSource),
PricingFingerprint: fingerprint,
CatalogSource: strings.TrimSpace(in.CatalogSource),
}
q.Valuations[currency] = Valuation{
Money: q.Original, Basis: BasisIdentity,
Source: firstNonEmpty(in.CatalogSource, "rate_card"),
AsOf: occurred.UTC().Format("2006-01-02"),
}
state := &quoteBuildState{
input: in, currency: currency, amount: amount, occurred: occurred,
valuationFailures: map[string]string{}, quote: q,
}
if !usageHasFacts(in.Usage) {
state.markIncomplete("missing_price_or_usage")
}
state.matchOfficialCatalog()
state.applyOfficialSchedule()
return state
}
// applyOfficialSchedule re-prices the turn at the vendor's rate for the hour it
// actually happened. Only a rate confirmed as the vendor's own earns this: a
// custom price is the user's own arrangement, and projecting someone else's
// peak schedule onto it would invent a number nobody published.
func (s *quoteBuildState) applyOfficialSchedule() {
if !s.isOfficial || s.official.Peak == nil {
return
}
s.quote.RateBand = RateBandOffPeak
if s.official.Window.IsPeak(s.occurred) {
s.quote.RateBand = RateBandPeak
}
rates := s.official.RatesAt(s.occurred)
if rates.CacheHit == s.input.Rates.CacheHit && rates.Input == s.input.Rates.Input && rates.Output == s.input.Rates.Output {
return
}
s.amount = OriginalCostAmount(rates, s.input.Usage)
s.quote.Original = MoneyOf(s.amount, s.currency)
s.quote.PricingFingerprint = PricingFingerprint(rates)
s.quote.Valuations[s.currency] = Valuation{
Money: s.quote.Original, Basis: BasisOfficialTable,
Source: s.official.DocURL,
AsOf: s.occurred.UTC().Format("2006-01-02"),
}
}
func quoteDisplayRequest(in QuoteInput) DisplayRequest {
request := in.Display
if request.Currency == "" && in.DisplayCurrency != "" {
request.Currency = in.DisplayCurrency
if request.Source == "" {
request.Source = DisplaySourceExplicit
}
}
if request.Source == "" {
request.Source = DisplaySourceAuto
}
return request
}
func usageHasFacts(u UsageTokens) bool {
return u.PromptTokens > 0 || u.CompletionTokens > 0 || u.CacheHitTokens > 0 ||
u.CacheMissTokens > 0 || u.CacheWriteTokens > 0 || u.CacheWriteBilledTokens > 0
}
func (s *quoteBuildState) matchOfficialCatalog() {
providerKind, modelID := resolveCatalogIdentity(s.input)
s.official, s.isOfficial = MatchesCatalog(providerKind, modelID, s.input.Rates)
if !s.isOfficial && providerKind != "" && modelID != "" {
rates := s.input.Rates
rates.Currency = s.currency
s.official, s.isOfficial = MatchesCatalog(providerKind, modelID, rates)
}
if s.isOfficial && s.quote.CatalogSource == "" {
s.quote.CatalogSource = s.official.DocURL
}
}
func (s *quoteBuildState) addTargetValuation(target string) {
if target == s.currency {
return
}
if s.addOfficialValuation(target) {
return
}
// Runtime FX has intentionally been removed. A custom price that does not
// match the official catalog remains in its original currency.
s.valuationFailures[target] = "display_unavailable"
}
func (s *quoteBuildState) addOfficialValuation(target string) bool {
if !s.isOfficial {
return false
}
peer, ok := LookupCatalog(
s.official.Provider, s.official.Model, target, s.official.BillingMode,
)
if !ok {
return false
}
s.quote.Valuations[target] = Valuation{
Money: MoneyOf(OriginalCostAmount(peer.RatesAt(s.occurred), s.input.Usage), target),
Basis: BasisOfficialTable, Source: peer.DocURL,
AsOf: s.occurred.UTC().Format("2006-01-02"),
}
return true
}
func (s *quoteBuildState) selectDisplay() {
if !s.quote.CostComplete {
s.quote.Selected = nil
s.quote.DisplayComplete = false
s.quote.Complete = false
s.quote.DisplayStatus = DisplayStatusUnavailable
return
}
display := quoteDisplayRequest(s.input).NormalizedCurrency()
if display == "" {
selected := s.quote.Original
s.quote.Selected = &selected
return
}
if valuation, ok := s.quote.Valuations[display]; ok {
selected := valuation.Money
s.quote.Selected = &selected
return
}
// The original price-book amount is still a valid cost fact. Keep it as a
// visible fallback and distinguish the display mismatch from no pricing.
selected := s.quote.Original
s.quote.Selected = &selected
s.quote.DisplayComplete = false
s.quote.Complete = false
s.quote.DisplayStatus = DisplayStatusFallbackOriginal
s.quote.IncompleteReason = firstNonEmpty(s.valuationFailures[display], "display_unavailable")
}
func (s *quoteBuildState) markIncomplete(reason string) {
s.quote.CostComplete = false
s.quote.Coverage = CoverageIncomplete
s.quote.DisplayComplete = false
s.quote.Complete = false
s.quote.DisplayStatus = DisplayStatusUnavailable
if s.quote.IncompleteReason == "" {
s.quote.IncompleteReason = reason
}
}
// SelectForDisplay returns the money for a display currency preference without
// recomputing rates. Empty display returns original.
func (q CostQuote) SelectForDisplay(display string) (Money, bool) {
display = NormalizeCurrency(display)
if display == "" {
return q.Original, true
}
if v, ok := q.Valuations[display]; ok {
return v.Money, true
}
if NormalizeCurrency(q.Original.Currency) == display {
return q.Original, true
}
return Money{}, false
}
// WithSelected returns a copy with Selected set for display.
func (q CostQuote) WithSelected(display string) CostQuote {
out := q
if m, ok := q.SelectForDisplay(display); ok {
out.Selected = &m
if quoteHasCompleteCostFact(q) {
out.Complete = true
out.IncompleteReason = ""
}
} else {
out.Selected = nil
out.Complete = false
if out.IncompleteReason == "" {
out.IncompleteReason = "display_unavailable"
}
}
return out
}
// LegacyCostFloat returns the selected (or original) amount as float64 for
// compatibility aliases cost / costUsd / total_cost.
func (q CostQuote) LegacyCostFloat() float64 {
if q.Selected != nil {
return q.Selected.Float64()
}
return q.Original.Float64()
}
// LegacyCurrencySymbol returns a display symbol for the selected/original currency.
func (q CostQuote) LegacyCurrencySymbol() string {
if q.Selected != nil && q.Selected.Currency != "" {
return CurrencySymbol(q.Selected.Currency)
}
return CurrencySymbol(q.Original.Currency)
}
// LegacyCurrencyCode returns the ISO code for selected/original.
func (q CostQuote) LegacyCurrencyCode() string {
if q.Selected != nil && q.Selected.Currency != "" {
return NormalizeCurrency(q.Selected.Currency)
}
return NormalizeCurrency(q.Original.Currency)
}
// AggregateQuotes combines occurrence-time quotes into a session/run total.
// Every entry must have the requested valuation; partial totals stay unavailable.
func AggregateQuotes(quotes []CostQuote, display string) CostQuote {
if len(quotes) == 0 {
return emptyAggregate(NormalizeCurrency(display))
}
accumulator := newQuoteAccumulator(display)
for _, quote := range quotes {
accumulator.add(quote)
}
return accumulator.finish()
}
type quoteAccumulator struct {
out CostQuote
display string
totals map[string]Amount
originalCurrency string
originalTotal Amount
originalTotals map[string]Amount
originalComplete bool
coverage string
displayComplete bool
modes map[string]struct{}
rateBands map[string]struct{}
unbanded bool
}
func emptyAggregate(display string) CostQuote {
out := CostQuote{
Original: MoneyOf(Zero, display), Valuations: map[string]Valuation{}, Estimated: true,
CostComplete: false, DisplayComplete: false, Complete: false,
Coverage: CoverageNone,
DisplayStatus: DisplayStatusUnavailable, AggregateMode: AggregateModeSingleCurrency,
IncompleteReason: "no_usage",
}
return out
}
func newQuoteAccumulator(display string) *quoteAccumulator {
return &quoteAccumulator{
out: CostQuote{Valuations: map[string]Valuation{}, Estimated: true},
display: NormalizeCurrency(display), totals: map[string]Amount{}, originalTotals: map[string]Amount{},
originalComplete: true, coverage: CoverageNone, displayComplete: true,
modes: map[string]struct{}{}, rateBands: map[string]struct{}{},
}
}
func (a *quoteAccumulator) add(quote CostQuote) {
quote = NormalizeQuote(quote)
a.coverage = FoldCoverage(a.coverage, quote.Coverage)
if quote.Coverage != CoverageComplete && a.out.IncompleteReason == "" {
a.out.IncompleteReason = quote.IncompleteReason
}
a.out.LegacyEstimate = a.out.LegacyEstimate || quote.LegacyEstimate
if quote.BillingMode != "" {
a.modes[quote.BillingMode] = struct{}{}
}
a.addRateBand(quote.RateBand)
if quote.RateDate > a.out.RateDate {
a.out.RateDate = quote.RateDate
}
originalCurrency := NormalizeCurrency(quote.Original.Currency)
if len(quote.OriginalTotals) > 0 {
for _, original := range quote.OriginalTotals {
a.addOriginal(original, NormalizeCurrency(original.Currency))
}
} else {
a.addOriginal(quote.Original, originalCurrency)
}
if a.display != "" {
_, found := quote.Valuations[a.display]
a.displayComplete = a.displayComplete && (found || originalCurrency == a.display)
}
originalIncluded := false
for code, valuation := range quote.Valuations {
code = NormalizeCurrency(code)
if code == originalCurrency {
originalIncluded = true
}
a.addValuation(code, valuation)
}
if originalCurrency != "" && !originalIncluded {
a.addValuation(originalCurrency, Valuation{
Money: quote.Original, Basis: BasisIdentity, Source: "aggregate",
AsOf: time.Now().UTC().Format("2006-01-02"),
})
}
}
func (a *quoteAccumulator) addRateBand(band string) {
switch band {
case RateBandPeak, RateBandOffPeak:
a.rateBands[band] = struct{}{}
case RateBandMixed:
a.rateBands[RateBandPeak], a.rateBands[RateBandOffPeak] = struct{}{}, struct{}{}
default:
a.unbanded = true
}
}
func (a *quoteAccumulator) rateBand() string {
if a.unbanded {
return ""
}
switch len(a.rateBands) {
case 1:
for band := range a.rateBands {
return band
}
case 2:
return RateBandMixed
}
return ""
}
func (a *quoteAccumulator) addOriginal(original Money, currency string) {
if currency != "" {
a.originalTotals[currency] = a.originalTotals[currency].Add(original.AmountValue())
}
if a.originalCurrency == "" {
a.originalCurrency = currency
a.originalTotal = original.AmountValue()
return
}
if currency != a.originalCurrency {
a.originalComplete = false
return
}
a.originalTotal = a.originalTotal.Add(original.AmountValue())
}
func (a *quoteAccumulator) addValuation(code string, valuation Valuation) {
if code == "" {
return
}
a.totals[code] = a.totals[code].Add(valuation.Money.AmountValue())
current, found := a.out.Valuations[code]
if !found {
current = valuation
}
current.Money = MoneyOf(a.totals[code], code)
current.Stale = current.Stale || valuation.Stale
a.out.Valuations[code] = current
}
// costFactsComplete reads the accumulated coverage rather than tracking the
// same fact a second time: two answers to one question drift.
func (a *quoteAccumulator) costFactsComplete() bool {
return a.coverage == CoverageComplete
}
func (a *quoteAccumulator) finish() CostQuote {
a.out.Coverage = a.coverage
a.out.CostComplete = a.costFactsComplete()
if a.originalComplete && a.originalCurrency != "" {
a.out.Original = MoneyOf(a.originalTotal, a.originalCurrency)
a.syncOriginalValuation()
} else {
a.out.Original = Money{Amount: "0"}
}
if len(a.originalTotals) > 1 {
codes := make([]string, 0, len(a.originalTotals))
for code := range a.originalTotals {
codes = append(codes, code)
}
slices.Sort(codes)
for _, code := range codes {
a.out.OriginalTotals = append(a.out.OriginalTotals, MoneyOf(a.originalTotals[code], code))
}
}
a.out.RateBand = a.rateBand()
if len(a.modes) == 1 {
for mode := range a.modes {
a.out.BillingMode = mode
}
}
if a.display == "" && a.originalComplete && a.costFactsComplete() {
selected := a.out.Original
a.out.Selected = &selected
a.out.DisplayComplete = true
a.out.Complete = true
a.out.DisplayStatus = DisplayStatusMatched
a.out.AggregateMode = AggregateModeSingleCurrency
return a.out
}
valuation, found := a.out.Valuations[a.display]
if found && a.displayComplete && a.costFactsComplete() {
selected := valuation.Money
a.out.Selected = &selected
a.out.DisplayComplete = true
a.out.Complete = true
a.out.DisplayStatus = DisplayStatusMatched
if len(a.originalTotals) > 1 {
a.out.AggregateMode = AggregateModeCommonValuation
} else {
a.out.AggregateMode = AggregateModeSingleCurrency
}
return a.out
}
a.out.DisplayComplete = false
a.out.Complete = false
if !a.costFactsComplete() {
a.out.DisplayStatus = DisplayStatusUnavailable
a.out.IncompleteReason = firstNonEmpty(a.out.IncompleteReason, "incomplete_cost_fact")
} else if a.originalComplete && a.originalCurrency != "" {
selected := a.out.Original
a.out.Selected = &selected
a.out.DisplayStatus = DisplayStatusFallbackOriginal
a.out.AggregateMode = AggregateModeSingleCurrency
a.out.IncompleteReason = firstNonEmpty(a.out.IncompleteReason, "display_unavailable")
} else {
a.out.DisplayStatus = DisplayStatusBucketed
a.out.AggregateMode = AggregateModeCurrencyBuckets
a.out.IncompleteReason = firstNonEmpty(a.out.IncompleteReason, "currency_buckets")
}
return a.out
}
// NormalizeQuote fills fields introduced after the first CostQuote wire shape.
// It is used at every persistence and transport boundary so old telemetry and
// old eventwire payloads remain safe without inventing a new amount.
func NormalizeQuote(q CostQuote) CostQuote {
if !q.CostComplete && !q.DisplayComplete && q.Complete {
q.CostComplete = true
q.DisplayComplete = true
}
if q.DisplayStatus != "" && q.DisplayStatus != DisplayStatusMatched && q.DisplayStatus != DisplayStatusFallbackOriginal && q.DisplayStatus != DisplayStatusBucketed && q.DisplayStatus != DisplayStatusUnavailable {
q.DisplayStatus = ""
}
if q.RateBand != RateBandPeak && q.RateBand != RateBandOffPeak && q.RateBand != RateBandMixed {
q.RateBand = ""
}
if q.AggregateMode != "" && q.AggregateMode != AggregateModeSingleCurrency && q.AggregateMode != AggregateModeCommonValuation && q.AggregateMode != AggregateModeCurrencyBuckets {
q.AggregateMode = ""
}
if q.DisplayStatus == "" {
switch {
case q.Complete:
q.DisplayStatus = DisplayStatusMatched
case q.Original.Currency != "" && q.Original.Amount != "" && q.IncompleteReason != "no_price":
q.DisplayStatus = DisplayStatusFallbackOriginal
default:
q.DisplayStatus = DisplayStatusUnavailable
}
}
if q.AggregateMode == "" {
if len(q.OriginalTotals) > 1 {
q.AggregateMode = AggregateModeCurrencyBuckets
} else if q.Selected != nil {
q.AggregateMode = AggregateModeSingleCurrency
}
}
if !ValidCoverage(q.Coverage) {
q.Coverage = CoverageIncomplete
if quoteHasCompleteCostFact(q) {
q.Coverage = CoverageComplete
}
}
q.Complete = q.DisplayComplete
return q
}
// quoteHasCompleteCostFact separates a known original-currency charge from a
// display-only valuation failure must not poison an exact original total,
// while unpriced and unrecoverable legacy records stay incomplete in every
// display currency.
func quoteHasCompleteCostFact(q CostQuote) bool {
if q.CostComplete {
return true
}
switch q.IncompleteReason {
case "no_price", "missing_price_or_usage", "legacy_unrecoverable",
"legacy_wiped_or_zero", "legacy_invalid_amount", "mixed_original_currencies",
"incomplete_cost_fact":
return false
}
if q.Complete {
return true
}
currency := NormalizeCurrency(q.Original.Currency)
if currency == "" {
return false
}
valuation, ok := q.Valuations[currency]
return ok && SameCurrency(valuation.Money.Currency, currency)
}
func (a *quoteAccumulator) syncOriginalValuation() {
valuation, found := a.out.Valuations[a.originalCurrency]
if !found {
valuation = Valuation{
Basis: BasisIdentity, Source: "aggregate",
AsOf: time.Now().UTC().Format("2006-01-02"),
}
}
valuation.Money = a.out.Original
a.out.Valuations[a.originalCurrency] = valuation
}
func firstNonEmpty(values ...string) string {
for _, v := range values {
if strings.TrimSpace(v) != "" {
return strings.TrimSpace(v)
}
}
return ""
}
// resolveCatalogIdentity names which official table, if any, may price a quote.
// The vendor comes only from ProviderKind, which the caller reads off the
// endpoint's host; guessing it from the entry's name or the model id would bill
// a relay as the vendor it resells. Unnamed means no official table — the user's
// own rate card, which is right rather than merely plausible.
func resolveCatalogIdentity(in QuoteInput) (provider, model string) {
provider = strings.ToLower(strings.TrimSpace(in.ProviderKind))
model = strings.TrimSpace(in.ModelID)
if model == "" {
ref := strings.TrimSpace(in.ModelRef)
if i := strings.LastIndex(ref, "/"); i >= 0 && i+1 < len(ref) {
model = ref[i+1:]
} else {
model = ref
}
}
return provider, model
}
Loaded 100 of 7424 files, more files were not shown because too many files have changed in this diff. Show more