mirror of
https://github.com/esengine/DeepSeek-Reasonix.git
synced 2026-11-11 23:53:19 -10:00
Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
58c43fdf6c |
No files matched your search
@@ -5,7 +5,6 @@
|
||||
*.bat text eol=crlf
|
||||
*.css text eol=lf
|
||||
*.go text eol=lf
|
||||
*.golden text eol=lf
|
||||
*.html text eol=lf
|
||||
*.js text eol=lf
|
||||
*.json text eol=lf
|
||||
@@ -26,8 +25,6 @@
|
||||
*.xml text eol=lf
|
||||
*.yaml text eol=lf
|
||||
*.yml text eol=lf
|
||||
Makefile text eol=lf
|
||||
.githooks/** text eol=lf
|
||||
benchmarks/**/*.sh text eol=lf
|
||||
internal/runtime/agent/testdata/perseveration/** -text
|
||||
desktop/frontend-next/perf/media/delivery.pdf binary
|
||||
@@ -1,7 +0,0 @@
|
||||
# Logins allowed to open pull requests, one per line, matched case-insensitively.
|
||||
# Repository owners, members and collaborators, and bots, are always allowed.
|
||||
lancy69
|
||||
chuiwenwei
|
||||
xxoingr
|
||||
kevin9327
|
||||
imcreazy
|
||||
@@ -2,9 +2,6 @@
|
||||
Target branch: 2.x features and fixes go to `studio`. 1.x takes bug fixes,
|
||||
provider/API compatibility, release/updater, security and platform stability
|
||||
only, on `main-v2`.
|
||||
|
||||
Pull requests are limited to maintainers and invited contributors; one from
|
||||
anyone else is closed automatically. Open an issue instead.
|
||||
-->
|
||||
|
||||
## Summary
|
||||
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 138 KiB |
@@ -0,0 +1,19 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<svg id="_图层_2" data-name="图层 2" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 145.55 113.29">
|
||||
<defs>
|
||||
<style>
|
||||
.cls-1 {
|
||||
fill: #ffffff;
|
||||
stroke: #ffffff;
|
||||
stroke-miterlimit: 10;
|
||||
}
|
||||
</style>
|
||||
</defs>
|
||||
<g id="_图层_1-2" data-name="图层 1">
|
||||
<g>
|
||||
<path class="cls-1" d="M144.68,38.49l-.06-.23c-.88-3.28-2.5-5.94-4.58-8.06.14,5.65-2.96,11.02-6.22,16.66l-.39.68c-2.26,3.91-4.66,7.94-6.57,11.1l-.86,1.38c-3.36,5.44-6.27,10.14-12.18,14.18-8.34,5.87-18.2,5.81-26.92,5.76-2.81-.02-5.48-.03-7.95.14l-.35.02c-3.22.06-5.96,1.57-8.17,4.49l-.14.18c-.86,1.06-1.7,2.26-2.58,3.54-3.43,4.92-7.69,11.04-16.17,12.42-4.37.86-9.98.84-14.94.83h-1.95c-.52,0-1.06,0-1.61,0-6.95,0-16.08-.89-21.94-6.55-.15-.15-.3-.3-.44-.45.5,4.73,2.33,8.64,5.44,11.65,5.86,5.66,14.98,6.55,21.94,6.55.55,0,1.09,0,1.61-.01h1.93c4.96.02,10.58.04,14.96-.83,8.48-1.37,12.74-7.5,16.17-12.42.88-1.27,1.72-2.48,2.58-3.54l.14-.18c2.21-2.92,4.95-4.43,8.17-4.49l.35-.02c2.47-.17,5.13-.16,7.95-.14,8.72.05,18.58.11,26.91-5.76,5.92-4.03,8.82-8.73,12.19-14.17l.86-1.39c1.91-3.15,4.3-7.18,6.57-11.09l.39-.69c3.81-6.6,7.41-12.84,5.86-19.57ZM120.9,23.02c-.28,0-.56,0-.83,0-9.68-.19-24.03-.09-35.57-.01l-2.04.02c-.93.01-1.82.02-2.67.03-8.36.08-14.4.13-23.33,3.82l-.27.12c-10.76,4.68-16.91,12.16-22.83,21.95-5.53,8.76-12.62,20.57-16.32,26.79-.41.69-.77,1.3-1.09,1.84l-.49.84c-1.49,2.53-3.23,5.49-4.21,8.95,3.98,1.9,8.52,2.65,12.71,2.91.27-.77.64-1.56,1.07-2.38.48-.94,1.05-1.9,1.63-2.89l.49-.84c.96-1.62,2.38-3.99,4.05-6.78,3.82-6.36,8.98-14.88,13.19-21.55l.07-.11c5.02-8.31,9.2-13.45,16.91-16.81l.11-.05c6.57-2.7,10.54-2.74,18.43-2.81.87-.01,1.78-.02,2.69-.03l1.99-.02c9.17-.06,20.13-.14,29.01-.07,2.26.01,4.39.04,6.32.08h.12s.11,0,.11,0c1.25-.01,2.91.11,4.61.46,1.03.22,2.08.52,3.05.93.21-.35.41-.71.62-1.06l.39-.68c1.92-3.33,3.79-6.57,4.97-9.82-4.08-1.92-8.7-2.76-12.89-2.82Z"/>
|
||||
<path class="cls-1" d="M115.07,11.81c-9.7-.19-24.07-.09-35.62-.01h-1.99c-.93.03-1.82.04-2.67.04-8.36.08-14.4.14-23.33,3.83l-.27.12c-10.76,4.67-16.91,12.16-22.83,21.95-6.13,9.71-14.19,23.21-17.41,28.63l-.5.84c-2.08,3.55-4.68,7.97-4.94,13.39v.2s0,.21,0,.21c.01.81.06,1.61.15,2.38.14.15.29.3.44.45,1.53,1.47,3.28,2.63,5.15,3.52,3.98,1.9,8.52,2.65,12.71,2.91,1.41.09,2.78.12,4.08.12.55,0,1.09-.01,1.61-.01h1.95c4.95.01,10.57.03,14.94-.83,8.48-1.38,12.74-7.5,16.17-12.42.88-1.28,1.72-2.48,2.58-3.54l.14-.18c2.21-2.92,4.95-4.44,8.17-4.49l.35-.02c2.47-.17,5.14-.16,7.95-.14,8.71.05,18.58.1,26.92-5.76,5.91-4.04,8.82-8.74,12.18-14.18l.86-1.39c1.74-2.86,3.87-6.45,5.95-10.03.21-.35.41-.71.62-1.06l.39-.68c1.92-3.33,3.79-6.57,4.97-9.82.82-2.26,1.31-4.53,1.25-6.84-5.18-5.29-13.22-7.25-19.97-7.19ZM122.56,40.37l-.39.67c-2.21,3.81-4.55,7.77-6.39,10.79l-.84,1.36c-3.01,4.87-4.83,7.81-8.48,10.29l-.1.07c-4.94,3.49-11.95,3.45-19.38,3.41-2.88-.02-5.87-.04-8.79.16-7.1.19-13.51,3.58-18.07,9.57-1.13,1.4-2.12,2.83-3.08,4.21-2.92,4.18-4.71,6.57-7.64,7.02l-.31.06c-3.09.63-8.28.61-12.45.6h-2.16c-3.85.07-7.01-.16-9.45-.69-2.24-.48-3.87-1.22-4.89-2.2-.66-.64-1.54-1.81-1.63-4.65.11-1.35.71-2.82,1.52-4.35.48-.94,1.05-1.9,1.63-2.89l.49-.84c3.17-5.33,11.19-18.75,17.24-28.33l.07-.11c5.02-8.32,9.2-13.45,16.92-16.81l.1-.05c6.57-2.7,10.54-2.74,18.43-2.82.87,0,1.78,0,2.69-.03h1.94c11.52-.09,25.86-.19,35.38,0h.23c1.25-.01,2.92.11,4.61.46,3.15.66,6.4,2.12,7.27,5.02.2,1.2-.89,3.62-2.27,6.18-.7,1.31-1.48,2.65-2.2,3.9ZM120.07,23.01c-9.68-.19-24.03-.09-35.57-.01l-2.04.02c-.93.01-1.82.02-2.67.03-8.36.08-14.4.13-23.33,3.82l-.27.12c-10.76,4.68-16.91,12.16-22.83,21.95-5.53,8.76-12.62,20.57-16.32,26.79-.78-.35-1.41-.77-1.9-1.24-.66-.64-1.54-1.81-1.63-4.65.18-2.18,1.62-4.64,3.15-7.25l.49-.83c3.17-5.32,11.18-18.73,17.24-28.33l.07-.12c5.02-8.31,9.2-13.45,16.92-16.81l.1-.04c6.57-2.7,10.54-2.74,18.43-2.82.87-.01,1.78-.01,2.69-.03h1.99c11.5-.09,25.82-.19,35.33,0h.23c3.57-.04,10.55,1.02,11.88,5.48.14.84-.35,2.27-1.13,3.93-.28,0-.56,0-.83,0Z"/>
|
||||
<path class="cls-1" d="M134.68,16.09l-.06-.23c-3.07-11.45-15.08-15.33-24.55-15.25-9.68-.19-24.03-.09-35.57-.01h-2.04c-.93.03-1.82.04-2.67.04-8.36.08-14.4.14-23.33,3.83l-.27.12c-10.76,4.67-16.91,12.16-22.83,21.95-6.14,9.73-14.2,23.22-17.41,28.62l-.49.85c-2.09,3.55-4.69,7.96-4.95,13.39v.2s0,.21,0,.21c.09,5.57,1.82,10.13,5.15,13.58.14.15.29.3.44.45,1.53,1.47,3.28,2.63,5.15,3.52,3.98,1.9,8.52,2.65,12.71,2.91,1.41.09,2.78.12,4.08.12.55,0,1.09-.01,1.61-.01h1.95c4.95.01,10.57.03,14.94-.83,8.48-1.38,12.74-7.5,16.17-12.42.88-1.28,1.72-2.48,2.58-3.54l.14-.18c2.21-2.92,4.95-4.44,8.17-4.49l.35-.02c2.47-.17,5.14-.16,7.95-.14,8.71.05,18.58.1,26.92-5.76,5.91-4.04,8.82-8.74,12.18-14.18l.86-1.39c1.74-2.86,3.87-6.45,5.95-10.03.21-.35.41-.71.62-1.06l.39-.68c1.92-3.33,3.79-6.57,4.97-9.82.82-2.26,1.31-4.53,1.25-6.84-.02-.96-.14-1.93-.36-2.91ZM119.76,25.27c-.7,1.31-1.48,2.65-2.2,3.9l-.39.67c-1.19,2.04-2.41,4.13-3.57,6.09-1.01,1.7-1.97,3.31-2.82,4.7l-.84,1.36c-3.01,4.87-4.83,7.81-8.48,10.29l-.1.07c-4.94,3.49-11.96,3.45-19.38,3.41-2.89-.02-5.87-.04-8.79.16-7.1.18-13.51,3.58-18.07,9.57-1.13,1.4-2.12,2.83-3.08,4.21-2.92,4.18-4.71,6.57-7.64,7.02l-.31.06c-3.09.63-8.27.61-12.45.6h-2.16c-3.85.07-7.01-.16-9.45-.69-1.16-.25-2.16-.57-2.99-.96-.78-.35-1.41-.77-1.9-1.24-.66-.64-1.54-1.81-1.63-4.65.18-2.18,1.62-4.64,3.15-7.25l.49-.83c3.17-5.32,11.18-18.73,17.24-28.33l.07-.12c5.02-8.31,9.2-13.45,16.92-16.81l.1-.04c6.57-2.7,10.54-2.74,18.43-2.82.87-.01,1.78-.01,2.69-.03h1.99c11.5-.09,25.82-.19,35.33,0h.23c3.57-.04,10.55,1.02,11.88,5.48.14.84-.35,2.27-1.13,3.93-.34.72-.73,1.48-1.14,2.25Z"/>
|
||||
</g>
|
||||
</g>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 5.4 KiB |
@@ -0,0 +1,19 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<svg id="_图层_2" data-name="图层 2" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 145.55 113.29">
|
||||
<defs>
|
||||
<style>
|
||||
.cls-1 {
|
||||
fill: #111111;
|
||||
stroke: #111111;
|
||||
stroke-miterlimit: 10;
|
||||
}
|
||||
</style>
|
||||
</defs>
|
||||
<g id="_图层_1-2" data-name="图层 1">
|
||||
<g>
|
||||
<path class="cls-1" d="M144.68,38.49l-.06-.23c-.88-3.28-2.5-5.94-4.58-8.06.14,5.65-2.96,11.02-6.22,16.66l-.39.68c-2.26,3.91-4.66,7.94-6.57,11.1l-.86,1.38c-3.36,5.44-6.27,10.14-12.18,14.18-8.34,5.87-18.2,5.81-26.92,5.76-2.81-.02-5.48-.03-7.95.14l-.35.02c-3.22.06-5.96,1.57-8.17,4.49l-.14.18c-.86,1.06-1.7,2.26-2.58,3.54-3.43,4.92-7.69,11.04-16.17,12.42-4.37.86-9.98.84-14.94.83h-1.95c-.52,0-1.06,0-1.61,0-6.95,0-16.08-.89-21.94-6.55-.15-.15-.3-.3-.44-.45.5,4.73,2.33,8.64,5.44,11.65,5.86,5.66,14.98,6.55,21.94,6.55.55,0,1.09,0,1.61-.01h1.93c4.96.02,10.58.04,14.96-.83,8.48-1.37,12.74-7.5,16.17-12.42.88-1.27,1.72-2.48,2.58-3.54l.14-.18c2.21-2.92,4.95-4.43,8.17-4.49l.35-.02c2.47-.17,5.13-.16,7.95-.14,8.72.05,18.58.11,26.91-5.76,5.92-4.03,8.82-8.73,12.19-14.17l.86-1.39c1.91-3.15,4.3-7.18,6.57-11.09l.39-.69c3.81-6.6,7.41-12.84,5.86-19.57ZM120.9,23.02c-.28,0-.56,0-.83,0-9.68-.19-24.03-.09-35.57-.01l-2.04.02c-.93.01-1.82.02-2.67.03-8.36.08-14.4.13-23.33,3.82l-.27.12c-10.76,4.68-16.91,12.16-22.83,21.95-5.53,8.76-12.62,20.57-16.32,26.79-.41.69-.77,1.3-1.09,1.84l-.49.84c-1.49,2.53-3.23,5.49-4.21,8.95,3.98,1.9,8.52,2.65,12.71,2.91.27-.77.64-1.56,1.07-2.38.48-.94,1.05-1.9,1.63-2.89l.49-.84c.96-1.62,2.38-3.99,4.05-6.78,3.82-6.36,8.98-14.88,13.19-21.55l.07-.11c5.02-8.31,9.2-13.45,16.91-16.81l.11-.05c6.57-2.7,10.54-2.74,18.43-2.81.87-.01,1.78-.02,2.69-.03l1.99-.02c9.17-.06,20.13-.14,29.01-.07,2.26.01,4.39.04,6.32.08h.12s.11,0,.11,0c1.25-.01,2.91.11,4.61.46,1.03.22,2.08.52,3.05.93.21-.35.41-.71.62-1.06l.39-.68c1.92-3.33,3.79-6.57,4.97-9.82-4.08-1.92-8.7-2.76-12.89-2.82Z"/>
|
||||
<path class="cls-1" d="M115.07,11.81c-9.7-.19-24.07-.09-35.62-.01h-1.99c-.93.03-1.82.04-2.67.04-8.36.08-14.4.14-23.33,3.83l-.27.12c-10.76,4.67-16.91,12.16-22.83,21.95-6.13,9.71-14.19,23.21-17.41,28.63l-.5.84c-2.08,3.55-4.68,7.97-4.94,13.39v.2s0,.21,0,.21c.01.81.06,1.61.15,2.38.14.15.29.3.44.45,1.53,1.47,3.28,2.63,5.15,3.52,3.98,1.9,8.52,2.65,12.71,2.91,1.41.09,2.78.12,4.08.12.55,0,1.09-.01,1.61-.01h1.95c4.95.01,10.57.03,14.94-.83,8.48-1.38,12.74-7.5,16.17-12.42.88-1.28,1.72-2.48,2.58-3.54l.14-.18c2.21-2.92,4.95-4.44,8.17-4.49l.35-.02c2.47-.17,5.14-.16,7.95-.14,8.71.05,18.58.1,26.92-5.76,5.91-4.04,8.82-8.74,12.18-14.18l.86-1.39c1.74-2.86,3.87-6.45,5.95-10.03.21-.35.41-.71.62-1.06l.39-.68c1.92-3.33,3.79-6.57,4.97-9.82.82-2.26,1.31-4.53,1.25-6.84-5.18-5.29-13.22-7.25-19.97-7.19ZM122.56,40.37l-.39.67c-2.21,3.81-4.55,7.77-6.39,10.79l-.84,1.36c-3.01,4.87-4.83,7.81-8.48,10.29l-.1.07c-4.94,3.49-11.95,3.45-19.38,3.41-2.88-.02-5.87-.04-8.79.16-7.1.19-13.51,3.58-18.07,9.57-1.13,1.4-2.12,2.83-3.08,4.21-2.92,4.18-4.71,6.57-7.64,7.02l-.31.06c-3.09.63-8.28.61-12.45.6h-2.16c-3.85.07-7.01-.16-9.45-.69-2.24-.48-3.87-1.22-4.89-2.2-.66-.64-1.54-1.81-1.63-4.65.11-1.35.71-2.82,1.52-4.35.48-.94,1.05-1.9,1.63-2.89l.49-.84c3.17-5.33,11.19-18.75,17.24-28.33l.07-.11c5.02-8.32,9.2-13.45,16.92-16.81l.1-.05c6.57-2.7,10.54-2.74,18.43-2.82.87,0,1.78,0,2.69-.03h1.94c11.52-.09,25.86-.19,35.38,0h.23c1.25-.01,2.92.11,4.61.46,3.15.66,6.4,2.12,7.27,5.02.2,1.2-.89,3.62-2.27,6.18-.7,1.31-1.48,2.65-2.2,3.9ZM120.07,23.01c-9.68-.19-24.03-.09-35.57-.01l-2.04.02c-.93.01-1.82.02-2.67.03-8.36.08-14.4.13-23.33,3.82l-.27.12c-10.76,4.68-16.91,12.16-22.83,21.95-5.53,8.76-12.62,20.57-16.32,26.79-.78-.35-1.41-.77-1.9-1.24-.66-.64-1.54-1.81-1.63-4.65.18-2.18,1.62-4.64,3.15-7.25l.49-.83c3.17-5.32,11.18-18.73,17.24-28.33l.07-.12c5.02-8.31,9.2-13.45,16.92-16.81l.1-.04c6.57-2.7,10.54-2.74,18.43-2.82.87-.01,1.78-.01,2.69-.03h1.99c11.5-.09,25.82-.19,35.33,0h.23c3.57-.04,10.55,1.02,11.88,5.48.14.84-.35,2.27-1.13,3.93-.28,0-.56,0-.83,0Z"/>
|
||||
<path class="cls-1" d="M134.68,16.09l-.06-.23c-3.07-11.45-15.08-15.33-24.55-15.25-9.68-.19-24.03-.09-35.57-.01h-2.04c-.93.03-1.82.04-2.67.04-8.36.08-14.4.14-23.33,3.83l-.27.12c-10.76,4.67-16.91,12.16-22.83,21.95-6.14,9.73-14.2,23.22-17.41,28.62l-.49.85c-2.09,3.55-4.69,7.96-4.95,13.39v.2s0,.21,0,.21c.09,5.57,1.82,10.13,5.15,13.58.14.15.29.3.44.45,1.53,1.47,3.28,2.63,5.15,3.52,3.98,1.9,8.52,2.65,12.71,2.91,1.41.09,2.78.12,4.08.12.55,0,1.09-.01,1.61-.01h1.95c4.95.01,10.57.03,14.94-.83,8.48-1.38,12.74-7.5,16.17-12.42.88-1.28,1.72-2.48,2.58-3.54l.14-.18c2.21-2.92,4.95-4.44,8.17-4.49l.35-.02c2.47-.17,5.14-.16,7.95-.14,8.71.05,18.58.1,26.92-5.76,5.91-4.04,8.82-8.74,12.18-14.18l.86-1.39c1.74-2.86,3.87-6.45,5.95-10.03.21-.35.41-.71.62-1.06l.39-.68c1.92-3.33,3.79-6.57,4.97-9.82.82-2.26,1.31-4.53,1.25-6.84-.02-.96-.14-1.93-.36-2.91ZM119.76,25.27c-.7,1.31-1.48,2.65-2.2,3.9l-.39.67c-1.19,2.04-2.41,4.13-3.57,6.09-1.01,1.7-1.97,3.31-2.82,4.7l-.84,1.36c-3.01,4.87-4.83,7.81-8.48,10.29l-.1.07c-4.94,3.49-11.96,3.45-19.38,3.41-2.89-.02-5.87-.04-8.79.16-7.1.18-13.51,3.58-18.07,9.57-1.13,1.4-2.12,2.83-3.08,4.21-2.92,4.18-4.71,6.57-7.64,7.02l-.31.06c-3.09.63-8.27.61-12.45.6h-2.16c-3.85.07-7.01-.16-9.45-.69-1.16-.25-2.16-.57-2.99-.96-.78-.35-1.41-.77-1.9-1.24-.66-.64-1.54-1.81-1.63-4.65.18-2.18,1.62-4.64,3.15-7.25l.49-.83c3.17-5.32,11.18-18.73,17.24-28.33l.07-.12c5.02-8.31,9.2-13.45,16.92-16.81l.1-.04c6.57-2.7,10.54-2.74,18.43-2.82.87-.01,1.78-.01,2.69-.03h1.99c11.5-.09,25.82-.19,35.33,0h.23c3.57-.04,10.55,1.02,11.88,5.48.14.84-.35,2.27-1.13,3.93-.34.72-.73,1.48-1.14,2.25Z"/>
|
||||
</g>
|
||||
</g>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 5.4 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 2.4 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 138 KiB |
@@ -1,37 +0,0 @@
|
||||
name: acp-conformance
|
||||
|
||||
# The agent is driven over stdio by the official ACP TypeScript SDK, and every
|
||||
# message it writes is held to the schema that SDK ships. Not a required check:
|
||||
# it runs when the ACP frontend or the harness changes, and on request.
|
||||
on:
|
||||
pull_request:
|
||||
branches: [main-v2, studio]
|
||||
paths:
|
||||
- "internal/frontend/acp/**"
|
||||
- "tools/acp-conformance/**"
|
||||
- ".github/workflows/acp-conformance.yml"
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: acp-conformance-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
conformance:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7
|
||||
with:
|
||||
go-version-file: go.mod
|
||||
cache: true
|
||||
- uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7
|
||||
with:
|
||||
node-version: "22"
|
||||
- run: make acp-conformance
|
||||
+41
-679
@@ -15,7 +15,7 @@ permissions:
|
||||
|
||||
concurrency:
|
||||
group: ci-${{ github.ref }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || github.ref == 'refs/heads/studio' }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
|
||||
|
||||
jobs:
|
||||
# Cheap path gate for pull requests. PRs confined to docs/release-notes
|
||||
@@ -88,7 +88,7 @@ jobs:
|
||||
with:
|
||||
go-version-file: go.mod
|
||||
cache: true
|
||||
- run: node --test scripts/linux-install-smoke.test.mjs scripts/studio-artifacts.test.mjs scripts/ci-platform-scope.test.mjs scripts/cli-archive-build.test.mjs scripts/cli-archive-contents.test.mjs scripts/release-credits.test.mjs scripts/update-contributors.test.mjs scripts/update-acknowledgments.test.mjs scripts/markdown-refs.test.mjs scripts/studio-release-notes.test.mjs scripts/publish-cli-pointer.test.mjs scripts/studio-release-flags.test.mjs scripts/studio-notes-publish.test.mjs scripts/withdraw-version.test.mjs scripts/brew-cask-guard.test.mjs scripts/feedback-sync.test.mjs npm/publish.test.mjs
|
||||
- run: node --test scripts/linux-install-smoke.test.mjs scripts/studio-artifacts.test.mjs scripts/ci-platform-scope.test.mjs scripts/cli-archive-build.test.mjs scripts/release-credits.test.mjs scripts/update-contributors.test.mjs scripts/update-acknowledgments.test.mjs scripts/markdown-refs.test.mjs scripts/studio-release-notes.test.mjs scripts/publish-cli-pointer.test.mjs scripts/studio-release-flags.test.mjs scripts/studio-notes-publish.test.mjs scripts/withdraw-version.test.mjs scripts/brew-cask-guard.test.mjs scripts/feedback-sync.test.mjs npm/publish.test.mjs
|
||||
|
||||
# The ruleset requires the per-OS check names (test (ubuntu-latest) etc.).
|
||||
# A matrix job skipped at job level reports no per-leg checks at all, so
|
||||
@@ -179,27 +179,9 @@ jobs:
|
||||
REASONIX_RELEASE_CACHE_GUARD: "1"
|
||||
run: go test ./...
|
||||
|
||||
- name: test (Windows composition links)
|
||||
if: env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request'
|
||||
timeout-minutes: 5
|
||||
shell: bash
|
||||
run: |
|
||||
set -o pipefail
|
||||
go test -count=1 -timeout=3m -json -run '^Test(CaptureDirectoryMaterializesContainedLinks|CaptureDirectoryRefusesUnsafeLinks|CaptureHookFileFlattensContainedLink|MaterializeStillRejectsLinkedRetainedFilesAndRoot|WindowsCompositionJunctions|WindowsLiveSkillJunctions|CompositionRetainsLinkedSkillAcrossRestart|CompositionLinkedSkillRejectsLinkedResources|CompositionLinkedExtensionResources|CompositionLinkedHookResources)$' ./internal/state/agentpreset ./internal/ext/skill ./internal/assembly/boot | tee "$RUNNER_TEMP/composition-links.jsonl"
|
||||
jq -se '
|
||||
["TestCaptureDirectoryMaterializesContainedLinks", "TestCaptureDirectoryRefusesUnsafeLinks",
|
||||
"TestCaptureHookFileFlattensContainedLink", "TestMaterializeStillRejectsLinkedRetainedFilesAndRoot",
|
||||
"TestWindowsCompositionJunctions", "TestWindowsLiveSkillJunctions", "TestCompositionRetainsLinkedSkillAcrossRestart",
|
||||
"TestCompositionLinkedSkillRejectsLinkedResources", "TestCompositionLinkedExtensionResources",
|
||||
"TestCompositionLinkedHookResources"] as $required |
|
||||
[.[] | select(.Action == "pass") | .Test] as $passed |
|
||||
all(.[]; .Action != "skip") and (($required - $passed) | length == 0)
|
||||
' "$RUNNER_TEMP/composition-links.jsonl"
|
||||
|
||||
# Pull requests run a Windows smoke suite: the packages that actually
|
||||
# carry *_windows.go code, the startup/checkpoint packages with portable
|
||||
# filesystem contracts, plus cmd/. Isolation/worktree/mutation have a
|
||||
# complete JSON inventory below. The full ./... sweep stays on pushes
|
||||
# filesystem contracts, plus cmd/. The full ./... sweep stays on pushes
|
||||
# to main-v2; the Unix legs always run the full suite. Keep the job-level
|
||||
# budget above normal 7-9 minute runner variance; each package test binary
|
||||
# remains independently bounded by Go's per-package timeout below.
|
||||
@@ -223,543 +205,6 @@ jobs:
|
||||
WINDOWS_SANDBOX_WAIT_MS: "20000"
|
||||
run: go test -count=1 -p 4 -timeout=18m ./internal/runtime/agent/... ./internal/platform/appidentity/... ./internal/assembly/boot/... ./internal/state/checkpoint/... ./internal/frontend/cli/... ./internal/session/control/... ./internal/base/filelock/... ./internal/base/fileutil/... ./internal/ext/hook/... ./internal/state/instruction/... ./internal/ext/mcplaunch/... ./internal/platform/notify/... ./internal/base/proc/... ./internal/platform/remote/... ./internal/platform/repair/... ./internal/safety/sandbox/... ./internal/base/sysproxy/... ./internal/state/workspacelease/... ./cmd/...
|
||||
|
||||
- name: test (Windows published command job owner)
|
||||
if: env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request'
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
evidence="$RUNNER_TEMP/published-job-owner"
|
||||
mkdir -p "$evidence"
|
||||
go test -count=1 -timeout=3m -json -run '^TestPublishedJob' ./internal/base/proc | tee "$evidence/positive.jsonl"
|
||||
node scripts/published-job-evidence.mjs positive "$evidence/positive.jsonl"
|
||||
node scripts/published-job-evidence.mjs overlay "$evidence"
|
||||
set +e
|
||||
go test -count=1 -timeout=3m -json -overlay="$evidence/overlay.json" -run '^TestPublishedJob' ./internal/base/proc > "$evidence/negative.jsonl"
|
||||
old_result=$?
|
||||
set -e
|
||||
test "$old_result" -eq 1
|
||||
node scripts/published-job-evidence.mjs negative "$evidence/negative.jsonl"
|
||||
|
||||
- name: upload Windows published command job owner evidence
|
||||
if: always() && env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request'
|
||||
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
|
||||
with:
|
||||
name: published-job-owner-windows-${{ github.run_attempt }}
|
||||
path: ${{ runner.temp }}/published-job-owner/
|
||||
if-no-files-found: error
|
||||
retention-days: 7
|
||||
|
||||
- name: test (Windows isolation owners)
|
||||
if: env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request'
|
||||
timeout-minutes: 25
|
||||
shell: bash
|
||||
run: |
|
||||
set -o pipefail
|
||||
go test -count=1 -p 4 -timeout=18m -json ./internal/platform/worktree/... ./internal/state/mutation/... | tee "$RUNNER_TEMP/isolation-owners.jsonl" |
|
||||
jq -rj --unbuffered 'select(.Action == "output" and .Test == null) | .Output'
|
||||
jq -se '
|
||||
[.[] | select(.Test == null and .Action == "pass") | .Package] as $passed |
|
||||
[.[] | select(.Test != null and .Action == "pass") | .Test] as $tests |
|
||||
(["reasonix/internal/platform/worktree", "reasonix/internal/state/mutation"] - $passed | length) == 0 and
|
||||
(["TestHeldPathsCoverOnlyTheSameFileAcrossCaseSpellings",
|
||||
"TestLegacyWaitDoesNotConsumeExecutionCapacity"] - $tests | length) == 0
|
||||
' "$RUNNER_TEMP/isolation-owners.jsonl"
|
||||
jq -se '
|
||||
. as $events |
|
||||
{"reasonix/internal/platform/worktree": [
|
||||
"TestWriteChangesRejectsRedirectedDeletion",
|
||||
"TestWriteChangesUnlinksLeafDirectoryLink"
|
||||
]} |
|
||||
to_entries | all(.[]; . as $package |
|
||||
([ $events[] | select(.Package == $package.key and .Test == null and
|
||||
(.Action == "pass" or .Action == "fail" or .Action == "skip")) ] |
|
||||
length == 1 and .[0].Action == "pass") and
|
||||
all(.value[]; . as $test |
|
||||
[ $events[] | select(.Package == $package.key and .Test == $test and
|
||||
(.Action == "pass" or .Action == "fail" or .Action == "skip")) ] |
|
||||
length == 1 and .[0].Action == "pass"))
|
||||
' "$RUNNER_TEMP/isolation-owners.jsonl"
|
||||
|
||||
- name: upload Windows isolation test inventory
|
||||
if: always() && env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request'
|
||||
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
|
||||
with:
|
||||
name: isolation-owners-windows-${{ github.run_attempt }}
|
||||
path: ${{ runner.temp }}/isolation-owners.jsonl
|
||||
if-no-files-found: ignore
|
||||
retention-days: 7
|
||||
|
||||
- name: test (Windows delegated Goal usage scope)
|
||||
if: env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request'
|
||||
shell: bash
|
||||
run: |
|
||||
set -o pipefail
|
||||
go test -count=1 -timeout=3m -json -run '^Test(UsageScope.*)$' ./internal/contract/event ./internal/session/control ./internal/assembly/boot | tee "$RUNNER_TEMP/goal-usage-scope.jsonl"
|
||||
jq -se '
|
||||
["TestUsageScopeKeepsEarlierEventOwner",
|
||||
"TestUsageScopeWithoutCapability" ] as $required |
|
||||
[.[] | select(.Test != null and .Action == "pass") | .Test] as $passed |
|
||||
[.[] | select(.Test == null and .Action == "pass") | .Package] as $packages |
|
||||
all(.[]; .Action != "fail" and .Action != "skip") and
|
||||
(($required - $passed) | length == 0) and
|
||||
((["reasonix/internal/contract/event", "reasonix/internal/session/control", "reasonix/internal/assembly/boot"] - $packages) | length == 0)
|
||||
' "$RUNNER_TEMP/goal-usage-scope.jsonl"
|
||||
|
||||
- name: upload Windows delegated Goal usage scope inventory
|
||||
if: always() && env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request'
|
||||
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
|
||||
with:
|
||||
name: goal-usage-scope-windows-${{ github.run_attempt }}
|
||||
path: ${{ runner.temp }}/goal-usage-scope.jsonl
|
||||
if-no-files-found: error
|
||||
retention-days: 7
|
||||
|
||||
- name: test (Windows Controller routing root budget)
|
||||
if: env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request'
|
||||
timeout-minutes: 5
|
||||
shell: bash
|
||||
run: |
|
||||
set -o pipefail
|
||||
go test -count=1 -timeout=3m -json -run '^Test(SemanticRootUsage)$' ./internal/runtime/capability | tee "$RUNNER_TEMP/controller-routing-budget.jsonl"
|
||||
jq -se '
|
||||
[.[] | select(.Test == null and .Action == "pass") | .Package] as $packages |
|
||||
[.[] | select(.Test != null and .Action == "pass") | .Test] as $tests |
|
||||
all(.[]; .Action != "skip" and .Action != "fail") and
|
||||
(["reasonix/internal/runtime/capability"] - $packages | length) == 0 and
|
||||
([
|
||||
"TestSemanticRootUsage",
|
||||
"TestSemanticRootUsage/billed_error",
|
||||
"TestSemanticRootUsage/cache",
|
||||
"TestSemanticRootUsage/estimated",
|
||||
"TestSemanticRootUsage/invalid_answer",
|
||||
"TestSemanticRootUsage/missing",
|
||||
"TestSemanticRootUsage/nil_sink",
|
||||
"TestSemanticRootUsage/no_observer",
|
||||
"TestSemanticRootUsage/reported"
|
||||
] - $tests | length) == 0
|
||||
' "$RUNNER_TEMP/controller-routing-budget.jsonl"
|
||||
|
||||
- name: upload Windows Controller routing root budget inventory
|
||||
if: always() && env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request'
|
||||
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
|
||||
with:
|
||||
name: controller-routing-budget-windows-${{ github.run_attempt }}
|
||||
path: ${{ runner.temp }}/controller-routing-budget.jsonl
|
||||
if-no-files-found: ignore
|
||||
retention-days: 7
|
||||
|
||||
- name: test (Windows Coordinator root budget)
|
||||
if: env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request'
|
||||
timeout-minutes: 5
|
||||
shell: bash
|
||||
run: |
|
||||
set -o pipefail
|
||||
go test -count=1 -timeout=3m -json -run '^Test(PreparedRootBudget|CoordinatorRootBudget|EffectCoordinatorRoot)' ./internal/runtime/agent ./internal/runtime/coordinator | tee "$RUNNER_TEMP/coordinator-root-budget.jsonl"
|
||||
jq -se '
|
||||
[.[] | select(.Test == null and .Action == "pass") | .Package] as $packages |
|
||||
[.[] | select(.Test != null and .Action == "pass") | .Test] as $tests |
|
||||
all(.[]; .Action != "skip" and .Action != "fail") and
|
||||
(["reasonix/internal/runtime/agent", "reasonix/internal/runtime/coordinator"] - $packages | length) == 0 and
|
||||
(["TestCoordinatorRootBudgetRoutes",
|
||||
"TestCoordinatorRootBudgetRoutes/raw", "TestCoordinatorRootBudgetRoutes/tool_planner",
|
||||
"TestCoordinatorRootBudgetRoutes/failure_fallback", "TestCoordinatorRootBudgetRoutes/plan_only_continuation",
|
||||
"TestCoordinatorRootBudgetRoutes/approval_declined_continuation", "TestCoordinatorRootBudgetRoutes/approval_allowed"
|
||||
] - $tests | length) == 0
|
||||
' "$RUNNER_TEMP/coordinator-root-budget.jsonl"
|
||||
|
||||
- name: upload Windows Coordinator root budget inventory
|
||||
if: always() && env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request'
|
||||
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
|
||||
with:
|
||||
name: coordinator-root-budget-windows-${{ github.run_attempt }}
|
||||
path: ${{ runner.temp }}/coordinator-root-budget.jsonl
|
||||
if-no-files-found: ignore
|
||||
retention-days: 7
|
||||
|
||||
- name: test (Windows resumed Goal planner budget)
|
||||
if: env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request'
|
||||
shell: bash
|
||||
run: |
|
||||
set -o pipefail
|
||||
go test -count=1 -timeout=3m -json -run '^Test(GoalResumeRenewsPlannerBudget|GoalNonBudgetResumeKeepsPlannerBudget|GoalTokenBudgetPausesAndResumes|EffectGoalResumeRenewsPlannerBudget)$' ./internal/session/control ./internal/assembly/boot | tee "$RUNNER_TEMP/goal-planner-resume.jsonl"
|
||||
jq -se '
|
||||
["TestGoalResumeRenewsPlannerBudget",
|
||||
"TestGoalNonBudgetResumeKeepsPlannerBudget",
|
||||
"TestGoalNonBudgetResumeKeepsPlannerBudget/manual_pause",
|
||||
"TestGoalNonBudgetResumeKeepsPlannerBudget/not_resumable",
|
||||
"TestGoalTokenBudgetPausesAndResumes",
|
||||
"TestEffectGoalResumeRenewsPlannerBudget"] as $required |
|
||||
[.[] | select(.Test != null and .Action == "pass") | .Test] as $passed |
|
||||
[.[] | select(.Test == null and .Action == "pass") | .Package] as $packages |
|
||||
all(.[]; .Action != "fail" and .Action != "skip") and
|
||||
(($required - $passed) | length == 0) and
|
||||
((["reasonix/internal/session/control", "reasonix/internal/assembly/boot"] - $packages) | length == 0)
|
||||
' "$RUNNER_TEMP/goal-planner-resume.jsonl"
|
||||
|
||||
- name: upload Windows resumed Goal planner budget inventory
|
||||
if: always() && env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request'
|
||||
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
|
||||
with:
|
||||
name: goal-planner-resume-windows-${{ github.run_attempt }}
|
||||
path: ${{ runner.temp }}/goal-planner-resume.jsonl
|
||||
if-no-files-found: error
|
||||
retention-days: 7
|
||||
|
||||
- name: test (Windows root wall budget)
|
||||
if: env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request'
|
||||
timeout-minutes: 5
|
||||
shell: bash
|
||||
run: |
|
||||
set -o pipefail
|
||||
go test -count=1 -timeout=3m -json -run '^Test(RootWall|EffectRootWall)' ./internal/runtime/agent ./internal/assembly/boot | tee "$RUNNER_TEMP/root-wall-budget.jsonl"
|
||||
jq -se '
|
||||
[.[] | select(.Test == null and .Action == "pass") | .Package] as $packages |
|
||||
[.[] | select(.Test != null and .Action == "pass") | .Test] as $tests |
|
||||
all(.[]; .Action != "skip" and .Action != "fail") and
|
||||
(["reasonix/internal/runtime/agent", "reasonix/internal/assembly/boot"] - $packages | length) == 0 and
|
||||
(["TestRootWallBudgetReachesDelegatedAgents", "TestRootWallBudgetReachesDelegatedAgents/foreground",
|
||||
"TestRootWallBudgetReachesDelegatedAgents/nested", "TestRootWallBudgetReachesDelegatedAgents/siblings",
|
||||
"TestRootWallBudgetReachesDelegatedAgents/context", "TestRootWallBudgetReachesDelegatedAgents/unconfigured",
|
||||
"TestRootWallBudgetReachesDelegatedAgents/explicit_zero", "TestRootWallBudgetContinuationAndReset",
|
||||
"TestRootWallBudgetContinuationAndReset/continuation", "TestRootWallBudgetContinuationAndReset/new_task",
|
||||
"TestRootWallBudgetContinuationAndReset/reset", "TestRootWallBudgetContinuationAndReset/override"
|
||||
] - $tests | length) == 0
|
||||
' "$RUNNER_TEMP/root-wall-budget.jsonl"
|
||||
|
||||
- name: upload Windows root wall budget inventory
|
||||
if: always() && env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request'
|
||||
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
|
||||
with:
|
||||
name: root-wall-budget-windows-${{ github.run_attempt }}
|
||||
path: ${{ runner.temp }}/root-wall-budget.jsonl
|
||||
if-no-files-found: ignore
|
||||
retention-days: 7
|
||||
|
||||
- name: test (Windows root token budget)
|
||||
if: env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request'
|
||||
timeout-minutes: 5
|
||||
shell: bash
|
||||
run: |
|
||||
set -o pipefail
|
||||
go test -count=1 -timeout=3m -json -run '^Test(RootToken|EffectRootToken|BudgetCountsKnownReportedTokens|ReportedToken)' ./internal/runtime/agent | tee "$RUNNER_TEMP/root-token-budget.jsonl"
|
||||
jq -se '
|
||||
[.[] | select(.Test == null and .Action == "pass") | .Package] as $packages |
|
||||
[.[] | select(.Test != null and .Action == "pass") | .Test] as $tests |
|
||||
all(.[]; .Action != "skip" and .Action != "fail") and
|
||||
(["reasonix/internal/runtime/agent"] - $packages | length) == 0 and
|
||||
(["TestRootTokenBudgetIncludesChildProviderUsage", "TestRootTokenBudgetIncludesChildProviderUsage/configured",
|
||||
"TestRootTokenBudgetIncludesChildProviderUsage/context", "TestRootTokenBudgetIncludesChildProviderUsage/unconfigured",
|
||||
"TestRootTokenBudgetIncludesChildProviderUsage/explicit_zero", "TestRootTokenBudgetSurvivesNestedAndConcurrentChildren",
|
||||
"TestRootTokenBudgetSurvivesNestedAndConcurrentChildren/nested", "TestRootTokenBudgetSurvivesNestedAndConcurrentChildren/siblings",
|
||||
"TestRootTokenBudgetSurvivesNestedAndConcurrentChildren/child_override", "TestRootTokenBudgetContinuationAndReset",
|
||||
"TestRootTokenBudgetContinuationAndReset/continuation", "TestRootTokenBudgetContinuationAndReset/new_task",
|
||||
"TestRootTokenBudgetContinuationAndReset/reset" ] - $tests | length) == 0
|
||||
' "$RUNNER_TEMP/root-token-budget.jsonl"
|
||||
jq -se '
|
||||
. as $events |
|
||||
{"reasonix/internal/runtime/agent": [
|
||||
"TestBudgetCountsKnownReportedTokens",
|
||||
"TestBudgetCountsKnownReportedTokens/split",
|
||||
"TestBudgetCountsKnownReportedTokens/total_only",
|
||||
"TestBudgetCountsKnownReportedTokens/partial_split",
|
||||
"TestBudgetCountsKnownReportedTokens/lower_total",
|
||||
"TestBudgetCountsKnownReportedTokens/missing_total",
|
||||
"TestBudgetCountsKnownReportedTokens/negative_split",
|
||||
"TestBudgetCountsKnownReportedTokens/zero",
|
||||
"TestBudgetCountsKnownReportedTokens/negative",
|
||||
"TestReportedTokenCountersSaturate"]} |
|
||||
to_entries | all(.[]; . as $package |
|
||||
([ $events[] | select(.Package == $package.key and .Test == null and
|
||||
(.Action == "pass" or .Action == "fail" or .Action == "skip")) ] |
|
||||
length == 1 and .[0].Action == "pass") and
|
||||
all(.value[]; . as $test |
|
||||
[ $events[] | select(.Package == $package.key and .Test == $test and
|
||||
(.Action == "pass" or .Action == "fail" or .Action == "skip")) ] |
|
||||
length == 1 and .[0].Action == "pass"))
|
||||
' "$RUNNER_TEMP/root-token-budget.jsonl"
|
||||
|
||||
- name: upload Windows root token budget inventory
|
||||
if: always() && env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request'
|
||||
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
|
||||
with:
|
||||
name: root-token-budget-windows-${{ github.run_attempt }}
|
||||
path: ${{ runner.temp }}/root-token-budget.jsonl
|
||||
if-no-files-found: ignore
|
||||
retention-days: 7
|
||||
|
||||
- name: test (Windows budget usage completeness)
|
||||
if: env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request'
|
||||
timeout-minutes: 5
|
||||
shell: bash
|
||||
run: |
|
||||
set -o pipefail
|
||||
go test -count=1 -timeout=3m -json -run '^Test(RequestOnlyUsage|RunBudgetMissingUsage|TaskBudgetKnownCost|RunBudgetExplicitZero|TaskBudgetUsageCompleteness|TaskBudgetCostUses|EffectBudgetCompleteness|TaskBudgetGateIgnoresCostWhenUnpriced)' ./internal/contract/provider ./internal/runtime/agent ./internal/assembly/boot | tee "$RUNNER_TEMP/budget-usage.jsonl"
|
||||
jq -se '
|
||||
[.[] | select(.Test == null and .Action == "pass") | .Package] as $packages |
|
||||
[.[] | select(.Test != null and .Action == "pass") | .Test] as $tests |
|
||||
all(.[]; .Action != "skip" and .Action != "fail") and
|
||||
(["reasonix/internal/contract/provider", "reasonix/internal/runtime/agent", "reasonix/internal/assembly/boot"] - $packages | length) == 0 and
|
||||
(["TestRequestOnlyUsageDoesNotCertifyTokenAccounting", "TestRunBudgetMissingUsageKeepsCostIncomplete",
|
||||
"TestRunBudgetMissingUsageKeepsCostIncomplete/complete", "TestRunBudgetMissingUsageKeepsCostIncomplete/missing_first",
|
||||
"TestRunBudgetMissingUsageKeepsCostIncomplete/missing_last", "TestRunBudgetMissingUsageKeepsCostIncomplete/all_missing",
|
||||
"TestTaskBudgetKnownCostStillLandsAfterMissingUsage", "TestRunBudgetExplicitZeroUsageIsPriced",
|
||||
"TestTaskBudgetUsageCompletenessFollowsItsLifetime", "TestTaskBudgetUsageCompletenessFollowsItsLifetime/continuation",
|
||||
"TestTaskBudgetUsageCompletenessFollowsItsLifetime/new_task", "TestTaskBudgetUsageCompletenessFollowsItsLifetime/explicit_reset",
|
||||
"TestTaskBudgetCostUsesOnlyTheReportedSubtotal", "TestTaskBudgetGateIgnoresCostWhenUnpriced",
|
||||
"TestEffectBudgetCompletenessFromOpenAIStream", "TestEffectBudgetCompletenessFromOpenAIStream/missing_0",
|
||||
"TestEffectBudgetCompletenessFromOpenAIStream/missing_1", "TestEffectBudgetCompletenessFromOpenAIStream/missing_2"] - $tests | length) == 0
|
||||
' "$RUNNER_TEMP/budget-usage.jsonl"
|
||||
|
||||
- name: upload Windows budget usage inventory
|
||||
if: always() && env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request'
|
||||
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
|
||||
with:
|
||||
name: budget-usage-windows-${{ github.run_attempt }}
|
||||
path: ${{ runner.temp }}/budget-usage.jsonl
|
||||
if-no-files-found: ignore
|
||||
retention-days: 7
|
||||
|
||||
|
||||
- name: test (Windows Git storage environment)
|
||||
if: env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request'
|
||||
timeout-minutes: 5
|
||||
shell: bash
|
||||
run: |
|
||||
set -o pipefail
|
||||
go test -count=1 -timeout=3m -json -run '^Test(HostStorageEnvironment|HostObjectStorage|HostExplicitStorageControls|DetachedStorageEnvironment)$' ./internal/platform/gitcmd ./internal/platform/worktree ./internal/assembly/boot | tee "$RUNNER_TEMP/git-storage-environment.jsonl"
|
||||
jq -se '
|
||||
[.[] | select(.Test == null and .Action == "pass") | .Package] as $packages |
|
||||
[.[] | select(.Test != null and .Action == "pass") | .Test] as $tests |
|
||||
all(.[]; .Action != "skip" and .Action != "fail") and
|
||||
(["reasonix/internal/platform/gitcmd", "reasonix/internal/platform/worktree", "reasonix/internal/assembly/boot"] - $packages | length) == 0 and
|
||||
([
|
||||
"TestDetachedStorageEnvironment",
|
||||
"TestDetachedStorageEnvironment/explicit_env",
|
||||
"TestDetachedStorageEnvironment/inherited_env",
|
||||
"TestHostExplicitStorageControls",
|
||||
"TestHostExplicitStorageControls/explicit_object_directory",
|
||||
"TestHostExplicitStorageControls/private_index",
|
||||
"TestHostExplicitStorageControls/repository_declared_alternate",
|
||||
"TestHostObjectStorage",
|
||||
"TestHostObjectStorage/ambient_alternate_not_read",
|
||||
"TestHostObjectStorage/writes_stay_in_repository",
|
||||
"TestHostStorageEnvironment",
|
||||
"TestHostStorageEnvironment/GIT_ALTERNATE_OBJECT_DIRECTORIES",
|
||||
"TestHostStorageEnvironment/GIT_INDEX_FILE",
|
||||
"TestHostStorageEnvironment/GIT_OBJECT_DIRECTORY"
|
||||
] - $tests | length) == 0
|
||||
' "$RUNNER_TEMP/git-storage-environment.jsonl"
|
||||
|
||||
- name: upload Windows Git storage environment inventory
|
||||
if: always() && env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request'
|
||||
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
|
||||
with:
|
||||
name: git-storage-environment-windows-${{ github.run_attempt }}
|
||||
path: ${{ runner.temp }}/git-storage-environment.jsonl
|
||||
if-no-files-found: ignore
|
||||
retention-days: 7
|
||||
|
||||
- name: test (Windows Anthropic interrupted usage)
|
||||
if: env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request'
|
||||
timeout-minutes: 5
|
||||
shell: bash
|
||||
run: |
|
||||
set -o pipefail
|
||||
go test -count=1 -timeout=3m -json -run '^Test(InterruptedUsage|EffectAnthropicPartialUsageRecovery)' ./internal/model/anthropic ./internal/assembly/boot | tee "$RUNNER_TEMP/anthropic-interrupted-usage.jsonl"
|
||||
jq -se '
|
||||
[.[] | select(.Test == null and .Action == "pass") | .Package] as $packages |
|
||||
[.[] | select(.Test != null and .Action == "pass") | .Test] as $tests |
|
||||
all(.[]; .Action != "skip" and .Action != "fail") and
|
||||
(["reasonix/internal/model/anthropic", "reasonix/internal/assembly/boot"] - $packages | length) == 0 and
|
||||
(["TestInterruptedUsageTerminal", "TestInterruptedUsageTerminal/upstream_error",
|
||||
"TestInterruptedUsageTerminal/payload_error", "TestInterruptedUsageTerminal/premature_eof",
|
||||
"TestInterruptedUsageTerminal/decode_error", "TestInterruptedUsageTerminal/reported_zero",
|
||||
"TestInterruptedUsageTerminal/missing_control", "TestInterruptedUsageTerminal/complete_control",
|
||||
"TestInterruptedUsageBoundedConsumer", "TestInterruptedUsageReadErrorAndCancellation",
|
||||
"TestInterruptedUsageReadErrorAndCancellation/read_error", "TestInterruptedUsageReadErrorAndCancellation/cancelled_buffered_reader",
|
||||
"TestInterruptedUsageAbandonedConsumer", "TestInterruptedUsageIdleTimeout",
|
||||
"TestEffectAnthropicPartialUsageRecovery", "TestEffectAnthropicPartialUsageRecovery/upstream_error",
|
||||
"TestEffectAnthropicPartialUsageRecovery/premature_eof", "TestEffectAnthropicPartialUsageRecovery/complete_control"] - $tests | length) == 0
|
||||
' "$RUNNER_TEMP/anthropic-interrupted-usage.jsonl"
|
||||
|
||||
- name: upload Windows Anthropic interrupted usage inventory
|
||||
if: always() && env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request'
|
||||
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
|
||||
with:
|
||||
name: anthropic-interrupted-usage-windows-${{ github.run_attempt }}
|
||||
path: ${{ runner.temp }}/anthropic-interrupted-usage.jsonl
|
||||
if-no-files-found: ignore
|
||||
retention-days: 7
|
||||
|
||||
- name: test (Windows root websearch budget)
|
||||
if: env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request'
|
||||
timeout-minutes: 5
|
||||
shell: bash
|
||||
run: |
|
||||
set -o pipefail
|
||||
go test -count=1 -timeout=3m -json -run '^Test(RootWebSearch|EffectRootWebSearch)' ./internal/tools/websearch | tee "$RUNNER_TEMP/root-websearch-budget.jsonl"
|
||||
jq -se '
|
||||
[.[] | select(.Test == null and .Action == "pass") | .Package] as $packages |
|
||||
[.[] | select(.Test != null and .Action == "pass") | .Test] as $tests |
|
||||
all(.[]; .Action != "skip" and .Action != "fail") and
|
||||
(["reasonix/internal/tools/websearch"] - $packages | length) == 0 and
|
||||
(["TestRootWebSearchUsageTerminal", "TestRootWebSearchUsageTerminal/reported",
|
||||
"TestRootWebSearchUsageTerminal/no_reporter", "TestRootWebSearchUsageTerminal/missing",
|
||||
"TestRootWebSearchUsageTerminal/no_observer", "TestRootWebSearchUsageTerminal/input_refused",
|
||||
"TestRootWebSearchUsageTerminal/no_factory", "TestRootWebSearchUsageTerminal/factory_error",
|
||||
"TestRootWebSearchUsageTerminal/billed_failure", "TestRootWebSearchUsageTerminal/cancel_after_usage",
|
||||
"TestRootWebSearchUsageTerminal/cancel_before_start", "TestRootWebSearchUsageTerminal/stream_error"
|
||||
] - $tests | length) == 0
|
||||
' "$RUNNER_TEMP/root-websearch-budget.jsonl"
|
||||
|
||||
- name: upload Windows root websearch budget inventory
|
||||
if: always() && env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request'
|
||||
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
|
||||
with:
|
||||
name: root-websearch-budget-windows-${{ github.run_attempt }}
|
||||
path: ${{ runner.temp }}/root-websearch-budget.jsonl
|
||||
if-no-files-found: ignore
|
||||
retention-days: 7
|
||||
|
||||
- name: test (Windows explicit skill root budget)
|
||||
if: env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request'
|
||||
timeout-minutes: 5
|
||||
shell: bash
|
||||
run: |
|
||||
set -o pipefail
|
||||
go test -count=1 -timeout=3m -json -run '^(TestProfileRootBudgetLifetime|TestProfileBudgetPreparationUsesTurnAdmission|TestEffectExplicitSkillRootBudget)$' ./internal/session/control | tee "$RUNNER_TEMP/explicit-skill-root-budget.jsonl"
|
||||
jq -se '
|
||||
[.[] | select(.Test == null and .Action == "pass") | .Package] as $packages |
|
||||
[.[] | select(.Test != null and .Action == "pass") | .Test] as $tests |
|
||||
all(.[]; .Action != "skip" and .Action != "fail") and
|
||||
(["reasonix/internal/session/control"] - $packages | length) == 0 and
|
||||
(["TestProfileBudgetPreparationUsesTurnAdmission" ] - $tests | length) == 0
|
||||
' "$RUNNER_TEMP/explicit-skill-root-budget.jsonl"
|
||||
|
||||
- name: upload Windows explicit skill root budget inventory
|
||||
if: always() && env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request'
|
||||
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
|
||||
with:
|
||||
name: explicit-skill-root-budget-windows-${{ github.run_attempt }}
|
||||
path: ${{ runner.temp }}/explicit-skill-root-budget.jsonl
|
||||
if-no-files-found: ignore
|
||||
retention-days: 7
|
||||
|
||||
- name: test (Windows root triage budget)
|
||||
if: env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request'
|
||||
timeout-minutes: 5
|
||||
shell: bash
|
||||
run: |
|
||||
set -o pipefail
|
||||
go test -count=1 -timeout=3m -json -run '^Test(RootTriage|EffectRootTriage)' ./internal/runtime/agent | tee "$RUNNER_TEMP/root-triage-budget.jsonl"
|
||||
jq -se '
|
||||
[.[] | select(.Test == null and .Action == "pass") | .Package] as $packages |
|
||||
[.[] | select(.Test != null and .Action == "pass") | .Test] as $tests |
|
||||
all(.[]; .Action != "skip" and .Action != "fail") and
|
||||
(["reasonix/internal/runtime/agent"] - $packages | length) == 0 and
|
||||
(["TestRootTriageTokenBudget", "TestRootTriageTokenBudget/classifier", "TestRootTriageTokenBudget/screen",
|
||||
"TestRootTriageTokenBudget/billed_failure", "TestRootTriageTokenBudget/missing", "TestRootTriageTokenBudget/disabled",
|
||||
"TestRootTriageTokenBudget/no_sink", "TestRootTriageTokenBudget/counts_without_total",
|
||||
"TestRootTriageOutsideRunDoesNotCreateScope" ] - $tests | length) == 0
|
||||
' "$RUNNER_TEMP/root-triage-budget.jsonl"
|
||||
|
||||
- name: upload Windows root triage budget inventory
|
||||
if: always() && env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request'
|
||||
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
|
||||
with:
|
||||
name: root-triage-budget-windows-${{ github.run_attempt }}
|
||||
path: ${{ runner.temp }}/root-triage-budget.jsonl
|
||||
if-no-files-found: ignore
|
||||
retention-days: 7
|
||||
|
||||
- name: test (Windows local cost quote)
|
||||
if: env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request'
|
||||
shell: bash
|
||||
run: |
|
||||
set -o pipefail
|
||||
go test -count=1 -timeout=3m -json -run '^Test(LocalBudgetUsesOccurrenceQuote|LocalCost.*|UsageQuote.*)$' ./internal/contract/event ./internal/runtime/agent | tee "$RUNNER_TEMP/local-cost-quote.jsonl"
|
||||
jq -se '
|
||||
["TestLocalBudgetUsesOccurrenceQuote", "TestLocalBudgetUsesOccurrenceQuote/peak", "TestLocalBudgetUsesOccurrenceQuote/offpeak",
|
||||
"TestLocalCostKeepsOriginalCurrencyBuckets", "TestLocalCostFixedPointThresholdAndUnknown",
|
||||
"TestUsageQuoteUsesOneOwnerAndOccurrence", "TestUsageQuoteLegacyAndNilSinks"
|
||||
] as $required |
|
||||
[.[] | select(.Test != null and .Action == "pass") | .Test] as $passed |
|
||||
[.[] | select(.Test == null and .Action == "pass") | .Package] as $packages |
|
||||
all(.[]; .Action != "fail" and .Action != "skip") and
|
||||
(($required - $passed) | length == 0) and
|
||||
((["reasonix/internal/contract/event", "reasonix/internal/runtime/agent"] - $packages) | length == 0)
|
||||
' "$RUNNER_TEMP/local-cost-quote.jsonl"
|
||||
|
||||
- name: upload Windows local cost quote inventory
|
||||
if: always() && env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request'
|
||||
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
|
||||
with:
|
||||
name: local-cost-quote-windows-${{ github.run_attempt }}
|
||||
path: ${{ runner.temp }}/local-cost-quote.jsonl
|
||||
if-no-files-found: error
|
||||
retention-days: 7
|
||||
|
||||
- name: test (Windows estimated usage quotes)
|
||||
if: env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request'
|
||||
timeout-minutes: 5
|
||||
shell: bash
|
||||
run: |
|
||||
set -o pipefail
|
||||
go test -count=1 -timeout=3m -json -run '^Test(EstimatedUsage|EffectEstimatedUsage|ToWireUsagePayloadJSON$|StatusExtensionTracksMultipleSessionsAndUsage$|CacheSplitProvenanceIsRecorded$|RecordedPlainCallIsUnchanged$)' ./internal/contract/pricing ./internal/contract/event ./internal/contract/config ./internal/assembly/boot ./internal/contract/eventwire ./internal/frontend/acp ./internal/state/stats | tee "$RUNNER_TEMP/estimated-quote.jsonl"
|
||||
jq -se '
|
||||
[.[] | select(.Test == null and .Action == "pass") | .Package] as $packages |
|
||||
[.[] | select(.Test != null and .Action == "pass") | .Test] as $tests |
|
||||
all(.[]; .Action != "skip" and .Action != "fail") and
|
||||
(["reasonix/internal/contract/pricing", "reasonix/internal/contract/event", "reasonix/internal/contract/config", "reasonix/internal/assembly/boot",
|
||||
"reasonix/internal/contract/eventwire", "reasonix/internal/frontend/acp", "reasonix/internal/state/stats"] - $packages | length) == 0 and
|
||||
(["TestEstimatedUsageQuotePreservesAmountsWithoutCompleteFacts", "TestEstimatedUsageQuotePreservesAmountsWithoutCompleteFacts/CNY",
|
||||
"TestEstimatedUsageQuotePreservesAmountsWithoutCompleteFacts/USD", "TestEstimatedUsageLedgerRetainsPartialCoverage",
|
||||
"TestEstimatedUsageLedgerRetainsPartialCoverage/separate_keys", "TestEstimatedUsageLedgerRetainsPartialCoverage/same_key",
|
||||
"TestEstimatedUsageSavedQuoteReader", "TestEstimatedUsageQuoteSinkCoversTurnAndGuardian",
|
||||
"TestEstimatedUsageQuoteSinkCoversTurnAndGuardian/turn", "TestEstimatedUsageQuoteSinkCoversTurnAndGuardian/guardian",
|
||||
"TestEstimatedUsageConfiguredQuote", "TestEffectEstimatedUsageQuoteAfterStreamRecovery",
|
||||
"TestEffectEstimatedUsageQuoteAfterStreamRecovery/reported_control", "TestEffectEstimatedUsageQuoteAfterStreamRecovery/interrupted",
|
||||
"TestToWireUsagePayloadJSON", "TestStatusExtensionTracksMultipleSessionsAndUsage", "TestCacheSplitProvenanceIsRecorded", "TestRecordedPlainCallIsUnchanged"] - $tests | length) == 0
|
||||
' "$RUNNER_TEMP/estimated-quote.jsonl"
|
||||
|
||||
- name: upload Windows estimated quote inventory
|
||||
if: always() && env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request'
|
||||
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
|
||||
with:
|
||||
name: estimated-quote-windows-${{ github.run_attempt }}
|
||||
path: ${{ runner.temp }}/estimated-quote.jsonl
|
||||
if-no-files-found: ignore
|
||||
retention-days: 7
|
||||
|
||||
- name: test (Windows bounded root budget)
|
||||
if: env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request'
|
||||
timeout-minutes: 5
|
||||
shell: bash
|
||||
run: |
|
||||
set -o pipefail
|
||||
go test -count=1 -timeout=3m -json -run '^Test(AuxiliaryUsageObserver|BoundedUsageObserver|RootBounded|EffectRootBounded)' ./internal/contract/provider ./internal/model/boundedllm ./internal/runtime/agent | tee "$RUNNER_TEMP/bounded-root-budget.jsonl"
|
||||
jq -se '
|
||||
[.[] | select(.Test == null and .Action == "pass") | .Package] as $packages |
|
||||
[.[] | select(.Test != null and .Action == "pass") | .Test] as $tests |
|
||||
all(.[]; .Action != "skip" and .Action != "fail") and
|
||||
(["reasonix/internal/contract/provider", "reasonix/internal/model/boundedllm",
|
||||
"reasonix/internal/runtime/agent"] - $packages | length) == 0 and
|
||||
(["TestAuxiliaryUsageObserverReplacesAndCopies",
|
||||
"TestBoundedUsageObserverOwnsOneTerminal", "TestBoundedUsageObserverOwnsOneTerminal/reported",
|
||||
"TestBoundedUsageObserverOwnsOneTerminal/billed_failure", "TestBoundedUsageObserverOwnsOneTerminal/without_sink",
|
||||
"TestBoundedUsageObserverOwnsOneTerminal/missing", "TestBoundedUsageObserverOwnsOneTerminal/input_refused"
|
||||
] - $tests | length) == 0
|
||||
' "$RUNNER_TEMP/bounded-root-budget.jsonl"
|
||||
|
||||
- name: upload Windows bounded root budget inventory
|
||||
if: always() && env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request'
|
||||
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
|
||||
with:
|
||||
name: bounded-root-budget-windows-${{ github.run_attempt }}
|
||||
path: ${{ runner.temp }}/bounded-root-budget.jsonl
|
||||
if-no-files-found: ignore
|
||||
retention-days: 7
|
||||
|
||||
# The general Windows PR smoke list intentionally omits internal/contract/tool.
|
||||
# Keep the session-temp portability contract covered without widening the
|
||||
# leg to every tool test: two real PowerShell launches must share the
|
||||
@@ -777,6 +222,40 @@ jobs:
|
||||
timeout-minutes: 5
|
||||
run: go test -count=1 -timeout=3m -run '^TestBashPowerShellExecuteDetailedContract$|^TestBashPowerShell51PreflightRejectsAndAndDetailed$|^TestBashPowerShellOutputIsUTF8$|^TestBashPowerShellSurfacesNonZeroExit$|^TestBashPowerShellRejectsChaining$' ./internal/tools/builtin
|
||||
|
||||
# Full push suite uses the same 18m package timeout as Windows smoke and a
|
||||
# 35m step budget above it. -count=1 as at the smoke step. A failure prints
|
||||
# the failed tests' output and the 15 slowest boot tests.
|
||||
- name: test (full)
|
||||
if: env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name != 'pull_request'
|
||||
timeout-minutes: 35
|
||||
env:
|
||||
# Run the prompt-cache prefix-stability guard (TestCacheHit*) in CI: a
|
||||
# regression there silently tanks the cache hit rate the project is
|
||||
# built around.
|
||||
REASONIX_RELEASE_CACHE_GUARD: "1"
|
||||
# Bound sandbox helper children in Windows tests so a failed OS-level
|
||||
# launch cannot pin the Actions step after Go's package timeout fires.
|
||||
WINDOWS_SANDBOX_WAIT_MS: "20000"
|
||||
shell: bash
|
||||
run: |
|
||||
go test -count=1 -p 4 -timeout=18m -json ./... | tee "$RUNNER_TEMP/full.jsonl" |
|
||||
jq -rj --unbuffered 'select(.Action == "output" and .Test == null) | .Output'
|
||||
|
||||
- name: test (full) failure detail
|
||||
if: failure() && env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name != 'pull_request'
|
||||
shell: bash
|
||||
run: |
|
||||
f="$RUNNER_TEMP/full.jsonl"
|
||||
[ -s "$f" ] || exit 0
|
||||
echo "::group::output of failed tests"
|
||||
jq -rn '[inputs | select(.Test != null)] | group_by(.Package + "/" + .Test)
|
||||
| map(select(any(.[]; .Action == "fail")))[]
|
||||
| map(select(.Action == "output") | .Output) | join("")' "$f" || true
|
||||
echo "::endgroup::"
|
||||
echo "slowest internal/assembly/boot tests (seconds, name):"
|
||||
jq -rn '[inputs | select((.Package | endswith("/internal/assembly/boot")) and .Test != null and .Action == "pass")]
|
||||
| sort_by(-.Elapsed)[:15][] | "\(.Elapsed)\t\(.Test)"' "$f" || true
|
||||
|
||||
race:
|
||||
needs: changes
|
||||
if: always() && (github.event_name != 'pull_request' || needs.changes.outputs.code != 'false')
|
||||
@@ -811,130 +290,13 @@ jobs:
|
||||
if: github.event_name == 'pull_request'
|
||||
env:
|
||||
REASONIX_RELEASE_CACHE_GUARD: "1"
|
||||
run: go test -race ./internal/runtime/agent/... ./internal/runtime/delegation/... ./internal/platform/worktree/... ./internal/state/mutation/... ./internal/ext/plugin/... ./internal/tools/jobs/... ./internal/base/proc/... ./internal/safety/sandbox/... ./internal/base/filelock/... ./internal/contract/eventwire/... ./internal/platform/remote/... ./internal/ext/extension/... ./internal/assembly/boot/... ./internal/session/control/... ./internal/contract/tool/...
|
||||
run: go test -race ./internal/runtime/agent/... ./internal/ext/plugin/... ./internal/tools/jobs/... ./internal/base/proc/... ./internal/safety/sandbox/... ./internal/base/filelock/... ./internal/contract/eventwire/... ./internal/platform/remote/... ./internal/ext/extension/... ./internal/assembly/boot/... ./internal/session/control/... ./internal/contract/tool/...
|
||||
|
||||
# Push-only full suites. internal/frontend/serve and internal/assembly/boot run
|
||||
# hundreds of serial tests and set the wall time of one whole-tree `go test`,
|
||||
# so each is cut by test name into shards that run on their own runners. The
|
||||
# -run / -skip patterns partition the names exactly: every test lands in one
|
||||
# shard. `rest` is every other package. Pull requests keep the focused steps
|
||||
# in the `test` and `race` jobs above.
|
||||
test-windows-shards:
|
||||
if: github.event_name != 'pull_request'
|
||||
name: test (windows-latest, ${{ matrix.shard }})
|
||||
runs-on: windows-latest
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- { shard: rest }
|
||||
- { shard: serve-1, pkg: ./internal/frontend/serve, flag: -run, filter: '^Test[A-E]' }
|
||||
- { shard: serve-2, pkg: ./internal/frontend/serve, flag: -run, filter: '^Test[F-R]' }
|
||||
- { shard: serve-3, pkg: ./internal/frontend/serve, flag: -skip, filter: '^Test[A-R]' }
|
||||
- { shard: boot-1, pkg: ./internal/assembly/boot, flag: -skip, filter: '^TestEffect([^0-9A-C]|$)' }
|
||||
- { shard: boot-2, pkg: ./internal/assembly/boot, flag: -run, filter: '^TestEffect[D-O]' }
|
||||
- { shard: boot-3, pkg: ./internal/assembly/boot, flag: -run, filter: '^TestEffect([^0-9A-O]|$)' }
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
|
||||
- uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7
|
||||
with:
|
||||
go-version-file: go.mod
|
||||
cache: true
|
||||
|
||||
- name: test (shard)
|
||||
timeout-minutes: 40
|
||||
env:
|
||||
# Run the prompt-cache prefix-stability guard (TestCacheHit*) in CI: a
|
||||
# regression there silently tanks the cache hit rate the project is
|
||||
# built around.
|
||||
REASONIX_RELEASE_CACHE_GUARD: "1"
|
||||
# Bound sandbox helper children in Windows tests so a failed OS-level
|
||||
# launch cannot pin the Actions step after Go's package timeout fires.
|
||||
WINDOWS_SANDBOX_WAIT_MS: "20000"
|
||||
SHARD: ${{ matrix.shard }}
|
||||
PKG: ${{ matrix.pkg }}
|
||||
FLAG: ${{ matrix.flag }}
|
||||
FILTER: ${{ matrix.filter }}
|
||||
shell: bash
|
||||
run: |
|
||||
if [ "$SHARD" = rest ]; then
|
||||
mapfile -t pkgs < <(go list ./... | tr -d '\r' | grep -v -E '/internal/(frontend/serve|assembly/boot)$')
|
||||
else
|
||||
pkgs=("$PKG")
|
||||
fi
|
||||
filter=()
|
||||
if [ -n "$FILTER" ]; then filter=("$FLAG" "$FILTER"); fi
|
||||
go test -count=1 -p 4 -timeout=36m -json "${filter[@]}" "${pkgs[@]}" | tee "$RUNNER_TEMP/shard.jsonl" |
|
||||
jq -rj --unbuffered 'select(.Action == "output" and .Test == null) | .Output'
|
||||
|
||||
- name: test (shard) failure detail
|
||||
if: failure()
|
||||
shell: bash
|
||||
run: |
|
||||
f="$RUNNER_TEMP/shard.jsonl"
|
||||
[ -s "$f" ] || exit 0
|
||||
echo "::group::output of failed tests"
|
||||
jq -rn '[inputs | select(.Test != null)] | group_by(.Package + "/" + .Test)
|
||||
| map(select(any(.[]; .Action == "fail")))[]
|
||||
| map(select(.Action == "output") | .Output) | join("")' "$f" || true
|
||||
echo "::endgroup::"
|
||||
echo "slowest tests (seconds, package, name):"
|
||||
jq -rn '[inputs | select(.Test != null and .Action == "pass")]
|
||||
| sort_by(-.Elapsed)[:15][] | "\(.Elapsed) \(.Package) \(.Test)"' "$f" || true
|
||||
|
||||
race-shards:
|
||||
if: github.event_name != 'pull_request'
|
||||
name: race (${{ matrix.shard }})
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- { shard: rest }
|
||||
- { shard: serve-1, pkg: ./internal/frontend/serve, flag: -run, filter: '^Test[A-E]' }
|
||||
- { shard: serve-2, pkg: ./internal/frontend/serve, flag: -run, filter: '^Test[F-R]' }
|
||||
- { shard: serve-3, pkg: ./internal/frontend/serve, flag: -skip, filter: '^Test[A-R]' }
|
||||
- { shard: boot-1, pkg: ./internal/assembly/boot, flag: -skip, filter: '^TestEffect([^0-9A-C]|$)' }
|
||||
- { shard: boot-2, pkg: ./internal/assembly/boot, flag: -run, filter: '^TestEffect[D-O]' }
|
||||
- { shard: boot-3, pkg: ./internal/assembly/boot, flag: -run, filter: '^TestEffect([^0-9A-O]|$)' }
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
|
||||
- uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7
|
||||
with:
|
||||
go-version-file: go.mod
|
||||
cache: true
|
||||
|
||||
- name: Install and verify Linux sandbox backend
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y bubblewrap
|
||||
if sysctl -n kernel.unprivileged_userns_clone >/dev/null 2>&1; then
|
||||
sudo sysctl -w kernel.unprivileged_userns_clone=1
|
||||
fi
|
||||
if sysctl -n kernel.apparmor_restrict_unprivileged_userns >/dev/null 2>&1; then
|
||||
sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0
|
||||
fi
|
||||
bwrap --ro-bind / / --dev /dev --proc /proc -- true || echo "::warning::bubblewrap is installed but this runner denies user namespaces; unavailable-backend tests will run fail-closed"
|
||||
|
||||
- name: test -race (shard)
|
||||
timeout-minutes: 30
|
||||
- name: test -race (full)
|
||||
if: github.event_name != 'pull_request'
|
||||
env:
|
||||
REASONIX_RELEASE_CACHE_GUARD: "1"
|
||||
SHARD: ${{ matrix.shard }}
|
||||
PKG: ${{ matrix.pkg }}
|
||||
FLAG: ${{ matrix.flag }}
|
||||
FILTER: ${{ matrix.filter }}
|
||||
run: |
|
||||
if [ "$SHARD" = rest ]; then
|
||||
mapfile -t pkgs < <(go list ./... | grep -v -E '/internal/(frontend/serve|assembly/boot)$')
|
||||
else
|
||||
pkgs=("$PKG")
|
||||
fi
|
||||
filter=()
|
||||
if [ -n "$FILTER" ]; then filter=("$FLAG" "$FILTER"); fi
|
||||
go test -race -timeout=25m "${filter[@]}" "${pkgs[@]}"
|
||||
run: go test -race ./...
|
||||
|
||||
# sdk/go is a nested stdlib-only module invisible to root `go test ./...`.
|
||||
# Its DTOs are generated from internal/ext/extension/protocol, and the
|
||||
|
||||
@@ -43,20 +43,20 @@ jobs:
|
||||
if: matrix.branch != ''
|
||||
run: echo "sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT"
|
||||
- name: Initialize CodeQL
|
||||
uses: github/codeql-action/init@33d7b85da1a04171e211befe2c739303b9a4c066 # v4.39.3
|
||||
uses: github/codeql-action/init@24c54180a607b1449ed407dd24f251e4e9147c8d # v4.38.3
|
||||
with:
|
||||
languages: ${{ matrix.language }}
|
||||
build-mode: ${{ matrix.language == 'go' && 'autobuild' || 'none' }}
|
||||
- name: Perform CodeQL Analysis
|
||||
if: matrix.branch == ''
|
||||
uses: github/codeql-action/analyze@33d7b85da1a04171e211befe2c739303b9a4c066 # v4.39.3
|
||||
uses: github/codeql-action/analyze@24c54180a607b1449ed407dd24f251e4e9147c8d # v4.38.3
|
||||
with:
|
||||
category: "/language:${{ matrix.language }}"
|
||||
# Results from a checked-out line are filed under that line's ref, not the
|
||||
# default branch the schedule happened to run from.
|
||||
- name: Perform CodeQL Analysis (${{ matrix.branch }})
|
||||
if: matrix.branch != ''
|
||||
uses: github/codeql-action/analyze@33d7b85da1a04171e211befe2c739303b9a4c066 # v4.39.3
|
||||
uses: github/codeql-action/analyze@24c54180a607b1449ed407dd24f251e4e9147c8d # v4.38.3
|
||||
with:
|
||||
category: "/language:${{ matrix.language }}"
|
||||
ref: refs/heads/${{ matrix.branch }}
|
||||
|
||||
@@ -20,7 +20,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
- uses: pnpm/action-setup@73e46992e05b19abd31f2f3c20e0e7370d762115 # v6.1.1
|
||||
- uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413 # v6.1.0
|
||||
with:
|
||||
version: 10
|
||||
run_install: false
|
||||
|
||||
@@ -20,7 +20,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
- uses: pnpm/action-setup@73e46992e05b19abd31f2f3c20e0e7370d762115 # v6.1.1
|
||||
- uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413 # v6.1.0
|
||||
with:
|
||||
version: 10
|
||||
run_install: false
|
||||
|
||||
@@ -23,7 +23,7 @@ jobs:
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
|
||||
- uses: pnpm/action-setup@73e46992e05b19abd31f2f3c20e0e7370d762115 # v6.1.1
|
||||
- uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413 # v6.1.0
|
||||
with:
|
||||
version: 10
|
||||
run_install: false
|
||||
|
||||
@@ -51,7 +51,7 @@ jobs:
|
||||
go-version-file: go.mod
|
||||
cache: true
|
||||
|
||||
- uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7
|
||||
- uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7
|
||||
with:
|
||||
python-version: '3.12'
|
||||
|
||||
|
||||
@@ -1,32 +0,0 @@
|
||||
name: Deploy website
|
||||
|
||||
# GitHub requires workflow_dispatch to be registered on the default branch.
|
||||
# The production implementation is the same workflow path on website.
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
release_version:
|
||||
description: "Optional product release version associated with this publication"
|
||||
required: false
|
||||
default: ""
|
||||
type: string
|
||||
release_request:
|
||||
description: "Optional release workflow correlation ID"
|
||||
required: false
|
||||
default: ""
|
||||
type: string
|
||||
studio_version:
|
||||
description: "Studio version that must be present in the generated changelog"
|
||||
required: false
|
||||
default: ""
|
||||
type: string
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
require-website:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- run: |
|
||||
echo "::error::Deploy the website with --ref website; production is restricted to that branch."
|
||||
exit 1
|
||||
@@ -1,27 +0,0 @@
|
||||
name: PR author gate
|
||||
|
||||
# Pull requests are limited to maintainers and the logins in .github/pr-authors.txt.
|
||||
# This runs in the base repository's context, checks out the base branch only, and
|
||||
# never executes pull-request code; the allowlist it reads is the base branch's.
|
||||
on:
|
||||
pull_request_target:
|
||||
types: [opened]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: write
|
||||
|
||||
concurrency:
|
||||
group: pr-author-gate-${{ github.event.pull_request.number }}
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
gate:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
- run: node scripts/pr-author-gate.mjs
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
@@ -200,7 +200,7 @@ jobs:
|
||||
cache: true
|
||||
cache-dependency-path: go.sum
|
||||
|
||||
- uses: pnpm/action-setup@73e46992e05b19abd31f2f3c20e0e7370d762115 # v6.1.1
|
||||
- uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413 # v6.1.0
|
||||
with:
|
||||
version: 10
|
||||
run_install: false
|
||||
@@ -389,14 +389,6 @@ jobs:
|
||||
version="${{ needs.resolve.outputs.version }}"
|
||||
bash scripts/linux-install-smoke.sh "dist/ReasonixStudio-linux-$arch.deb" "$arch" "${version#v}"
|
||||
|
||||
- name: Verify the AppImage
|
||||
if: runner.os == 'Linux'
|
||||
shell: bash
|
||||
run: |
|
||||
platform="${{ matrix.platform }}"
|
||||
arch="${platform#linux/}"
|
||||
bash scripts/verify-linux-appimage.sh "dist/ReasonixStudio-linux-$arch.AppImage" "$arch"
|
||||
|
||||
# With signing on, this leg only hands the bundle on: the signing jobs
|
||||
# run nothing but the signer, and windows-package packages in between.
|
||||
- name: Upload the unsigned Windows bundle for signing
|
||||
@@ -608,7 +600,7 @@ jobs:
|
||||
ref: ${{ needs.resolve.outputs.sha }}
|
||||
persist-credentials: false
|
||||
|
||||
- uses: pnpm/action-setup@73e46992e05b19abd31f2f3c20e0e7370d762115 # v6.1.1
|
||||
- uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413 # v6.1.0
|
||||
with:
|
||||
version: 10
|
||||
run_install: false
|
||||
@@ -908,31 +900,12 @@ jobs:
|
||||
go-version-file: go.mod
|
||||
cache: true
|
||||
|
||||
# The page every archive carries beside the executable, built once here
|
||||
# rather than once per target: serve.FindPage looks for
|
||||
# frontend-next/dist next to the binary.
|
||||
- uses: pnpm/action-setup@73e46992e05b19abd31f2f3c20e0e7370d762115 # v6.1.1
|
||||
with:
|
||||
version: 10
|
||||
run_install: false
|
||||
|
||||
- uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7
|
||||
with:
|
||||
node-version: "24"
|
||||
cache: pnpm
|
||||
cache-dependency-path: desktop/frontend-next/pnpm-lock.yaml
|
||||
|
||||
- name: Build the page
|
||||
working-directory: desktop/frontend-next
|
||||
run: pnpm install --frozen-lockfile && pnpm build
|
||||
|
||||
# Every target the release line claims to support, from one runner: the
|
||||
# CLI is CGO-free, so none of them needs a toolchain of its own.
|
||||
- name: Cross-compile and archive
|
||||
env:
|
||||
VERSION: ${{ needs.resolve.outputs.version }}
|
||||
SHA: ${{ needs.resolve.outputs.sha }}
|
||||
PAGE_DIR: desktop/frontend-next/dist
|
||||
run: |
|
||||
set -euo pipefail
|
||||
mkdir -p dist
|
||||
@@ -944,7 +917,11 @@ jobs:
|
||||
CGO_ENABLED=0 GOOS="$os" GOARCH="$arch" go build -trimpath \
|
||||
-ldflags "-s -w -X main.version=$VERSION -X main.gitCommit=${SHA:0:12} -X main.buildTimeUTC=$built -X $pkg.linkedVersion=$VERSION -X $pkg.linkedRevision=$SHA" \
|
||||
-o "reasonix$ext" ./cmd/reasonix
|
||||
scripts/package-cli-archive.sh "$os" "$arch" "reasonix$ext" "$PAGE_DIR" dist
|
||||
if [ "$os" = windows ]; then
|
||||
zip -qj "dist/reasonix-$os-$arch.zip" "reasonix$ext"
|
||||
else
|
||||
tar czf "dist/reasonix-$os-$arch.tar.gz" "reasonix$ext"
|
||||
fi
|
||||
rm -f "reasonix$ext"
|
||||
done
|
||||
# The digest list the remote verifies against. Resolved on the
|
||||
@@ -956,7 +933,13 @@ jobs:
|
||||
# A release that published the archives without the digest list is worse
|
||||
# than one that published neither: the routes resolve SHA256SUMS first.
|
||||
- name: Check the set is complete
|
||||
run: python3 scripts/check-cli-archives.py dist
|
||||
run: |
|
||||
set -euo pipefail
|
||||
for want in reasonix-darwin-amd64.tar.gz reasonix-darwin-arm64.tar.gz \
|
||||
reasonix-linux-amd64.tar.gz reasonix-linux-arm64.tar.gz \
|
||||
reasonix-windows-amd64.zip reasonix-windows-arm64.zip SHA256SUMS; do
|
||||
[ -f "dist/$want" ] || { echo "::error::no dist/$want"; exit 1; }
|
||||
done
|
||||
|
||||
- uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
|
||||
with:
|
||||
@@ -1207,7 +1190,7 @@ jobs:
|
||||
fi
|
||||
{
|
||||
printf '%s\n\n' '---'
|
||||
printf "**macOS** 已用 Developer ID 签名并公证,打开时不需要清除隔离属性。%s**Linux** Debian 系从 \`.deb\` 安装,Fedora、RHEL、openSUSE 等从 \`.rpm\` 安装(\`sudo dnf install ./ReasonixStudio-linux-amd64.rpm\`)。x86_64 用文件名带 amd64 的包,aarch64(统信 UOS、麒麟等)用带 arm64 的包。也提供 \`.AppImage\`(只提供下载,不会自更新;需要 FUSE 2,或用 \`--appimage-extract-and-run\` 启动)。\n\n" "$windows"
|
||||
printf "**macOS** 已用 Developer ID 签名并公证,打开时不需要清除隔离属性。%s**Linux** Debian 系从 \`.deb\` 安装,Fedora、RHEL、openSUSE 等从 \`.rpm\` 安装(\`sudo dnf install ./ReasonixStudio-linux-amd64.rpm\`)。x86_64 用文件名带 amd64 的包,aarch64(统信 UOS、麒麟等)用带 arm64 的包。\n\n" "$windows"
|
||||
printf "%s\n" "三个平台都能自更新:Linux 的 \`.deb\` 走自己的包(\`.rpm\` 的自更新尚未提供,请手动下载新包),Windows 运行下一个安装器,macOS 替换自身 bundle。"
|
||||
} >> "$notes"
|
||||
assets=()
|
||||
@@ -1302,25 +1285,6 @@ jobs:
|
||||
--cache-control "public, max-age=300, stale-if-error=86400"
|
||||
echo "Studio catalog -> $VERSION"
|
||||
|
||||
website:
|
||||
name: publish the Studio website changelog
|
||||
needs: [resolve, publish]
|
||||
if: github.repository == 'esengine/DeepSeek-Reasonix' && needs.resolve.outputs.candidate != 'true'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
permissions:
|
||||
contents: read
|
||||
actions: write
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
with:
|
||||
ref: ${{ github.workflow_sha }}
|
||||
- name: Deploy and verify the website changelog
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
VERSION: ${{ needs.resolve.outputs.version }}
|
||||
run: bash scripts/publish-studio-changelog.sh
|
||||
|
||||
# A CLI channel that moves while 1.x still publishes would have two lines
|
||||
# writing it, so every CLI job waits for this check. It runs no repository code.
|
||||
cli-gate:
|
||||
|
||||
@@ -41,7 +41,7 @@ jobs:
|
||||
cache: true
|
||||
cache-dependency-path: go.sum
|
||||
|
||||
- uses: pnpm/action-setup@73e46992e05b19abd31f2f3c20e0e7370d762115 # v6.1.1
|
||||
- uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413 # v6.1.0
|
||||
with:
|
||||
version: 10
|
||||
run_install: false
|
||||
|
||||
@@ -16,7 +16,7 @@ permissions:
|
||||
|
||||
concurrency:
|
||||
group: studio-${{ github.ref }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || github.ref == 'refs/heads/studio' }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
|
||||
|
||||
jobs:
|
||||
frontend:
|
||||
@@ -28,7 +28,7 @@ jobs:
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
|
||||
- uses: pnpm/action-setup@73e46992e05b19abd31f2f3c20e0e7370d762115 # v6.1.1
|
||||
- uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413 # v6.1.0
|
||||
with:
|
||||
version: 10
|
||||
run_install: false
|
||||
@@ -87,7 +87,7 @@ jobs:
|
||||
browser-guards:
|
||||
name: browser guards
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 25
|
||||
timeout-minutes: 15
|
||||
defaults:
|
||||
run:
|
||||
working-directory: desktop/frontend-next
|
||||
@@ -108,7 +108,7 @@ jobs:
|
||||
fi
|
||||
|
||||
- if: steps.spa.outputs.run == 'true'
|
||||
uses: pnpm/action-setup@73e46992e05b19abd31f2f3c20e0e7370d762115 # v6.1.1
|
||||
uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413 # v6.1.0
|
||||
with:
|
||||
version: 10
|
||||
run_install: false
|
||||
@@ -143,170 +143,30 @@ jobs:
|
||||
node perf/hoverradius.mjs
|
||||
node perf/jump-ends.mjs
|
||||
node perf/modelrows.mjs
|
||||
node perf/model-menu-badge.mjs
|
||||
node perf/explorersize.mjs
|
||||
node perf/git-panel.mjs
|
||||
node perf/session-move-to.mjs
|
||||
node perf/shell-refused.mjs
|
||||
node perf/external-origin.mjs
|
||||
node perf/rail-search-hold.mjs
|
||||
node perf/rail-toggle-panes.mjs
|
||||
node perf/session-move.mjs
|
||||
node perf/session-ref.mjs
|
||||
node perf/browser-full.mjs
|
||||
node perf/mod-ui.mjs
|
||||
node perf/codeeditor-size.mjs
|
||||
node perf/composer-reflow.mjs
|
||||
node perf/composer-toolbar-row.mjs
|
||||
node perf/control-icons.mjs
|
||||
node perf/nav-selection.mjs
|
||||
node perf/action-icons.mjs
|
||||
node perf/rowhit.mjs
|
||||
node perf/prefs-wide-centered.mjs
|
||||
node perf/settings-fullscreen.mjs
|
||||
node perf/settings-page-column.mjs
|
||||
node perf/sidebar-tools.mjs
|
||||
node perf/select-alignment.mjs
|
||||
node perf/provider-fields.mjs
|
||||
node perf/thought-clock.mjs
|
||||
node perf/tab-pip.mjs
|
||||
node perf/syntax-tokens.mjs
|
||||
node perf/tab-close-inside.mjs
|
||||
node perf/plan-chip.mjs
|
||||
node perf/flow-feather.mjs
|
||||
node perf/context-card-hover.mjs
|
||||
node perf/sessquick.mjs
|
||||
node perf/sessionrows.mjs
|
||||
node perf/session-row-marker.mjs
|
||||
node perf/session-selected-row.mjs
|
||||
node perf/wheel.mjs
|
||||
node perf/branch-tooltip.mjs
|
||||
node perf/branch-alignment.mjs
|
||||
node perf/branch-menu.mjs
|
||||
node perf/image-preview.mjs
|
||||
node perf/user-images.mjs
|
||||
node perf/toast-anchor.mjs
|
||||
node perf/addfolder-rest.mjs
|
||||
node perf/scroll-jump.mjs
|
||||
node perf/add-project-selects.mjs
|
||||
node perf/project-header.mjs
|
||||
node perf/setup-defer.mjs
|
||||
node perf/search-toggle.mjs
|
||||
node perf/narrow.mjs
|
||||
node perf/settings-rows.mjs
|
||||
node perf/narrow-steps.mjs
|
||||
node perf/job-names.mjs
|
||||
node perf/mcp-actions.mjs
|
||||
node perf/web-access.mjs
|
||||
node perf/peers-settings.mjs
|
||||
node perf/output-cap.mjs
|
||||
node perf/tool-output-clip.mjs
|
||||
node perf/agent-run-tab.mjs
|
||||
node perf/changes-cap.mjs
|
||||
node perf/session-menu-fit.mjs
|
||||
node perf/rail-rows-typography.mjs
|
||||
node perf/settings-nav-fit.mjs
|
||||
node perf/rail-rows-align.mjs
|
||||
node perf/progress-tail.mjs
|
||||
node perf/trust-wait.mjs
|
||||
node perf/nest-nodes.mjs
|
||||
node perf/provider-ports.mjs
|
||||
node perf/peer-threads.mjs
|
||||
node perf/text-size.mjs
|
||||
node perf/fs-scale.mjs
|
||||
node perf/composer-feel.mjs
|
||||
node perf/side-panel.mjs
|
||||
node perf/card-shell.mjs
|
||||
node perf/line-cards.mjs
|
||||
node perf/detail-panel.mjs
|
||||
node perf/brand-lockup.mjs
|
||||
node perf/notice-line.mjs
|
||||
node perf/brought-line.mjs
|
||||
node perf/modal-focus.mjs
|
||||
node perf/focus-ring.mjs
|
||||
node perf/narrow-forms.mjs
|
||||
node perf/i18n-rendered.mjs
|
||||
node perf/narrow-rail.mjs
|
||||
node perf/selfcheck.mjs branch-tooltip
|
||||
|
||||
- name: Keep UI control previews
|
||||
if: always() && steps.spa.outputs.run == 'true'
|
||||
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
|
||||
with:
|
||||
name: studio-ui-control-previews
|
||||
path: |
|
||||
desktop/frontend-next/perf/shots/control-icons-*.png
|
||||
desktop/frontend-next/perf/shots/nav-selection-*.png
|
||||
desktop/frontend-next/perf/shots/settings-full-*.png
|
||||
desktop/frontend-next/perf/shots/sidebar-tools-*.png
|
||||
desktop/frontend-next/perf/shots/select-alignment-*.png
|
||||
desktop/frontend-next/perf/shots/provider-fields-*.png
|
||||
desktop/frontend-next/perf/shots/thought-clock-*.png
|
||||
retention-days: 7
|
||||
|
||||
# What a hosted page does over the relay, end to end: the production bundle,
|
||||
# the real Studio host and kernel, a real Chromium; only the accounts API and
|
||||
# the relay are stand-ins (tools/relayfake). Event-transport changes are
|
||||
# judged here, because a unit test of one side cannot see the other.
|
||||
relay-harness:
|
||||
name: relay harness
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Transport changed?
|
||||
id: scope
|
||||
env:
|
||||
BASE_REF: ${{ github.base_ref }}
|
||||
run: |
|
||||
paths="desktop/frontend-next internal/frontend/serve internal/platform/remotecloud internal/platform/account tools/relayfake cmd/reasonix-studio-host"
|
||||
if [ "$GITHUB_EVENT_NAME" = "pull_request" ] &&
|
||||
git diff --quiet "origin/$BASE_REF...HEAD" -- $paths; then
|
||||
echo "run=false" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "run=true" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
- if: steps.scope.outputs.run == 'true'
|
||||
uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7
|
||||
with:
|
||||
go-version-file: go.mod
|
||||
cache: true
|
||||
|
||||
- if: steps.scope.outputs.run == 'true'
|
||||
uses: pnpm/action-setup@73e46992e05b19abd31f2f3c20e0e7370d762115 # v6.1.1
|
||||
with:
|
||||
version: 10
|
||||
run_install: false
|
||||
|
||||
- if: steps.scope.outputs.run == 'true'
|
||||
uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7
|
||||
with:
|
||||
node-version: "24"
|
||||
cache: pnpm
|
||||
cache-dependency-path: desktop/frontend-next/pnpm-lock.yaml
|
||||
|
||||
- if: steps.scope.outputs.run == 'true'
|
||||
name: Build the host, the stand-ins and the page
|
||||
run: |
|
||||
go build -o desktop/frontend-next/relay/bin/studio-host ./cmd/reasonix-studio-host
|
||||
go build -o desktop/frontend-next/relay/bin/relayfake ./tools/relayfake
|
||||
cd desktop/frontend-next
|
||||
pnpm install --frozen-lockfile
|
||||
pnpm exec vite build --outDir dist-relay --emptyOutDir
|
||||
pnpm exec playwright install --with-deps chromium
|
||||
|
||||
- if: steps.scope.outputs.run == 'true'
|
||||
name: Scenarios (hosted page over the relay; local page on a flapping network)
|
||||
working-directory: desktop/frontend-next
|
||||
env:
|
||||
RELAY_HOST_BIN: ${{ github.workspace }}/desktop/frontend-next/relay/bin/studio-host
|
||||
RELAY_FAKE_BIN: ${{ github.workspace }}/desktop/frontend-next/relay/bin/relayfake
|
||||
RELAY_DIST: ${{ github.workspace }}/desktop/frontend-next/dist-relay
|
||||
run: |
|
||||
node relay/live.mjs
|
||||
node relay/storm.mjs
|
||||
|
||||
# A pull request with no platform-relevant change (scripts/ci-platform-scope.sh)
|
||||
# moves the Windows and macOS legs onto an ubuntu runner with every step
|
||||
# skipped, so the check names still report without a scarce runner. A push to
|
||||
@@ -412,7 +272,7 @@ jobs:
|
||||
arm64=false
|
||||
platform=$(git diff -z --name-only --no-renames "$BASE_SHA" HEAD | bash scripts/ci-platform-scope.sh)
|
||||
if git diff --name-only --no-renames "$BASE_SHA" HEAD |
|
||||
grep -qE '^(desktop/electron/|desktop/packaging/|desktop/cmd/update-helper/|desktop/cmd/studio-manifest/|desktop/go\.(mod|sum)$|cmd/reasonix-studio-host/|internal/platform/(update|appupdate)/|go\.(mod|sum)$|scripts/(linux-install-smoke|verify-linux-package|verify-linux-appimage|verify-studio-artifacts)\.sh$|scripts/(windows-signing-lib|verify-windows-package|verify-windows-authenticode|install-smoke)\.ps1$|\.github/workflows/(studio|release-studio)\.yml$)'; then
|
||||
grep -qE '^(desktop/electron/|desktop/packaging/|desktop/cmd/update-helper/|desktop/cmd/studio-manifest/|desktop/go\.(mod|sum)$|cmd/reasonix-studio-host/|internal/platform/(update|appupdate)/|go\.(mod|sum)$|scripts/(linux-install-smoke|verify-linux-package|verify-studio-artifacts)\.sh$|scripts/(windows-signing-lib|verify-windows-package|verify-windows-authenticode|install-smoke)\.ps1$|\.github/workflows/(studio|release-studio)\.yml$)'; then
|
||||
arm64=true
|
||||
fi
|
||||
fi
|
||||
@@ -459,7 +319,7 @@ jobs:
|
||||
cache: true
|
||||
cache-dependency-path: desktop/go.sum
|
||||
|
||||
- uses: pnpm/action-setup@73e46992e05b19abd31f2f3c20e0e7370d762115 # v6.1.1
|
||||
- uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413 # v6.1.0
|
||||
with:
|
||||
version: 10
|
||||
run_install: false
|
||||
@@ -656,10 +516,6 @@ jobs:
|
||||
if: runner.os == 'Linux'
|
||||
run: bash scripts/linux-install-smoke.sh "dist/ReasonixStudio-linux-${{ matrix.arch }}.deb" "${{ matrix.arch }}" 0.0.0-ci
|
||||
|
||||
- name: The AppImage carries its architecture, the kernel and the page
|
||||
if: runner.os == 'Linux'
|
||||
run: bash scripts/verify-linux-appimage.sh "dist/ReasonixStudio-linux-${{ matrix.arch }}.AppImage" "${{ matrix.arch }}"
|
||||
|
||||
# Recorded, not asserted: the real bundle layout, so later work argues
|
||||
# from what electron-builder produced rather than from its documentation.
|
||||
- name: Record the macOS bundle layout
|
||||
|
||||
@@ -95,4 +95,3 @@ scripts/official-theme-art/__pycache__/
|
||||
# running, which on Windows is every rebuild while Studio is open.
|
||||
*.exe
|
||||
*.exe~
|
||||
/relayfake
|
||||
@@ -117,22 +117,6 @@ go test ./internal/runtime/agent/ -v # verbose, one package
|
||||
go test ./internal/tools/builtin/ -run TestGrep # one test
|
||||
```
|
||||
|
||||
`testenv.IsolateUserState` installs a disposable user home and config-edit lock
|
||||
scope. Default-path tests can clear `REASONIX_HOME` without losing the test
|
||||
registry, while explicitly isolated Reasonix homes retain their own locks.
|
||||
|
||||
`REASONIX_TEST_HOME` carries the scope to helper subprocesses. A helper that
|
||||
skips the isolation guard must install the inherited home with
|
||||
`testscope.InstallHome` before running tests.
|
||||
|
||||
The environment variable alone cannot activate a test scope. Production
|
||||
executables still use the OS account's home for shared config locks.
|
||||
|
||||
The guard removes its scratch home on normal completion. Scratch homes are
|
||||
created under the original home, so an interrupted run can still leave
|
||||
`.reasonix-test-home-*` directories; this is separate from persistent
|
||||
config-edit locks in the real `.reasonix/locks` directory.
|
||||
|
||||
### Code style
|
||||
|
||||
- `gofmt` is enforced by CI — format before committing
|
||||
|
||||
@@ -9,7 +9,7 @@ GOEXE := $(shell go env GOEXE)
|
||||
# One pin for the Makefile and the CI lint job; see .github/workflows/ci.yml.
|
||||
GOLANGCI_VERSION := $(shell cat .golangci-version)
|
||||
|
||||
.PHONY: build vet fmt lint lint-go lint-install lint-cross lint-update coverage-gate coverage-gate-update pricecheck acp-conformance check test studio-test sdk-test sdk-test-race hooks cross clean studio
|
||||
.PHONY: build vet fmt lint lint-go lint-install lint-cross lint-update coverage-gate coverage-gate-update pricecheck check test studio-test sdk-test sdk-test-race hooks cross clean studio
|
||||
|
||||
build:
|
||||
CGO_ENABLED=0 go build -ldflags "$(LDFLAGS)" -o bin/reasonix$(GOEXE) ./cmd/reasonix
|
||||
@@ -87,13 +87,6 @@ lint-cross:
|
||||
test: studio-test sdk-test
|
||||
go test ./...
|
||||
|
||||
# Drives `reasonix acp` with the official ACP TypeScript SDK and holds every
|
||||
# message it writes to the schema that SDK ships. Needs node and network for
|
||||
# `npm ci`, which runs without install scripts against a lockfile.
|
||||
acp-conformance: build
|
||||
cd tools/acp-conformance && npm ci --ignore-scripts --no-audit --no-fund
|
||||
cd tools/acp-conformance && REASONIX_BIN=$(CURDIR)/bin/reasonix$(GOEXE) npm test
|
||||
|
||||
# The shell is a nested module, so the root `test` target never reaches it.
|
||||
studio-test:
|
||||
cd desktop && go test ./...
|
||||
|
||||
@@ -2,9 +2,6 @@
|
||||
<img src="docs/logo-ghost-wave-effect.svg" alt="Reasonix" width="360"/>
|
||||
</p>
|
||||
|
||||
> [!WARNING]
|
||||
> **Reasonix is no longer maintained as of 2026-11-12.** The final release is v2.61.0; there will be no further releases, fixes or issue triage. The project is MIT-licensed and forks are welcome. See the [announcement](https://github.com/esengine/DeepSeek-Reasonix/issues/15109).
|
||||
|
||||
<p align="center">
|
||||
<strong>English</strong>
|
||||
·
|
||||
@@ -19,6 +16,10 @@
|
||||
<a href="./docs/SPEC.md">Spec</a>
|
||||
·
|
||||
<a href="https://esengine.github.io/DeepSeek-Reasonix/">Website</a>
|
||||
·
|
||||
<strong><a href="https://discord.gg/XF78rEME2D">Discord</a></strong>
|
||||
·
|
||||
<strong><a href="https://qm.qq.com/q/i59b0z2R8s">QQ</a></strong>
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
@@ -30,6 +31,9 @@
|
||||
<a href="https://github.com/esengine/DeepSeek-Reasonix/stargazers"><img src="https://img.shields.io/github/stars/esengine/DeepSeek-Reasonix.svg?style=flat-square&color=dbab09&labelColor=161b22&logo=github&logoColor=white" alt="GitHub stars"/></a>
|
||||
<a href="https://atomgit.com/esengine/DeepSeek-Reasonix"><img src="https://atomgit.com/esengine/DeepSeek-Reasonix/star/badge.svg" alt="AtomGit stars"/></a>
|
||||
<a href="https://github.com/esengine/DeepSeek-Reasonix/graphs/contributors"><img src="https://img.shields.io/github/contributors/esengine/DeepSeek-Reasonix.svg?style=flat-square&color=bc8cff&labelColor=161b22&logo=github&logoColor=white" alt="contributors"/></a>
|
||||
<a href="https://github.com/esengine/DeepSeek-Reasonix/discussions"><img src="https://img.shields.io/github/discussions/esengine/DeepSeek-Reasonix.svg?style=flat-square&color=58a6ff&labelColor=161b22&logo=github&logoColor=white" alt="Discussions"/></a>
|
||||
<a href="https://discord.gg/XF78rEME2D"><img src="https://img.shields.io/badge/discord-join-5865F2.svg?style=flat-square&labelColor=161b22&logo=discord&logoColor=white" alt="Discord"/></a>
|
||||
<a href="https://qm.qq.com/q/i59b0z2R8s"><img src="https://img.shields.io/badge/QQ%20group-1093562660-12B7F5.svg?style=flat-square&labelColor=161b22&logo=tencentqq&logoColor=white" alt="QQ group 1093562660"/></a>
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
@@ -44,8 +48,19 @@
|
||||
<p align="center">It reads your project, edits files and runs commands and tests. Have it ask before each step, or work on its own within limits you set. Files its edit tools changed can be rewound. New here? Start with the Studio desktop app.</p>
|
||||
<p align="center">Maintained by <strong>Huahui Yu</strong>.</p>
|
||||
|
||||
> [!IMPORTANT]
|
||||
> **Community · 加入社区** — bilingual Discord for setup help (`#help` / `#求助`), workflow showcases, and feature ideas. → **<https://discord.gg/XF78rEME2D>**
|
||||
>
|
||||
> QQ group **DeepSeek-Reasonix官方群** (`1093562660`) → **<https://qm.qq.com/q/i59b0z2R8s>** ([QR code](./docs/assets/qq-group.svg))
|
||||
|
||||
> Douyin: **做游戏的小鱼** (Douyin ID `22703872788`) · [QR code](./docs/assets/douyin.png)
|
||||
|
||||
<br/>
|
||||
|
||||
## ❤️Sponsor
|
||||
|
||||
<table><tr><td width="180" align="center" valign="middle"><a href="https://www.rightapi.ai/register?aff=e083bf73" target="_blank" rel="sponsored noopener noreferrer"><img src="./.github/sponsor/rightcode.png" alt="Right Code" width="64"></a></td><td valign="middle">Thanks to Right Code for sponsoring this project! Right Code reliably provides routing services for models such as Claude Code, Codex, and Gemini, with both pay-as-you-go and monthly subscription billing options available. Invoices are available upon top-up, and enterprise and team users can receive dedicated one-on-one support. Register via <a href="https://www.rightapi.ai/register?aff=e083bf73" target="_blank" rel="sponsored noopener noreferrer">this link</a>.</td></tr><tr><td width="180" align="center" valign="middle"><a href="https://www.packyapi.ai/register?aff=rIbQ" target="_blank" rel="sponsored noopener noreferrer"><picture><source media="(prefers-color-scheme: dark)" srcset="./.github/sponsor/packycode-dark.svg"><img src="./.github/sponsor/packycode-light.svg" alt="PackyCode" width="80"></picture></a></td><td valign="middle">Thanks to PackyCode for sponsoring this project! Access leading AI models through PackyCode with one API endpoint and one API key, with automatic failover and dedicated high-speed routes for Codex and Claude Code. Register via <a href="https://www.packyapi.ai/register?aff=rIbQ" target="_blank" rel="sponsored noopener noreferrer">this link</a>.</td></tr></table>
|
||||
|
||||
## What it is
|
||||
|
||||
Reasonix is an open-source AI coding assistant. The program runs on your computer; the AI model is a service you configure, which receives what it reads. You describe a task in plain language; it reads code, edits files and runs commands and tests inside the project folder you choose, and shows you each step.
|
||||
@@ -78,16 +93,6 @@ Prefer a terminal? Download the `reasonix` command-line tool, run `reasonix setu
|
||||
|
||||
All four are the same program underneath. What follows is for advanced users and contributors.
|
||||
|
||||
## Use it from DeepSeek Harness
|
||||
|
||||
DeepSeek Harness (`dsh`) can hand a task to Reasonix as a subagent over ACP. This repository publishes a plugin index for it:
|
||||
|
||||
1. Run `reasonix setup` on the machine that runs `dsh`. In the DSH Web UI turn on Custom plugin markets (Plugins, More) and add this index in the Discover tab: `https://raw.githubusercontent.com/esengine/DeepSeek-Reasonix/studio/integrations/dsh-plugin/market-index.json`
|
||||
2. Install "Reasonix subagent" from it.
|
||||
3. Open the plugin's page and turn its switch on. While it is off the model has no `reasonix` tool and may run `reasonix -p` in its own shell, which looks like a delegation and is not one.
|
||||
|
||||
Setup details, the permission policy and limits are in [Using Reasonix from DeepSeek Harness](./docs/DSH.md).
|
||||
|
||||
## Versions
|
||||
|
||||
Reasonix ships on two lines. See the [version roadmap announcement](https://github.com/esengine/DeepSeek-Reasonix/discussions/10748) for the reasoning behind the split.
|
||||
@@ -146,7 +151,7 @@ pre-releases):
|
||||
| --- | --- | --- |
|
||||
| macOS | `.dmg` or `.zip` | Apple Silicon (`arm64`) / Intel (`amd64`) |
|
||||
| Windows | Installer `-installer.exe` or portable `.zip` | x64 |
|
||||
| Linux | `.deb`, `.rpm`, or `.AppImage` (download only) | x64 (`amd64`) / ARM64 (`arm64`, aarch64) |
|
||||
| Linux | `.deb` or `.rpm` | x64 (`amd64`) / ARM64 (`arm64`, aarch64) |
|
||||
|
||||
Every package ships with a `.minisig` signature, and the release carries
|
||||
`SHA256SUMS`. Once installed, Studio updates itself in place.
|
||||
@@ -235,14 +240,12 @@ For advanced CLI usage and configuration, see the **[CLI reference](./docs/CLI.m
|
||||
## Documentation
|
||||
|
||||
- **Getting started:** [Guide](./docs/GUIDE.md) · [CLI reference](./docs/CLI.md) ·
|
||||
[Configuration paths](./docs/CONFIG_PATHS.md) · [ACP editor integration](./docs/ACP.md) ·
|
||||
[Use from DeepSeek Harness](./docs/DSH.md)
|
||||
[Configuration paths](./docs/CONFIG_PATHS.md) · [ACP editor integration](./docs/ACP.md)
|
||||
- **Features & troubleshooting:** [Subagent profiles](./docs/SUBAGENT_PROFILES.md) ·
|
||||
[Context Engine v2](./docs/SESSION_MEMORY_RETRIEVAL.md) ·
|
||||
[Capability diagnostics](./docs/CAPABILITY_DIAGNOSTICS.md) ·
|
||||
[Recovery and updates](./docs/RECOVERY.md) ·
|
||||
[Checkpoints & rewind](./docs/CHECKPOINTS.md)
|
||||
- **Agent presets:** [Session Agent compositions](./docs/AGENT_COMPOSITIONS.md)
|
||||
- **Engineering & migration:** [Spec](./docs/SPEC.md) ·
|
||||
[Task contracts & pause policy](./docs/TASK_CONTRACT.md) ·
|
||||
[Tool contract](./docs/TOOL_CONTRACT.md) · [Moving from 1.x to 2.x](./docs/MIGRATING.md) ·
|
||||
@@ -273,11 +276,11 @@ A small list of folks whose work has shaped Reasonix the most — the current to
|
||||
<!-- reasonix-top-contributors:start -->
|
||||
| Contributor | Contributor | Contributor | Contributor |
|
||||
| --- | --- | --- | --- |
|
||||
| [**SivanCola**](https://github.com/SivanCola) | [**KHG420**](https://github.com/KHG420) | [**ttmouse**](https://github.com/ttmouse) | [**chuiwenwei**](https://github.com/chuiwenwei) |
|
||||
| [**lifu963**](https://github.com/lifu963) | [**XTLine**](https://github.com/XTLine) | [**HUQIANTAO**](https://github.com/HUQIANTAO) | [**xxoingr**](https://github.com/xxoingr) |
|
||||
| [**Harbor404**](https://github.com/Harbor404) | [**GTC2080**](https://github.com/GTC2080) | [**mchenziyi**](https://github.com/mchenziyi) | [**Li-Charles-One**](https://github.com/Li-Charles-One) |
|
||||
| [**light-front-theory**](https://github.com/light-front-theory) | [**imcreazy**](https://github.com/imcreazy) | [**eghrhegpe**](https://github.com/eghrhegpe) | [**noreply**](https://github.com/noreply) |
|
||||
| [**par73e**](https://github.com/par73e) | [**Bernardxu123**](https://github.com/Bernardxu123) | [**lancy69**](https://github.com/lancy69) | [**Guan0923**](https://github.com/Guan0923) |
|
||||
| [**SivanCola**](https://github.com/SivanCola) | [**KHG420**](https://github.com/KHG420) | [**ttmouse**](https://github.com/ttmouse) | [**lifu963**](https://github.com/lifu963) |
|
||||
| [**HUQIANTAO**](https://github.com/HUQIANTAO) | [**Harbor404**](https://github.com/Harbor404) | [**GTC2080**](https://github.com/GTC2080) | [**mchenziyi**](https://github.com/mchenziyi) |
|
||||
| [**Li-Charles-One**](https://github.com/Li-Charles-One) | [**light-front-theory**](https://github.com/light-front-theory) | [**XTLine**](https://github.com/XTLine) | [**eghrhegpe**](https://github.com/eghrhegpe) |
|
||||
| [**noreply**](https://github.com/noreply) | [**chuiwenwei**](https://github.com/chuiwenwei) | [**xxoingr**](https://github.com/xxoingr) | [**Bernardxu123**](https://github.com/Bernardxu123) |
|
||||
| [**HaoyueQin**](https://github.com/HaoyueQin) | [**BuGlessRB**](https://github.com/BuGlessRB) | [**CVEngineer66**](https://github.com/CVEngineer66) | [**JesonChou**](https://github.com/JesonChou) |
|
||||
<!-- reasonix-top-contributors:end -->
|
||||
|
||||
Special thanks to [**Bernardxu123**](https://github.com/Bernardxu123) for designing the project logo and intro video.
|
||||
@@ -297,3 +300,19 @@ Special thanks to [**Bernardxu123**](https://github.com/Bernardxu123) for design
|
||||
<br/>
|
||||
<sub>Built by the community at <a href="https://github.com/esengine/DeepSeek-Reasonix/graphs/contributors">esengine/DeepSeek-Reasonix</a></sub>
|
||||
</p>
|
||||
|
||||
---
|
||||
|
||||
<p align="center"><sub><strong>Support this project</strong></sub></p>
|
||||
|
||||
If Reasonix has been useful and you'd like to say thanks, you can. It stays a
|
||||
coffee, not a contract — donations don't buy feature priority or change how
|
||||
issues get triaged.
|
||||
|
||||
- **International** — PayPal: [paypal.me/yuhuahui](https://paypal.me/yuhuahui)
|
||||
- **国内** — 微信支付 / 支付宝(扫码)
|
||||
|
||||
<p align="center">
|
||||
<img src=".github/sponsor/wechat-pay.jpg" alt="WeChat Pay QR code" width="180"/>
|
||||
<img src=".github/sponsor/alipay.jpg" alt="Alipay QR code" width="180"/>
|
||||
</p>
|
||||
+40
-21
@@ -2,9 +2,6 @@
|
||||
<img src="docs/logo-ghost-wave-effect.svg" alt="Reasonix" width="360"/>
|
||||
</p>
|
||||
|
||||
> [!WARNING]
|
||||
> **Reasonix 已于 2026-11-12 起停止维护。** 最后一个版本为 v2.61.0,此后不再发布新版本、修复缺陷或处理 Issue。项目采用 MIT 协议,欢迎 fork。详见[公告](https://github.com/esengine/DeepSeek-Reasonix/issues/15109)。
|
||||
|
||||
<p align="center">
|
||||
<a href="./README.md">English</a>
|
||||
·
|
||||
@@ -19,6 +16,10 @@
|
||||
<a href="./docs/SPEC.md">规格</a>
|
||||
·
|
||||
<a href="https://esengine.github.io/DeepSeek-Reasonix/">官方网站</a>
|
||||
·
|
||||
<strong><a href="https://discord.gg/XF78rEME2D">Discord</a></strong>
|
||||
·
|
||||
<strong><a href="https://qm.qq.com/q/i59b0z2R8s">QQ</a></strong>
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
@@ -30,6 +31,9 @@
|
||||
<a href="https://github.com/esengine/DeepSeek-Reasonix/stargazers"><img src="https://img.shields.io/github/stars/esengine/DeepSeek-Reasonix.svg?style=flat-square&color=dbab09&labelColor=161b22&logo=github&logoColor=white" alt="GitHub stars"/></a>
|
||||
<a href="https://atomgit.com/esengine/DeepSeek-Reasonix"><img src="https://atomgit.com/esengine/DeepSeek-Reasonix/star/badge.svg" alt="AtomGit stars"/></a>
|
||||
<a href="https://github.com/esengine/DeepSeek-Reasonix/graphs/contributors"><img src="https://img.shields.io/github/contributors/esengine/DeepSeek-Reasonix.svg?style=flat-square&color=bc8cff&labelColor=161b22&logo=github&logoColor=white" alt="contributors"/></a>
|
||||
<a href="https://github.com/esengine/DeepSeek-Reasonix/discussions"><img src="https://img.shields.io/github/discussions/esengine/DeepSeek-Reasonix.svg?style=flat-square&color=58a6ff&labelColor=161b22&logo=github&logoColor=white" alt="Discussions"/></a>
|
||||
<a href="https://discord.gg/XF78rEME2D"><img src="https://img.shields.io/badge/discord-join-5865F2.svg?style=flat-square&labelColor=161b22&logo=discord&logoColor=white" alt="Discord"/></a>
|
||||
<a href="https://qm.qq.com/q/i59b0z2R8s"><img src="https://img.shields.io/badge/QQ%20group-1093562660-12B7F5.svg?style=flat-square&labelColor=161b22&logo=tencentqq&logoColor=white" alt="QQ group 1093562660"/></a>
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
@@ -44,6 +48,17 @@
|
||||
<p align="center">它会读你的项目、改文件、跑命令和测试。每一步先问你,或在你划定的范围内自己做,由你选;它用编辑工具改过的文件可以退回去。第一次用,从桌面应用 Studio 开始。</p>
|
||||
<p align="center">维护者:<strong>Huahui Yu</strong>。</p>
|
||||
|
||||
> [!IMPORTANT]
|
||||
> **加入社区 · Community** — 双语 Discord,提供安装答疑(`#help` / `#求助`)、工作流展示与功能想法。→ **<https://discord.gg/XF78rEME2D>**
|
||||
>
|
||||
> QQ 群 **DeepSeek-Reasonix官方群**(群号 `1093562660`)→ **<https://qm.qq.com/q/i59b0z2R8s>**([二维码](./docs/assets/qq-group.svg))
|
||||
|
||||
> 抖音:**做游戏的小鱼**(抖音号 `22703872788`)· [二维码](./docs/assets/douyin.png)
|
||||
|
||||
## ❤️赞助商
|
||||
|
||||
<table><tr><td width="180" align="center" valign="middle"><a href="https://www.rightapi.ai/register?aff=e083bf73" target="_blank" rel="sponsored noopener noreferrer"><img src="./.github/sponsor/rightcode.png" alt="Right Code" width="64"></a></td><td valign="middle">感谢 Right Code 赞助了本项目!Right Code 稳定提供 Claude Code、Codex、Gemini 等模型的中转服务,并可选按量、包月两种计费模式。充值即可开票,企业、团队用户一对一对接。通过<a href="https://www.rightapi.ai/register?aff=e083bf73" target="_blank" rel="sponsored noopener noreferrer">此链接</a>注册。</td></tr><tr><td width="180" align="center" valign="middle"><a href="https://www.packyapi.ai/register?aff=rIbQ" target="_blank" rel="sponsored noopener noreferrer"><picture><source media="(prefers-color-scheme: dark)" srcset="./.github/sponsor/packycode-dark.svg"><img src="./.github/sponsor/packycode-light.svg" alt="PackyCode" width="80"></picture></a></td><td valign="middle">感谢 PackyCode 赞助了本项目!PackyCode 是一家稳定、高效的 API 中转服务商,统一域名、统一密钥、智能容灾切换。通过<a href="https://www.packyapi.ai/register?aff=rIbQ" target="_blank" rel="sponsored noopener noreferrer">此链接</a>注册。</td></tr></table>
|
||||
|
||||
## 它是什么
|
||||
|
||||
Reasonix 是一个开源的 AI 编程助手。程序跑在你的电脑上;AI 模型是你自己配置的服务,它读到的内容会发给这个服务。你用大白话描述任务,它在你选的项目文件夹里读代码、改文件、跑命令和测试,每一步都摆给你看。
|
||||
@@ -76,16 +91,6 @@ Reasonix 是一个开源的 AI 编程助手。程序跑在你的电脑上;AI
|
||||
|
||||
四种用法背后是同一套程序。下面的内容写给进阶用户和贡献者。
|
||||
|
||||
## 在 DeepSeek Harness 里使用
|
||||
|
||||
DeepSeek Harness(`dsh`)可以通过 ACP 把任务交给 Reasonix 作为子智能体去做。本仓库为此发布了一个插件索引:
|
||||
|
||||
1. 在运行 `dsh` 的机器上先运行 `reasonix setup`。在 DSH 网页界面打开“自定义插件市场”(插件,更多),再到 Discover 页添加这个索引:`https://raw.githubusercontent.com/esengine/DeepSeek-Reasonix/studio/integrations/dsh-plugin/market-index.json`
|
||||
2. 从中安装“Reasonix subagent”。
|
||||
3. 打开这个插件的页面,把它的开关打开。关着的时候模型没有 `reasonix` 工具,可能改在自己的 shell 里直接运行 `reasonix -p`,看起来像委派,其实不是。
|
||||
|
||||
配置细节、权限策略和限制见 [在 DeepSeek Harness 中使用 Reasonix](./docs/DSH.md)。
|
||||
|
||||
## 版本
|
||||
|
||||
Reasonix 分为两条版本线,调整的原因见[版本路线公告](https://github.com/esengine/DeepSeek-Reasonix/discussions/10748)。
|
||||
@@ -134,7 +139,7 @@ Reasonix 分为两条版本线,调整的原因见[版本路线公告](https://
|
||||
| --- | --- | --- |
|
||||
| macOS | `.dmg` 或 `.zip` | Apple Silicon(`arm64`)/ Intel(`amd64`) |
|
||||
| Windows | 安装器 `-installer.exe` 或便携 `.zip` | x64 |
|
||||
| Linux | `.deb`、`.rpm` 或 `.AppImage`(只提供下载) | x64(`amd64`)/ ARM64(`arm64`,aarch64) |
|
||||
| Linux | `.deb` 或 `.rpm` | x64(`amd64`)/ ARM64(`arm64`,aarch64) |
|
||||
|
||||
每个安装包都附带 `.minisig` 签名,发布页另有 `SHA256SUMS`。安装后 Studio 会在
|
||||
应用内自行更新。
|
||||
@@ -216,13 +221,12 @@ CLI 进阶用法和详细配置见 **[CLI 命令参考](./docs/CLI.zh-CN.md)**
|
||||
|
||||
- **开始使用:** [指南](./docs/GUIDE.zh-CN.md) ·
|
||||
[CLI 命令参考](./docs/CLI.zh-CN.md) · [配置路径](./docs/CONFIG_PATHS.md) ·
|
||||
[ACP 编辑器接入](./docs/ACP.md) · [在 DeepSeek Harness 中使用](./docs/DSH.md)
|
||||
[ACP 编辑器接入](./docs/ACP.md)
|
||||
- **功能与排障:** [子智能体 Profile](./docs/SUBAGENT_PROFILES.md) ·
|
||||
[Context Engine v2](./docs/SESSION_MEMORY_RETRIEVAL.md) ·
|
||||
[能力诊断](./docs/CAPABILITY_DIAGNOSTICS.md) ·
|
||||
[恢复与安全模式](./docs/RECOVERY.md) ·
|
||||
[Checkpoints 与 rewind](./docs/CHECKPOINTS.md)
|
||||
- **Agent 预设:** [会话级组合](./docs/AGENT_COMPOSITIONS.zh.md)
|
||||
- **工程与迁移:** [规格](./docs/SPEC.md) ·
|
||||
[任务合约与暂停策略](./docs/TASK_CONTRACT.md) ·
|
||||
[工具合约](./docs/TOOL_CONTRACT.md) ·
|
||||
@@ -254,11 +258,11 @@ CLI 进阶用法和详细配置见 **[CLI 命令参考](./docs/CLI.zh-CN.md)**
|
||||
<!-- reasonix-top-contributors:start -->
|
||||
| Contributor | Contributor | Contributor | Contributor |
|
||||
| --- | --- | --- | --- |
|
||||
| [**SivanCola**](https://github.com/SivanCola) | [**KHG420**](https://github.com/KHG420) | [**ttmouse**](https://github.com/ttmouse) | [**chuiwenwei**](https://github.com/chuiwenwei) |
|
||||
| [**lifu963**](https://github.com/lifu963) | [**XTLine**](https://github.com/XTLine) | [**HUQIANTAO**](https://github.com/HUQIANTAO) | [**xxoingr**](https://github.com/xxoingr) |
|
||||
| [**Harbor404**](https://github.com/Harbor404) | [**GTC2080**](https://github.com/GTC2080) | [**mchenziyi**](https://github.com/mchenziyi) | [**Li-Charles-One**](https://github.com/Li-Charles-One) |
|
||||
| [**light-front-theory**](https://github.com/light-front-theory) | [**imcreazy**](https://github.com/imcreazy) | [**eghrhegpe**](https://github.com/eghrhegpe) | [**noreply**](https://github.com/noreply) |
|
||||
| [**par73e**](https://github.com/par73e) | [**Bernardxu123**](https://github.com/Bernardxu123) | [**lancy69**](https://github.com/lancy69) | [**Guan0923**](https://github.com/Guan0923) |
|
||||
| [**SivanCola**](https://github.com/SivanCola) | [**KHG420**](https://github.com/KHG420) | [**ttmouse**](https://github.com/ttmouse) | [**lifu963**](https://github.com/lifu963) |
|
||||
| [**HUQIANTAO**](https://github.com/HUQIANTAO) | [**Harbor404**](https://github.com/Harbor404) | [**GTC2080**](https://github.com/GTC2080) | [**mchenziyi**](https://github.com/mchenziyi) |
|
||||
| [**Li-Charles-One**](https://github.com/Li-Charles-One) | [**light-front-theory**](https://github.com/light-front-theory) | [**XTLine**](https://github.com/XTLine) | [**eghrhegpe**](https://github.com/eghrhegpe) |
|
||||
| [**noreply**](https://github.com/noreply) | [**chuiwenwei**](https://github.com/chuiwenwei) | [**xxoingr**](https://github.com/xxoingr) | [**Bernardxu123**](https://github.com/Bernardxu123) |
|
||||
| [**HaoyueQin**](https://github.com/HaoyueQin) | [**BuGlessRB**](https://github.com/BuGlessRB) | [**CVEngineer66**](https://github.com/CVEngineer66) | [**JesonChou**](https://github.com/JesonChou) |
|
||||
<!-- reasonix-top-contributors:end -->
|
||||
|
||||
特别感谢 [**Bernardxu123**](https://github.com/Bernardxu123) 设计的项目 logo和开场视频。
|
||||
@@ -278,3 +282,18 @@ CLI 进阶用法和详细配置见 **[CLI 命令参考](./docs/CLI.zh-CN.md)**
|
||||
<br/>
|
||||
<sub>由 <a href="https://github.com/esengine/DeepSeek-Reasonix/graphs/contributors">esengine/DeepSeek-Reasonix</a> 社区共建</sub>
|
||||
</p>
|
||||
|
||||
---
|
||||
|
||||
<p align="center"><sub><strong>支持本项目</strong></sub></p>
|
||||
|
||||
如果 Reasonix 帮你省了时间或 token,欢迎请杯咖啡。捐助不会换来 feature
|
||||
优先级,也不会影响 issue 的处理顺序——就是「谢谢」。
|
||||
|
||||
- **国内** — 微信支付 / 支付宝(扫下方二维码)
|
||||
- **海外** — PayPal: [paypal.me/yuhuahui](https://paypal.me/yuhuahui)
|
||||
|
||||
<p align="center">
|
||||
<img src=".github/sponsor/wechat-pay.jpg" alt="微信支付收款码" width="180"/>
|
||||
<img src=".github/sponsor/alipay.jpg" alt="支付宝收款码" width="180"/>
|
||||
</p>
|
||||
+5
-18
@@ -373,10 +373,6 @@ out of how a path is spelled, which cannot tell `internal/auth` from
|
||||
- sensitive: internal/safety/egress/**
|
||||
- sensitive: internal/platform/browser/**
|
||||
- sensitive: internal/platform/computer/**
|
||||
- sensitive: internal/platform/gitpush/**
|
||||
- sensitive: internal/session/control/git_push.go
|
||||
- sensitive: internal/frontend/serve/git_push.go
|
||||
- sensitive: internal/frontend/serve/host_routes.go
|
||||
|
||||
One declaration, two effects: the same list is also the coverage gate's subject.
|
||||
`make coverage-gate` holds each path to the coverage it already has, because
|
||||
@@ -413,13 +409,8 @@ moving a number here must not move the promise.
|
||||
- Symlinks are recorded by `Lstat`, so a write through one lands outside what
|
||||
the walk compares. Confining the resolved target is the sandbox's job.
|
||||
- `internal/safety/sandbox` confines writes to the workspace, configured extras, temp
|
||||
and toolchain caches on macOS and Linux.
|
||||
- On Windows a user-enabled write-restricted token bounds the *creation* of files
|
||||
(create, append, overwrite, `mkdir`, HKCU writes, global git config, writes through links).
|
||||
- There, deleting or renaming away an existing file is not bounded, and with the
|
||||
setting off nothing the host enforces bounds `Writable`.
|
||||
- The caches a command may write can be poisoned for the user's later unsandboxed
|
||||
builds, on every platform.
|
||||
and toolchain caches — on macOS and Linux. Windows has no OS-level bash
|
||||
sandbox, so there nothing the host enforces bounds `Writable`.
|
||||
- The governed host authorities are a *named* set — ssh-agent, docker, podman.
|
||||
DBus, Wayland, X11, gpg-agent, keyrings and arbitrary sockets stay reachable,
|
||||
so the claim is "these endpoints are governed", never "host IPC is isolated".
|
||||
@@ -465,13 +456,9 @@ stop a connect. Possession is not authority: the key can be unreadable while an
|
||||
agent still signs with it.
|
||||
|
||||
A protection claim exists only where a backend has a demonstrated enforcement
|
||||
primitive for that dimension. Unsupported dimensions stay explicitly unsupported,
|
||||
because a claim that outruns its enforcement is worse than an absent one: it is
|
||||
believed.
|
||||
|
||||
Windows enforces Integrity on file creation only, when the user turns it on.
|
||||
Deletion and rename, Confidentiality and Authority (the named pipes of ssh-agent
|
||||
and docker, COM and RPC services) are not enforced there.
|
||||
primitive for that dimension. Unsupported dimensions stay explicitly unsupported
|
||||
— Windows currently enforces none of the three — because a claim that outruns
|
||||
its enforcement is worse than an absent one: it is believed.
|
||||
|
||||
**Known unmeasured surface — macOS host-mediated delegation:** Apple Events and
|
||||
Mach-service lookup are not denied by the current allow-default profile. Prior
|
||||
|
||||
+4
-64
@@ -1,6 +1,6 @@
|
||||
# Reasonix Benchmarks
|
||||
|
||||
Six harnesses live under `benchmarks/`; `cmd/e2ebench` also exposes a
|
||||
Five harnesses live under `benchmarks/`; `cmd/e2ebench` also exposes a
|
||||
SWE-bench Verified mode:
|
||||
|
||||
- `e2e/` — the committed end-to-end task suite, driven by
|
||||
@@ -19,10 +19,6 @@ SWE-bench Verified mode:
|
||||
- `fanout-width/` — a shape-instructed fleet whose members share nothing, for
|
||||
pricing what running them at the same time is worth. See
|
||||
[Pricing the run graph](#pricing-the-run-graph) below.
|
||||
- `realrepo/` — long tasks over a pinned snapshot of this repository, for
|
||||
measuring changes that cut model round trips (13–31 requests per task on
|
||||
`deepseek-flash`). See
|
||||
[its README](realrepo/README.md).
|
||||
|
||||
## Directory layout
|
||||
|
||||
@@ -420,7 +416,7 @@ own outcome).
|
||||
|
||||
| Flag | Default | Purpose |
|
||||
| --- | --- | --- |
|
||||
| `-mode` | `suite` | `suite` \| `diff` \| `swebench` \| `compare` \| `traj` (`diff` generates tests for the PR diff; `swebench` runs the official per-instance evaluation; `compare` renders KPI/Pareto readouts from 2+ `-json` reports; `traj` re-digests recorded trajectory files without spending tokens; `audit` checks a batch's published figures against its retained rows). |
|
||||
| `-mode` | `suite` | `suite` \| `diff` \| `swebench` \| `compare` \| `traj` (`diff` generates tests for the PR diff; `swebench` runs the official per-instance evaluation; `compare` renders KPI/Pareto readouts from 2+ `-json` reports; `traj` re-digests recorded trajectory files without spending tokens). |
|
||||
| `-suite` | `benchmarks/e2e` | Suite root (must contain `tasks/<id>/`). |
|
||||
| `-task` | *(all)* | Suite mode: run only these comma-separated task IDs (e.g. `-task fix-add-bug`); unknown IDs fail with the available list. |
|
||||
| `-attempts` | `1` | Suite and diff modes: retry a task until an attempt passes, up to N; enables the `Pass@≤N` KPI, and TTCS charges a retried solve with its failed attempts' wall. |
|
||||
@@ -433,7 +429,6 @@ own outcome).
|
||||
| `-ablate` | *(none)* | Ablation arm: comma-separated subsystems to switch off — `evidence`, `planner`, `subagent`, `retrieval`, `compaction`, `upstream` and `recall-search`; `none` \| `all`. |
|
||||
| `-out` | *(stdout)* | Write the markdown report here. |
|
||||
| `-json` | *(none)* | Write the JSON report here (optional). |
|
||||
| `-batch` | *(none)* | Suite mode: write `results.jsonl` (one row per attempt) and `batch.json` (manifest v1) here, so the report's figures can be audited later. Audit mode: read the batch from here. See [Auditing a batch](#auditing-a-batch). |
|
||||
| `-trajectories` | *(none)* | Suite mode: write one `<task-id>.trajectory.jsonl` per task into this directory (the agent's full event stream with timestamps — see `reasonix run --trajectory`). The report gains a time-attribution line (tools vs. model) and each JSON result a `trajectory` digest. |
|
||||
| `-force-planner` | `false` | Suite mode: prefix each prompt with a plan-first directive so the two-model turn engages regardless of the planner gate. Use for the "with planner" arm of an A/B; results carry `plan_forced` so arms are only comparable with equal forcing. |
|
||||
| `-anchor` | `blind` | Suite mode: which hypothesis the agent holds before it looks at anything — `blind` (none, the control) \| `correct` \| `wrong`. The seeded arms prefix each prompt with the task's authored seed and **skip** tasks that have none, so an unseeded control run never lands in a seeded denominator. Results carry `anchor`. See [Anchor resistance](#anchor-resistance). |
|
||||
@@ -456,65 +451,10 @@ Diff-mode flags:
|
||||
| `-timeout` | `1200` | Agent timeout in seconds (diff mode). |
|
||||
| `-attempts` | `1` | Diff mode: retry up to N times until a run passes (stochastic agent). |
|
||||
|
||||
## Auditing a batch
|
||||
|
||||
`-batch DIR` writes two files from a suite run: `results.jsonl` (one row per
|
||||
attempt — the raw record every figure is computed from) and `batch.json`
|
||||
(manifest v1). `-mode audit -batch DIR` re-derives the report's headline figures
|
||||
from `results.jsonl`, prints the manifest's claim beside the recomputed value,
|
||||
and exits non-zero naming any figure that moved. It also refuses a batch whose
|
||||
manifest is missing, records no commit, or whose rows are truncated.
|
||||
|
||||
This is an **aggregation audit, not a replay**: it proves the published figures
|
||||
follow from the retained rows. It does not re-execute the model, tools or
|
||||
environment, and cannot show the experiment would reproduce. A real replay would
|
||||
need the commit, a corpus fingerprint, the model and provider configuration, the
|
||||
seed, the flags, the raw results and the runtime version; it is not implemented.
|
||||
|
||||
`schema_version` is top level. The rest is three groups:
|
||||
|
||||
| Group | Fields | Answers |
|
||||
| --- | --- | --- |
|
||||
| `over` | `command.mode` and `command.axes` (every flag that changes what a figure means; paths omitted); `tasks.count` and `tasks.digest` (sha256 over sorted, NUL-separated task ids — the corpus identity, never the task text); `filters`; `denominator` (`unit`, `definition`, `value`) | what the figures are over |
|
||||
| `producer` | `commit` (or an explicit `unknown`, never a guess), `commit_source`, `dirty`, `model`, `provider`, `seed`, `started_at`, `finished_at`, `harness_version`, `go_version` | which run produced them |
|
||||
| `trust` | `results_sha256`, `claimed` (the headline figures), `ledger`, `retention` | why they can still be checked |
|
||||
|
||||
The ledger accounts for every planned run, and the audit checks that the counts
|
||||
form a funnel (`scored ≤ completed ≤ attempted`, `failed ≤ scored`,
|
||||
`scored = pass1 + failed`, `rows = attempted + skipped`). A row count is never a
|
||||
denominator: `denominator.value` is a count of scored tasks, and `rows` is
|
||||
published only to show what was retained.
|
||||
|
||||
| Count | Meaning |
|
||||
| --- | --- |
|
||||
| `planned` | graded runs the batch set out to attempt: filtered tasks × trials |
|
||||
| `attempted` | retained rows for runs that started (skipped rows are not attempts) |
|
||||
| `completed` | attempts that ended and left a row; this harness writes a row only after an attempt ends, so it equals `attempted` and the equality is checked, not assumed |
|
||||
| `scored` | completed first attempts of solvable, non-skipped tasks — the pass@1 denominator |
|
||||
| `failed` | scored attempts the grader did not pass |
|
||||
| `unmeasured` | attempts whose cost and tokens are unknown because no metrics landed; kept out of spend totals, never counted as zero |
|
||||
| `excluded` | rows kept out of the accuracy denominator: skipped, `no_solution`, and retry entries |
|
||||
|
||||
Spend and cache fields are omitted from `claimed` when nothing was measured, so
|
||||
an absent figure reads as unknown rather than as a measured zero. `seed` is
|
||||
omitted for the same reason: the harness does not set a sampling seed.
|
||||
`results_sha256` covers the values and the presence of every row; line endings,
|
||||
indentation and key order are normalised away because none of them is evidence,
|
||||
while number spelling is kept (a rewritten `1` is not `1.0`).
|
||||
A successful audit records the verifier version (`audit.verifier_version`) and
|
||||
the digest it checked in `batch.json`, so a re-verification can tell which code
|
||||
checked the batch.
|
||||
|
||||
`results.jsonl` is retained unredacted beside the manifest — task prompts and
|
||||
agent notes included — and is not public. The manifest itself carries a task-id
|
||||
digest, no prompt text, no absolute paths, no endpoints or keys and no model
|
||||
output. A claim whose rows are gone cannot be audited: there is nothing to
|
||||
recompute from, and this tool will not certify it.
|
||||
|
||||
## Dataset retention
|
||||
|
||||
Keep every `-json` report, `-trajectories` directory and `-batch` directory from
|
||||
real runs: they are the accumulating corpus — per-task contracts-to-be, full event
|
||||
Keep every `-json` report and `-trajectories` directory from real runs: they
|
||||
are the accumulating corpus — per-task contracts-to-be, full event
|
||||
trajectories, checkpoint oracle verdicts, stop curves and phase traces — that
|
||||
any future offline learning (routing, stop policies, budgets) would train
|
||||
and evaluate on. The control plane stays deterministic and interpretable
|
||||
|
||||
@@ -1,35 +0,0 @@
|
||||
# Compaction prefix probe
|
||||
|
||||
## Method
|
||||
|
||||
1. Measured on 2026-10-17 against the official `https://api.deepseek.com` OpenAI adapter, using the model returned by `/models`: `deepseek-flash`. Thinking was disabled. Inputs were synthetic; no workspace history or credentials are recorded here.
|
||||
2. Each of three text and three rich-history trials sent a main request to warm the prefix, then called the actual agent summary path, then an independent summary control. Rich history included a synthetic closed `read_file` call, result and reasoning. Warmup outputs were discarded, never executed.
|
||||
3. A summary had a 1,024-token output cap; warmups had 64. Success means a heading, preservation of the fixture's `ParseDuration` and zero-value constraint, no emitted tool calls, and `finish_reason=stop`. This checks the fixture's contract, not arbitrary-summary factual completeness.
|
||||
4. First-attempt reuse success counts successful reused summaries / six reused summaries. Fallback rate counts independent requests initiated by the reuse coordinator / six reuse trials; the six explicit independent controls are excluded. Both were 6/6 success and 0/6 fallbacks. Independent controls also succeeded 6/6.
|
||||
5. The raw result file records provider-reported usage and full stream latency, measured from adapter `Stream` until channel close. Token rows below sum only the three indicated summary requests; they exclude warmup, model listing and any earlier exploratory requests. Cache residency was uncontrolled; prior probes had warmed these synthetic prefixes.
|
||||
|
||||
## Results
|
||||
|
||||
| History | Shape | N | Prompt | Hit | Miss | Output | Median ms | Estimated USD |
|
||||
| --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: |
|
||||
| Text | Reused | 3 | 3,714 | 3,072 | 642 | 636 | 1,713 | 0.000487116 |
|
||||
| Text | Independent | 3 | 1,218 | 768 | 450 | 644 | 1,610 | 0.000456204 |
|
||||
| Rich | Reused | 3 | 3,780 | 3,072 | 708 | 742 | 1,774 | 0.000560616 |
|
||||
| Rich | Independent | 3 | 1,341 | 768 | 573 | 921 | 1,815 | 0.000640854 |
|
||||
|
||||
1. Estimates use the [official pricing](https://api-docs.deepseek.com/quick_start/pricing/) checked on the measurement date: off-peak Flash USD per million tokens, hit 0.003, miss 0.15, output 0.6. October 17 was a Saturday. Formula: `(hit*0.003 + miss*0.15 + output*0.6)/1_000_000`. These are token estimates, not account deductions.
|
||||
2. Reuse hit fractions were 82.7% (text) and 81.3% (rich), versus 63.1% and 57.3% for controls. This small fixture does **not** establish cost savings: retaining the system/catalog increased both total input and input cost. Text total cost increased 6.8%; rich total decreased 12.5% because output was shorter, not because input was cheaper.
|
||||
3. The six warmups consumed 7,032 prompt and 231 output tokens, estimated USD 0.000346680. All 18 recorded generation calls together cost an estimated USD 0.002491470. No failed-digest retry occurred in these live trials; the deterministic fallback tests establish its behavior and accounting.
|
||||
4. A rejected nonempty reused answer earns one independent-shape fallback. A separate truncation retry may enlarge the output cap, for at most three logical model calls total. Every actual attempt emits and records its own spend; returned usage is their sum. Empty output, stream errors and cancellation do not earn shape retries.
|
||||
5. These sequential, six-pair trials establish interoperability for one model, mode and short fixture. They do not estimate production failure probability, thinking-mode behavior, other models, cold-cache cost, or large-history savings.
|
||||
|
||||
## Reproduction
|
||||
|
||||
1. Provide a test key through an existing secret environment configuration. Do not put credentials in source, command arguments or captured logs. Select a model currently returned by `/models`.
|
||||
2. Run the opt-in probe; it skips without `DEEPSEEK_API_KEY`. The dedicated build tag avoids enabling unrelated live tests.
|
||||
|
||||
```sh
|
||||
REASONIX_LIVE_MODEL=deepseek-flash go test -tags live_compaction ./internal/runtime/agent -run '^TestLiveCompactionPrefix$' -v -count=1
|
||||
```
|
||||
|
||||
3. This probe exercises actual summary construction and the HTTP adapter. The hermetic `TestEffectCompactionSharesThePreviouslySentPrefix` separately exercises manual, automatic pressure and rich history through `boot.Build`, comparing actual JSON request-prefix and schema bytes. Its sparse-history case also proves an interleaved retained user turn survives the fold.
|
||||
@@ -1,512 +0,0 @@
|
||||
[
|
||||
{
|
||||
"model": "deepseek-flash",
|
||||
"rich": false,
|
||||
"trial": 1,
|
||||
"results": [
|
||||
{
|
||||
"kind": "warm",
|
||||
"milliseconds": 828,
|
||||
"digest": false,
|
||||
"grounded": false,
|
||||
"tool_calls": 1,
|
||||
"usage": {
|
||||
"PromptTokens": 1127,
|
||||
"CompletionTokens": 38,
|
||||
"TotalTokens": 1165,
|
||||
"CacheHitTokens": 896,
|
||||
"CacheMissTokens": 231,
|
||||
"CacheWriteTokens": 0,
|
||||
"CacheWriteBilledTokens": 0,
|
||||
"ReasoningTokens": 0,
|
||||
"FinishReason": "tool_calls",
|
||||
"Estimated": false,
|
||||
"RequestCount": 1,
|
||||
"ServerToolRequests": 0,
|
||||
"ContextPromptTokens": 0,
|
||||
"ContextCompletionTokens": 0,
|
||||
"ContextReasoningTokens": 0,
|
||||
"ContextCacheHitTokens": 0,
|
||||
"ContextCacheMissTokens": 0
|
||||
}
|
||||
},
|
||||
{
|
||||
"kind": "reuse",
|
||||
"milliseconds": 2006,
|
||||
"digest": true,
|
||||
"grounded": true,
|
||||
"tool_calls": 0,
|
||||
"usage": {
|
||||
"PromptTokens": 1238,
|
||||
"CompletionTokens": 234,
|
||||
"TotalTokens": 1472,
|
||||
"CacheHitTokens": 1024,
|
||||
"CacheMissTokens": 214,
|
||||
"CacheWriteTokens": 0,
|
||||
"CacheWriteBilledTokens": 0,
|
||||
"ReasoningTokens": 0,
|
||||
"FinishReason": "stop",
|
||||
"Estimated": false,
|
||||
"RequestCount": 1,
|
||||
"ServerToolRequests": 0,
|
||||
"ContextPromptTokens": 0,
|
||||
"ContextCompletionTokens": 0,
|
||||
"ContextReasoningTokens": 0,
|
||||
"ContextCacheHitTokens": 0,
|
||||
"ContextCacheMissTokens": 0
|
||||
}
|
||||
},
|
||||
{
|
||||
"kind": "independent",
|
||||
"milliseconds": 1817,
|
||||
"digest": true,
|
||||
"grounded": true,
|
||||
"tool_calls": 0,
|
||||
"usage": {
|
||||
"PromptTokens": 406,
|
||||
"CompletionTokens": 234,
|
||||
"TotalTokens": 640,
|
||||
"CacheHitTokens": 256,
|
||||
"CacheMissTokens": 150,
|
||||
"CacheWriteTokens": 0,
|
||||
"CacheWriteBilledTokens": 0,
|
||||
"ReasoningTokens": 0,
|
||||
"FinishReason": "stop",
|
||||
"Estimated": false,
|
||||
"RequestCount": 1,
|
||||
"ServerToolRequests": 0,
|
||||
"ContextPromptTokens": 0,
|
||||
"ContextCompletionTokens": 0,
|
||||
"ContextReasoningTokens": 0,
|
||||
"ContextCacheHitTokens": 0,
|
||||
"ContextCacheMissTokens": 0
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"model": "deepseek-flash",
|
||||
"rich": false,
|
||||
"trial": 2,
|
||||
"results": [
|
||||
{
|
||||
"kind": "warm",
|
||||
"milliseconds": 820,
|
||||
"digest": false,
|
||||
"grounded": false,
|
||||
"tool_calls": 1,
|
||||
"usage": {
|
||||
"PromptTokens": 1127,
|
||||
"CompletionTokens": 38,
|
||||
"TotalTokens": 1165,
|
||||
"CacheHitTokens": 896,
|
||||
"CacheMissTokens": 231,
|
||||
"CacheWriteTokens": 0,
|
||||
"CacheWriteBilledTokens": 0,
|
||||
"ReasoningTokens": 0,
|
||||
"FinishReason": "tool_calls",
|
||||
"Estimated": false,
|
||||
"RequestCount": 1,
|
||||
"ServerToolRequests": 0,
|
||||
"ContextPromptTokens": 0,
|
||||
"ContextCompletionTokens": 0,
|
||||
"ContextReasoningTokens": 0,
|
||||
"ContextCacheHitTokens": 0,
|
||||
"ContextCacheMissTokens": 0
|
||||
}
|
||||
},
|
||||
{
|
||||
"kind": "reuse",
|
||||
"milliseconds": 1633,
|
||||
"digest": true,
|
||||
"grounded": true,
|
||||
"tool_calls": 0,
|
||||
"usage": {
|
||||
"PromptTokens": 1238,
|
||||
"CompletionTokens": 187,
|
||||
"TotalTokens": 1425,
|
||||
"CacheHitTokens": 1024,
|
||||
"CacheMissTokens": 214,
|
||||
"CacheWriteTokens": 0,
|
||||
"CacheWriteBilledTokens": 0,
|
||||
"ReasoningTokens": 0,
|
||||
"FinishReason": "stop",
|
||||
"Estimated": false,
|
||||
"RequestCount": 1,
|
||||
"ServerToolRequests": 0,
|
||||
"ContextPromptTokens": 0,
|
||||
"ContextCompletionTokens": 0,
|
||||
"ContextReasoningTokens": 0,
|
||||
"ContextCacheHitTokens": 0,
|
||||
"ContextCacheMissTokens": 0
|
||||
}
|
||||
},
|
||||
{
|
||||
"kind": "independent",
|
||||
"milliseconds": 1555,
|
||||
"digest": true,
|
||||
"grounded": true,
|
||||
"tool_calls": 0,
|
||||
"usage": {
|
||||
"PromptTokens": 406,
|
||||
"CompletionTokens": 202,
|
||||
"TotalTokens": 608,
|
||||
"CacheHitTokens": 256,
|
||||
"CacheMissTokens": 150,
|
||||
"CacheWriteTokens": 0,
|
||||
"CacheWriteBilledTokens": 0,
|
||||
"ReasoningTokens": 0,
|
||||
"FinishReason": "stop",
|
||||
"Estimated": false,
|
||||
"RequestCount": 1,
|
||||
"ServerToolRequests": 0,
|
||||
"ContextPromptTokens": 0,
|
||||
"ContextCompletionTokens": 0,
|
||||
"ContextReasoningTokens": 0,
|
||||
"ContextCacheHitTokens": 0,
|
||||
"ContextCacheMissTokens": 0
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"model": "deepseek-flash",
|
||||
"rich": false,
|
||||
"trial": 3,
|
||||
"results": [
|
||||
{
|
||||
"kind": "warm",
|
||||
"milliseconds": 905,
|
||||
"digest": false,
|
||||
"grounded": false,
|
||||
"tool_calls": 1,
|
||||
"usage": {
|
||||
"PromptTokens": 1127,
|
||||
"CompletionTokens": 38,
|
||||
"TotalTokens": 1165,
|
||||
"CacheHitTokens": 896,
|
||||
"CacheMissTokens": 231,
|
||||
"CacheWriteTokens": 0,
|
||||
"CacheWriteBilledTokens": 0,
|
||||
"ReasoningTokens": 0,
|
||||
"FinishReason": "tool_calls",
|
||||
"Estimated": false,
|
||||
"RequestCount": 1,
|
||||
"ServerToolRequests": 0,
|
||||
"ContextPromptTokens": 0,
|
||||
"ContextCompletionTokens": 0,
|
||||
"ContextReasoningTokens": 0,
|
||||
"ContextCacheHitTokens": 0,
|
||||
"ContextCacheMissTokens": 0
|
||||
}
|
||||
},
|
||||
{
|
||||
"kind": "reuse",
|
||||
"milliseconds": 1713,
|
||||
"digest": true,
|
||||
"grounded": true,
|
||||
"tool_calls": 0,
|
||||
"usage": {
|
||||
"PromptTokens": 1238,
|
||||
"CompletionTokens": 215,
|
||||
"TotalTokens": 1453,
|
||||
"CacheHitTokens": 1024,
|
||||
"CacheMissTokens": 214,
|
||||
"CacheWriteTokens": 0,
|
||||
"CacheWriteBilledTokens": 0,
|
||||
"ReasoningTokens": 0,
|
||||
"FinishReason": "stop",
|
||||
"Estimated": false,
|
||||
"RequestCount": 1,
|
||||
"ServerToolRequests": 0,
|
||||
"ContextPromptTokens": 0,
|
||||
"ContextCompletionTokens": 0,
|
||||
"ContextReasoningTokens": 0,
|
||||
"ContextCacheHitTokens": 0,
|
||||
"ContextCacheMissTokens": 0
|
||||
}
|
||||
},
|
||||
{
|
||||
"kind": "independent",
|
||||
"milliseconds": 1610,
|
||||
"digest": true,
|
||||
"grounded": true,
|
||||
"tool_calls": 0,
|
||||
"usage": {
|
||||
"PromptTokens": 406,
|
||||
"CompletionTokens": 208,
|
||||
"TotalTokens": 614,
|
||||
"CacheHitTokens": 256,
|
||||
"CacheMissTokens": 150,
|
||||
"CacheWriteTokens": 0,
|
||||
"CacheWriteBilledTokens": 0,
|
||||
"ReasoningTokens": 0,
|
||||
"FinishReason": "stop",
|
||||
"Estimated": false,
|
||||
"RequestCount": 1,
|
||||
"ServerToolRequests": 0,
|
||||
"ContextPromptTokens": 0,
|
||||
"ContextCompletionTokens": 0,
|
||||
"ContextReasoningTokens": 0,
|
||||
"ContextCacheHitTokens": 0,
|
||||
"ContextCacheMissTokens": 0
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"model": "deepseek-flash",
|
||||
"rich": true,
|
||||
"trial": 1,
|
||||
"results": [
|
||||
{
|
||||
"kind": "warm",
|
||||
"milliseconds": 561,
|
||||
"digest": false,
|
||||
"grounded": false,
|
||||
"tool_calls": 1,
|
||||
"usage": {
|
||||
"PromptTokens": 1217,
|
||||
"CompletionTokens": 39,
|
||||
"TotalTokens": 1256,
|
||||
"CacheHitTokens": 1024,
|
||||
"CacheMissTokens": 193,
|
||||
"CacheWriteTokens": 0,
|
||||
"CacheWriteBilledTokens": 0,
|
||||
"ReasoningTokens": 0,
|
||||
"FinishReason": "tool_calls",
|
||||
"Estimated": false,
|
||||
"RequestCount": 1,
|
||||
"ServerToolRequests": 0,
|
||||
"ContextPromptTokens": 0,
|
||||
"ContextCompletionTokens": 0,
|
||||
"ContextReasoningTokens": 0,
|
||||
"ContextCacheHitTokens": 0,
|
||||
"ContextCacheMissTokens": 0
|
||||
}
|
||||
},
|
||||
{
|
||||
"kind": "reuse",
|
||||
"milliseconds": 1774,
|
||||
"digest": true,
|
||||
"grounded": true,
|
||||
"tool_calls": 0,
|
||||
"usage": {
|
||||
"PromptTokens": 1260,
|
||||
"CompletionTokens": 256,
|
||||
"TotalTokens": 1516,
|
||||
"CacheHitTokens": 1024,
|
||||
"CacheMissTokens": 236,
|
||||
"CacheWriteTokens": 0,
|
||||
"CacheWriteBilledTokens": 0,
|
||||
"ReasoningTokens": 0,
|
||||
"FinishReason": "stop",
|
||||
"Estimated": false,
|
||||
"RequestCount": 1,
|
||||
"ServerToolRequests": 0,
|
||||
"ContextPromptTokens": 0,
|
||||
"ContextCompletionTokens": 0,
|
||||
"ContextReasoningTokens": 0,
|
||||
"ContextCacheHitTokens": 0,
|
||||
"ContextCacheMissTokens": 0
|
||||
}
|
||||
},
|
||||
{
|
||||
"kind": "independent",
|
||||
"milliseconds": 2271,
|
||||
"digest": true,
|
||||
"grounded": true,
|
||||
"tool_calls": 0,
|
||||
"usage": {
|
||||
"PromptTokens": 447,
|
||||
"CompletionTokens": 381,
|
||||
"TotalTokens": 828,
|
||||
"CacheHitTokens": 256,
|
||||
"CacheMissTokens": 191,
|
||||
"CacheWriteTokens": 0,
|
||||
"CacheWriteBilledTokens": 0,
|
||||
"ReasoningTokens": 0,
|
||||
"FinishReason": "stop",
|
||||
"Estimated": false,
|
||||
"RequestCount": 1,
|
||||
"ServerToolRequests": 0,
|
||||
"ContextPromptTokens": 0,
|
||||
"ContextCompletionTokens": 0,
|
||||
"ContextReasoningTokens": 0,
|
||||
"ContextCacheHitTokens": 0,
|
||||
"ContextCacheMissTokens": 0
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"model": "deepseek-flash",
|
||||
"rich": true,
|
||||
"trial": 2,
|
||||
"results": [
|
||||
{
|
||||
"kind": "warm",
|
||||
"milliseconds": 610,
|
||||
"digest": false,
|
||||
"grounded": false,
|
||||
"tool_calls": 1,
|
||||
"usage": {
|
||||
"PromptTokens": 1217,
|
||||
"CompletionTokens": 39,
|
||||
"TotalTokens": 1256,
|
||||
"CacheHitTokens": 1024,
|
||||
"CacheMissTokens": 193,
|
||||
"CacheWriteTokens": 0,
|
||||
"CacheWriteBilledTokens": 0,
|
||||
"ReasoningTokens": 0,
|
||||
"FinishReason": "tool_calls",
|
||||
"Estimated": false,
|
||||
"RequestCount": 1,
|
||||
"ServerToolRequests": 0,
|
||||
"ContextPromptTokens": 0,
|
||||
"ContextCompletionTokens": 0,
|
||||
"ContextReasoningTokens": 0,
|
||||
"ContextCacheHitTokens": 0,
|
||||
"ContextCacheMissTokens": 0
|
||||
}
|
||||
},
|
||||
{
|
||||
"kind": "reuse",
|
||||
"milliseconds": 2263,
|
||||
"digest": true,
|
||||
"grounded": true,
|
||||
"tool_calls": 0,
|
||||
"usage": {
|
||||
"PromptTokens": 1260,
|
||||
"CompletionTokens": 276,
|
||||
"TotalTokens": 1536,
|
||||
"CacheHitTokens": 1024,
|
||||
"CacheMissTokens": 236,
|
||||
"CacheWriteTokens": 0,
|
||||
"CacheWriteBilledTokens": 0,
|
||||
"ReasoningTokens": 0,
|
||||
"FinishReason": "stop",
|
||||
"Estimated": false,
|
||||
"RequestCount": 1,
|
||||
"ServerToolRequests": 0,
|
||||
"ContextPromptTokens": 0,
|
||||
"ContextCompletionTokens": 0,
|
||||
"ContextReasoningTokens": 0,
|
||||
"ContextCacheHitTokens": 0,
|
||||
"ContextCacheMissTokens": 0
|
||||
}
|
||||
},
|
||||
{
|
||||
"kind": "independent",
|
||||
"milliseconds": 1783,
|
||||
"digest": true,
|
||||
"grounded": true,
|
||||
"tool_calls": 0,
|
||||
"usage": {
|
||||
"PromptTokens": 447,
|
||||
"CompletionTokens": 242,
|
||||
"TotalTokens": 689,
|
||||
"CacheHitTokens": 256,
|
||||
"CacheMissTokens": 191,
|
||||
"CacheWriteTokens": 0,
|
||||
"CacheWriteBilledTokens": 0,
|
||||
"ReasoningTokens": 0,
|
||||
"FinishReason": "stop",
|
||||
"Estimated": false,
|
||||
"RequestCount": 1,
|
||||
"ServerToolRequests": 0,
|
||||
"ContextPromptTokens": 0,
|
||||
"ContextCompletionTokens": 0,
|
||||
"ContextReasoningTokens": 0,
|
||||
"ContextCacheHitTokens": 0,
|
||||
"ContextCacheMissTokens": 0
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"model": "deepseek-flash",
|
||||
"rich": true,
|
||||
"trial": 3,
|
||||
"results": [
|
||||
{
|
||||
"kind": "warm",
|
||||
"milliseconds": 834,
|
||||
"digest": false,
|
||||
"grounded": false,
|
||||
"tool_calls": 1,
|
||||
"usage": {
|
||||
"PromptTokens": 1217,
|
||||
"CompletionTokens": 39,
|
||||
"TotalTokens": 1256,
|
||||
"CacheHitTokens": 1024,
|
||||
"CacheMissTokens": 193,
|
||||
"CacheWriteTokens": 0,
|
||||
"CacheWriteBilledTokens": 0,
|
||||
"ReasoningTokens": 0,
|
||||
"FinishReason": "tool_calls",
|
||||
"Estimated": false,
|
||||
"RequestCount": 1,
|
||||
"ServerToolRequests": 0,
|
||||
"ContextPromptTokens": 0,
|
||||
"ContextCompletionTokens": 0,
|
||||
"ContextReasoningTokens": 0,
|
||||
"ContextCacheHitTokens": 0,
|
||||
"ContextCacheMissTokens": 0
|
||||
}
|
||||
},
|
||||
{
|
||||
"kind": "reuse",
|
||||
"milliseconds": 1575,
|
||||
"digest": true,
|
||||
"grounded": true,
|
||||
"tool_calls": 0,
|
||||
"usage": {
|
||||
"PromptTokens": 1260,
|
||||
"CompletionTokens": 210,
|
||||
"TotalTokens": 1470,
|
||||
"CacheHitTokens": 1024,
|
||||
"CacheMissTokens": 236,
|
||||
"CacheWriteTokens": 0,
|
||||
"CacheWriteBilledTokens": 0,
|
||||
"ReasoningTokens": 0,
|
||||
"FinishReason": "stop",
|
||||
"Estimated": false,
|
||||
"RequestCount": 1,
|
||||
"ServerToolRequests": 0,
|
||||
"ContextPromptTokens": 0,
|
||||
"ContextCompletionTokens": 0,
|
||||
"ContextReasoningTokens": 0,
|
||||
"ContextCacheHitTokens": 0,
|
||||
"ContextCacheMissTokens": 0
|
||||
}
|
||||
},
|
||||
{
|
||||
"kind": "independent",
|
||||
"milliseconds": 1815,
|
||||
"digest": true,
|
||||
"grounded": true,
|
||||
"tool_calls": 0,
|
||||
"usage": {
|
||||
"PromptTokens": 447,
|
||||
"CompletionTokens": 298,
|
||||
"TotalTokens": 745,
|
||||
"CacheHitTokens": 256,
|
||||
"CacheMissTokens": 191,
|
||||
"CacheWriteTokens": 0,
|
||||
"CacheWriteBilledTokens": 0,
|
||||
"ReasoningTokens": 0,
|
||||
"FinishReason": "stop",
|
||||
"Estimated": false,
|
||||
"RequestCount": 1,
|
||||
"ServerToolRequests": 0,
|
||||
"ContextPromptTokens": 0,
|
||||
"ContextCompletionTokens": 0,
|
||||
"ContextReasoningTokens": 0,
|
||||
"ContextCacheHitTokens": 0,
|
||||
"ContextCacheMissTokens": 0
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
@@ -103,6 +103,10 @@ func newHarness(t *testingDir, p provider.Provider, window int, rec *callRecorde
|
||||
ContextWindow: window,
|
||||
ArchiveDir: filepath.Join(t.dir, "archive"),
|
||||
SessionPath: path,
|
||||
RecentKeep: 4,
|
||||
// boot's default when cfg.Agent.Keep is unset; without it the bench
|
||||
// would measure a configuration no real session runs.
|
||||
KeepPolicy: agent.KeepErrors,
|
||||
}, rec.sink())
|
||||
return &harness{sess: sess, agentA: a, path: path, calls: rec}
|
||||
}
|
||||
|
||||
@@ -108,6 +108,7 @@ func newAgent(p provider.Provider, sess *sessionstore.Session, path string, sink
|
||||
return agent.New(p, tool.NewRegistry(), sess, agent.Options{
|
||||
ContextWindow: windowTokens,
|
||||
CompactRatio: compactRatio,
|
||||
RecentKeep: 2,
|
||||
SessionPath: path,
|
||||
WorkspaceID: "cm-e2e",
|
||||
ModelRef: "deepseek/" + model,
|
||||
|
||||
@@ -0,0 +1,180 @@
|
||||
# Context retrieval bench
|
||||
|
||||
Two questions about long-horizon memory, and the harness that makes their
|
||||
answers trustworthy.
|
||||
|
||||
-mode=preflight build all 18 tasks under every arm and check the contract
|
||||
-mode=snippet does a rank-1 hit hand over the whole answer?
|
||||
-mode=adversarial send a model after the answer through every host surface
|
||||
-mode=run-search Experiment R: is searchable canonical a real capability?
|
||||
-mode=run-boundary Experiment I, paired: the 6 efficiency tasks across their boundary
|
||||
-mode=run-index Experiment I, full: the 6 efficiency tasks x 4 budgets
|
||||
-mode=calibrate find each index task's PlantAfterGen for its cue tier
|
||||
|
||||
`-dry` drives any run mode with a scripted provider, for nothing. Real runs need
|
||||
`DEEPSEEK_API_KEY`.
|
||||
|
||||
## Status
|
||||
|
||||
**Searchable canonical recall: proven.** Six of six recovered with search on,
|
||||
three of six with it off, and the successes without it came from enumerating
|
||||
addresses by hand at roughly nine times the page-in cost (139 vs 1292
|
||||
tokens/task). Measured after three separate leaks were closed, so the numbers
|
||||
are from a clean environment.
|
||||
|
||||
**Fold index marginal utility: direction reproduced, size still conditional.** On the v2
|
||||
substrate the direction holds in six of six tasks and in the pooled paired batch: with the
|
||||
cue visible the index addresses the target, so the same task is searched less and pages in
|
||||
less (0.67 searches and 70 recall tokens per task, against 1.50 and 120 with the cue
|
||||
absent).
|
||||
|
||||
A size is visible but is not yet quotable. On the four tasks whose absent-cue arm stayed
|
||||
out of a loop, the index cost 64,952 prompt tokens against 115,551 without, 44% less
|
||||
(`i106-budgetseal` -35,147, `i70-ingestfloor` -8,222, `i37-dispatchlag` -7,599, and
|
||||
`i112-lagquota` +369, the prefix rent a task that never searches pays). That is a
|
||||
**post-hoc subset**: `i118-fencelag` and `i47-tenantlease` are excluded because their
|
||||
absent-cue arm looped in this batch (five and three retrieval rounds, three escapes each;
|
||||
together 80% of the absent-cue total), and the qualification above does not screen for
|
||||
stopping behaviour — holding on both boundary arms across three passes shows a stable cue
|
||||
boundary, not a stopping arm. Quoting 44% as the index's effect would mean choosing the
|
||||
sample after seeing the result; a batch screened on stopping stability is what would make
|
||||
a size quotable, and one arm also reverses either way (`i70-ingestfloor` default, +11,135
|
||||
prompt tokens over three escapes).
|
||||
|
||||
Batch facts, because the counts differ: 36 runs were executed (12 paired, 24 index), and
|
||||
the token totals here cover the 24-run index batch — the paired batch ran before the
|
||||
per-run usage line existed.
|
||||
|
||||
**Things that turned out not to be problems**, each after being measured rather
|
||||
than argued about:
|
||||
|
||||
| Suspected | Measured |
|
||||
| --- | --- |
|
||||
| Model writes poor queries (2.17 searches/task) | 22/23 found the target on query 1; the extra searches are one round's parallel fan-out |
|
||||
| Model reaches for the workspace before memory (7/11) | Linear event order misread; by model round it is MemoryFirst 4/6 |
|
||||
| Snippets too short for multi-value answers | 240 runes covers 12/12 tasks at 100%; records are 30-179 runes |
|
||||
| A general stopping failure | 37/44 runs issued no search after the answer was in hand; all 7 that did belong to 2 tasks |
|
||||
|
||||
## Measurement distinctions this bench had to learn
|
||||
|
||||
Each of these reversed a conclusion once. They are enforced in the schema now,
|
||||
not left to whoever reads the numbers next.
|
||||
|
||||
- **TargetHit is not EvidenceSufficient.** A rank-1 hit can return a window too
|
||||
short to answer with. `FirstHitCoverage` records how much of the scored answer
|
||||
the first hit actually handed over.
|
||||
- **Linear event order is not model-round order.** Two calls in one round are a
|
||||
parallel fan-out, not a preference. Routing compares rounds.
|
||||
- **An isolated workdir is not an isolated host.** The sandbox mounts the host
|
||||
read-only by design. Three leaks were found and closed: the corpus source, the
|
||||
fixture transcript, and its event log sidecar.
|
||||
- **A count is not its evidence.** Query text, snippet contents and trajectories
|
||||
are all persisted now, because three analyses in a row needed a paid re-run to
|
||||
ask a question of data already collected.
|
||||
|
||||
## How a run stays honest
|
||||
|
||||
The corpus holds templates, never answers: every scored literal is a `{{var}}`
|
||||
instantiated per run, so grepping this directory reveals the question and not
|
||||
the answer. `TestNoAnswerLiteralExistsInTheRepository` asserts it with
|
||||
`git grep`. Preflight rebuilds every arm and checks the answer is unreadable in
|
||||
the real `provider.Request`, the probe query ranks the target within five, and
|
||||
an index task's cue sits at exactly the scales its tier names. The fixture is
|
||||
removed from disk once the agent holds it in memory, and the directory is
|
||||
scanned for answer literals before the first provider request and again after
|
||||
the turn. Any answer reaching the model through a tool that is not `recall`
|
||||
marks the run contaminated and takes it out of every statistic.
|
||||
|
||||
## Index corpus v2
|
||||
|
||||
The efficiency substrate is six tasks that hold on both arms: `i37-dispatchlag`
|
||||
and `i118-fencelag` (quarter), `i106-budgetseal` and `i70-ingestfloor` (half),
|
||||
`i47-tenantlease` and `i112-lagquota` (default).
|
||||
|
||||
They came out of five batches of candidates, each screened the same way: one
|
||||
`-mode=run-index` pass over the batch, then two more only for the candidates that
|
||||
were clean on both boundary arms, and only the candidates that held on both arms
|
||||
in all three passes stayed. A candidate that does not hold is dropped, not
|
||||
promoted. The numbers are candidates, per tier:
|
||||
|
||||
| batch | quarter | half | default |
|
||||
| --- | --- | --- | --- |
|
||||
| first pool, `i31`-`i54` | 1 of 8 | 0 of 8 | 2 of 8 |
|
||||
| second pool, `i61`-`i84` | 1 of 4 screened | 4 of 8 | 0 of 8 |
|
||||
| third pool, `i85`-`i108` | 2 of 8 | 8 of 16 | none authored |
|
||||
| replacement pool, `i109`-`i116` | - | - | 4 of 8 |
|
||||
| replacement pool, `i92`, `i117`-`i124` | 3 of 9 | - | - |
|
||||
|
||||
That table counts candidates that held on both arms in all three passes, which is
|
||||
not the size of the substrate. 89 candidates were authored, 77 were screened (the
|
||||
second pool's quarter tier stopped at 4 of 8 and its default tier was never run),
|
||||
and 25 held: 7 quarter, 12 half, 6 default. The substrate keeps two per tier, so
|
||||
the other 19 survivors are spares rather than corpus members. Candidates from the
|
||||
earlier round are not counted here.
|
||||
|
||||
| task | tier | cue present | cue absent |
|
||||
| --- | --- | ---: | ---: |
|
||||
| `i37-dispatchlag` | quarter | 3/3 | 3/3 |
|
||||
| `i118-fencelag` | quarter | 3/3 | 3/3 |
|
||||
| `i106-budgetseal` | half | 3/3 | 3/3 |
|
||||
| `i70-ingestfloor` | half | 3/3 | 3/3 |
|
||||
| `i47-tenantlease` | default | 3/3 | 3/3 |
|
||||
| `i112-lagquota` | default | 3/3 | 3/3 |
|
||||
|
||||
Across their 36 judged cells: `SnippetStop` 23, `SnippetThenRead` 10 and
|
||||
`DefensiveRead` 3 - every cell clean, nothing post-sufficient and nothing
|
||||
`NeverSufficient`.
|
||||
|
||||
Four of the first six were flagged in review for sharing answer values, probe
|
||||
queries and prompt wording within a tier. Separating them and re-screening those
|
||||
four changed two of them: `i37` and `i47` held, `i62` and `i51` did not, so they
|
||||
were replaced - `i90-recoverylag` was tried first and did not hold either, and
|
||||
`i118-fencelag` and `i112-lagquota` are what did. The separation was not
|
||||
cosmetic: the shared wording had been carrying two of the six.
|
||||
|
||||
Nothing in the task shape separates the survivors from the rest: within the half
|
||||
tier, plain-number answers survived 0 of 8 in the first pool and 4 of 8 in the
|
||||
second, and codename answers survived 8 of 16 in the third. An earlier claim here
|
||||
that the answer kind decides whether a candidate holds does not survive this
|
||||
table and is withdrawn. The dirty class was `PostSufficientRetrieval` in most
|
||||
failures - the model re-checking a value it already had - but which tasks hit it
|
||||
was not predictable from their shape at this sample.
|
||||
|
||||
`i01`-`i06` are the frozen study's six tasks, kept as `roleStopping`. Measured
|
||||
once each with `-mode=run-index -task <id>`, per task and per arm:
|
||||
|
||||
| task | tier | cue present | cue absent |
|
||||
| --- | --- | --- | --- |
|
||||
| `i01-transport-fallback` | quarter | SnippetStop | DefensiveRead |
|
||||
| `i02-scheduler-handoff` | quarter | PostSufficientRetrieval | PostSufficientRetrieval |
|
||||
| `i03-coalescing` | half | PostSufficientRetrieval | PostSufficientRunaway |
|
||||
| `i04-recovery-fence` | half | DefensiveRead | PostSufficientRunaway |
|
||||
| `i05-cache-probe` | default | SnippetThenRead | PostSufficientRetrieval |
|
||||
| `i06-dispatch-handoff` | default | SnippetStop | PostSufficientRetrieval |
|
||||
|
||||
Five of the six show post-sufficient behaviour in at least one arm (`i02`-`i06`),
|
||||
and `i01` is clean on both. The cue-present arm is clean for four of them
|
||||
(`i01`, `i04`, `i05`, `i06`), which is why they stay in the corpus as the
|
||||
stopping side rather than being retitled. All six are excluded from `run-index`
|
||||
and `run-boundary` batches and stay measurable one at a time with `-task <id>`. A
|
||||
task added to the corpus declares a role, so it cannot arrive without one.
|
||||
|
||||
A task qualifies only if it is clean on both of its boundary arms - the
|
||||
cue-present run and the cue-absent run - in every run of a screening batch. Clean
|
||||
means a `stopping_class` other than `PostSufficientRetrieval` or
|
||||
`PostSufficientRunaway`: `SnippetStop`, `SnippetThenRead` and `DefensiveRead`
|
||||
all qualify, and a defensive read is deliberately not counted as waste. The
|
||||
class comes from the three counts in that report — `searches_after_sufficient`,
|
||||
`reads_after_sufficient` and `escapes_after_sufficient` — together with how
|
||||
sufficiency was reached and how many rounds followed it. The remaining
|
||||
requirements still apply — successful tool call, answer in the output rather
|
||||
than the cue, rank 1, `FirstHitCoverage` complete, one or two simple values,
|
||||
nothing in the workspace.
|
||||
|
||||
`-mode=calibrate` is not a screening tool: it builds and reads back the
|
||||
fixtures, and reads nothing else, so its cue-visibility profile is the same for
|
||||
every task.
|
||||
|
||||
Then repeats rather than levels: cue-present against cue-absent on each task's
|
||||
own boundary, three runs per cell. What is unknown is the variance, and four
|
||||
budgets sampled once each cannot show it.
|
||||
@@ -0,0 +1,86 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// runBlockedBatch runs one dry index batch where every arm's runOne fails
|
||||
// before any model turn: the task's own directory is a regular file, so the
|
||||
// fixture cannot be built. In production the same early return happens when the
|
||||
// fixture build, the session load, or the answer-on-disk guard fails.
|
||||
func runBlockedBatch(t *testing.T) string {
|
||||
t.Helper()
|
||||
tk := indexTasks()[0]
|
||||
root := t.TempDir()
|
||||
if err := os.WriteFile(filepath.Join(root, tk.ID), []byte("not a directory\n"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if code := runExperiment("run-index", experimentIndex, root, true, []contextTask{tk}); code != 0 {
|
||||
t.Fatalf("runExperiment exit = %d, want 0 (every run failed, but the batch finished)", code)
|
||||
}
|
||||
return root
|
||||
}
|
||||
|
||||
func readResultLines(t *testing.T, root string) ([]contextMetrics, []string) {
|
||||
t.Helper()
|
||||
raw, err := os.ReadFile(filepath.Join(root, "results.jsonl"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var runs []contextMetrics
|
||||
var lines []string
|
||||
for text := range strings.SplitSeq(strings.TrimSpace(string(raw)), "\n") {
|
||||
var m contextMetrics
|
||||
if err := json.Unmarshal([]byte(text), &m); err != nil {
|
||||
t.Fatalf("results.jsonl: %v", err)
|
||||
}
|
||||
runs = append(runs, m)
|
||||
lines = append(lines, text)
|
||||
}
|
||||
return runs, lines
|
||||
}
|
||||
|
||||
// A run that failed before the model turn produced no measurement. Recording it
|
||||
// as a zero-valued contextMetrics puts it in the funnel's Scored denominator,
|
||||
// in "Where it broke", and in the dose curve as a real zero.
|
||||
func TestBatchDoesNotRecordAFailedRunAsAResult(t *testing.T) {
|
||||
root := runBlockedBatch(t)
|
||||
runs, lines := readResultLines(t, root)
|
||||
if len(runs) == 0 {
|
||||
t.Fatal("no runs recorded at all")
|
||||
}
|
||||
for i, m := range runs {
|
||||
if m.Task == "" || m.Arm == "" || m.FailureStage == "" {
|
||||
t.Fatalf("results.jsonl line %d records a run that never ran as a result "+
|
||||
"(task=%q arm=%q failure_stage=%q):\n %s", i+1, m.Task, m.Arm, m.FailureStage, lines[i])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The manifest is the artefact that says which runs counted, so a failed run
|
||||
// must not be added to CompletedRuns or to the task id list.
|
||||
func TestBatchManifestDoesNotReportFailedRunsAsCompleted(t *testing.T) {
|
||||
root := runBlockedBatch(t)
|
||||
var manifest batchManifest
|
||||
data, err := os.ReadFile(filepath.Join(root, "batch.json"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := json.Unmarshal(data, &manifest); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, id := range manifest.TaskIDs {
|
||||
if id == "" {
|
||||
t.Fatalf("manifest lists an empty task id: task_ids=%q (completed=%d planned=%d)",
|
||||
manifest.TaskIDs, manifest.CompletedRuns, manifest.PlannedRuns)
|
||||
}
|
||||
}
|
||||
if manifest.CompletedRuns == manifest.PlannedRuns {
|
||||
t.Fatalf("manifest reports every planned run completed although none produced evidence: "+
|
||||
"completed=%d planned=%d task_ids=%q", manifest.CompletedRuns, manifest.PlannedRuns, manifest.TaskIDs)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,155 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// batchInfo is what the mode itself knows before any run happens. Everything
|
||||
// else in the manifest is derived from the runs that were written, so a figure
|
||||
// in it cannot drift from the artifact beside it.
|
||||
type batchInfo struct {
|
||||
Mode string
|
||||
Planned int
|
||||
Started time.Time
|
||||
Dry bool
|
||||
// Repo is the checkout the batch ran from, captured before the harness
|
||||
// isolates its workspace: git cannot describe a directory that is gone.
|
||||
Repo string
|
||||
}
|
||||
|
||||
// batchManifest is written beside results.jsonl so a later reader does not have
|
||||
// to reconstruct what the batch was from a terminal scrollback.
|
||||
type batchManifest struct {
|
||||
SchemaVersion int `json:"schema_version"`
|
||||
Mode string `json:"mode"`
|
||||
StartedAt string `json:"started_at"`
|
||||
FinishedAt string `json:"finished_at"`
|
||||
GitCommit string `json:"git_commit"`
|
||||
GitDirty *bool `json:"git_dirty,omitempty"`
|
||||
Model string `json:"model"`
|
||||
Window int `json:"window"`
|
||||
Generations int `json:"generations"`
|
||||
TaskIDs []string `json:"task_ids"`
|
||||
Arms []string `json:"arms"`
|
||||
PlannedRuns int `json:"planned_runs"`
|
||||
CompletedRuns int `json:"completed_runs"`
|
||||
// FailedRuns counts the planned runs that ended before a transcript existed.
|
||||
// They are rows in results.jsonl, never part of CompletedRuns, and the field
|
||||
// is absent when there were none so an intact batch is unchanged.
|
||||
FailedRuns int `json:"failed_runs,omitempty"`
|
||||
// Usage totals are absent when nothing reported any: an unmeasured cost must
|
||||
// not read as a measured zero. UsageReportedRuns stays, because zero there
|
||||
// is a fact about the batch.
|
||||
UsageReportedRuns int `json:"usage_reported_runs"`
|
||||
UsageReportedRequests int `json:"usage_reported_requests,omitempty"`
|
||||
PromptTokens int `json:"prompt_tokens,omitempty"`
|
||||
CompletionTokens int `json:"completion_tokens,omitempty"`
|
||||
CacheHitTokens int `json:"cache_hit_tokens,omitempty"`
|
||||
CacheMissTokens int `json:"cache_miss_tokens,omitempty"`
|
||||
UsageEstimated bool `json:"usage_estimated,omitempty"`
|
||||
ResultsSHA256 string `json:"results_sha256"`
|
||||
TrajectoriesSHA256 string `json:"trajectories_sha256"`
|
||||
}
|
||||
|
||||
// gitCommitAndDirty reports the checkout the batch ran from. Unknown is written
|
||||
// rather than omitted, so an absent field is never mistaken for a clean tree.
|
||||
func gitCommitAndDirty(repo string) (commit string, dirty *bool) {
|
||||
commit = "unknown"
|
||||
if repo == "" {
|
||||
return commit, nil
|
||||
}
|
||||
in := func(args ...string) ([]byte, error) {
|
||||
cmd := exec.Command("git", args...)
|
||||
cmd.Dir = repo
|
||||
return cmd.Output()
|
||||
}
|
||||
if out, err := in("rev-parse", "--short", "HEAD"); err == nil {
|
||||
commit = strings.TrimSpace(string(out))
|
||||
}
|
||||
if out, err := in("status", "--porcelain"); err == nil {
|
||||
state := strings.TrimSpace(string(out)) != ""
|
||||
dirty = &state
|
||||
}
|
||||
return commit, dirty
|
||||
}
|
||||
|
||||
func fileSHA256(path string) (string, error) {
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return fmt.Sprintf("%x", sha256.Sum256(data)), nil
|
||||
}
|
||||
|
||||
func sortedKeys(set map[string]bool) []string {
|
||||
out := make([]string, 0, len(set))
|
||||
for k := range set {
|
||||
out = append(out, k)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// writeBatchManifest records the batch beside the runs it describes. It runs
|
||||
// last, so its digests are of what is already on disk.
|
||||
func writeBatchManifest(root string, info batchInfo, all []contextMetrics) error {
|
||||
ids, arms := map[string]bool{}, map[string]bool{}
|
||||
var prompt, completion, cacheHit, cacheMiss, requests, reported, failed int
|
||||
estimated := false
|
||||
for _, m := range all {
|
||||
// A failed run has no task, arm or evidence to contribute: it is counted
|
||||
// as a failure below and is not part of what the batch measured.
|
||||
if m.Failed {
|
||||
failed++
|
||||
continue
|
||||
}
|
||||
ids[m.Task] = true
|
||||
arms[m.Arm] = true
|
||||
if m.UsageReportedRequests > 0 {
|
||||
reported++
|
||||
requests += m.UsageReportedRequests
|
||||
prompt += m.PromptTokens
|
||||
completion += m.CompletionTokens
|
||||
cacheHit += m.CacheHitTokens
|
||||
cacheMiss += m.CacheMissTokens
|
||||
estimated = estimated || m.UsageEstimated
|
||||
}
|
||||
}
|
||||
commit, dirty := gitCommitAndDirty(info.Repo)
|
||||
model := runModel
|
||||
if info.Dry {
|
||||
model = "scripted"
|
||||
}
|
||||
manifest := batchManifest{
|
||||
SchemaVersion: 1, Mode: info.Mode,
|
||||
StartedAt: info.Started.UTC().Format(time.RFC3339),
|
||||
FinishedAt: time.Now().UTC().Format(time.RFC3339),
|
||||
GitCommit: commit, GitDirty: dirty,
|
||||
Model: model, Window: fixtureWindow, Generations: fixtureGenerations,
|
||||
TaskIDs: sortedKeys(ids), Arms: sortedKeys(arms),
|
||||
PlannedRuns: info.Planned, CompletedRuns: len(all) - failed, FailedRuns: failed,
|
||||
UsageReportedRuns: reported, UsageReportedRequests: requests,
|
||||
PromptTokens: prompt, CompletionTokens: completion,
|
||||
CacheHitTokens: cacheHit, CacheMissTokens: cacheMiss, UsageEstimated: estimated,
|
||||
}
|
||||
var err error
|
||||
if manifest.ResultsSHA256, err = fileSHA256(filepath.Join(root, "results.jsonl")); err != nil {
|
||||
return err
|
||||
}
|
||||
if manifest.TrajectoriesSHA256, err = fileSHA256(filepath.Join(root, "trajectories.jsonl")); err != nil {
|
||||
return err
|
||||
}
|
||||
data, err := json.MarshalIndent(manifest, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return os.WriteFile(filepath.Join(root, "batch.json"), append(data, '\n'), 0o644)
|
||||
}
|
||||
@@ -0,0 +1,91 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func seedArtifacts(t *testing.T, root string) {
|
||||
t.Helper()
|
||||
for _, name := range []string{"results.jsonl", "trajectories.jsonl"} {
|
||||
if err := os.WriteFile(filepath.Join(root, name), []byte("{}\n"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func readManifest(t *testing.T, root string) (batchManifest, string) {
|
||||
t.Helper()
|
||||
data, err := os.ReadFile(filepath.Join(root, "batch.json"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var got batchManifest
|
||||
if err := json.Unmarshal(data, &got); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return got, string(data)
|
||||
}
|
||||
|
||||
func TestBatchManifestRecordsWhatTheBatchWas(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
seedArtifacts(t, root)
|
||||
all := []contextMetrics{
|
||||
{Task: "b", Arm: "index-off"},
|
||||
{Task: "a", Arm: "index-off"},
|
||||
{Task: "a", Arm: "index-half", UsageReportedRequests: 2, PromptTokens: 100,
|
||||
CompletionTokens: 10, CacheHitTokens: 60, CacheMissTokens: 40},
|
||||
}
|
||||
if err := writeBatchManifest(root, batchInfo{Mode: "run-index", Planned: 4, Started: time.Now().Add(-time.Minute)}, all); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, _ := readManifest(t, root)
|
||||
if got.SchemaVersion != 1 || got.Mode != "run-index" || got.PlannedRuns != 4 || got.CompletedRuns != 3 {
|
||||
t.Fatalf("manifest header = %+v", got)
|
||||
}
|
||||
if strings.Join(got.TaskIDs, ",") != "a,b" || strings.Join(got.Arms, ",") != "index-half,index-off" {
|
||||
t.Fatalf("ids/arms = %v / %v, want sorted and unique", got.TaskIDs, got.Arms)
|
||||
}
|
||||
if got.UsageReportedRuns != 1 || got.UsageReportedRequests != 2 || got.PromptTokens != 100 ||
|
||||
got.CompletionTokens != 10 || got.CacheHitTokens != 60 || got.CacheMissTokens != 40 {
|
||||
t.Fatalf("usage totals = %+v", got)
|
||||
}
|
||||
// No repo means git could not describe the checkout: unknown commit, and no
|
||||
// dirty flag at all, because "clean" is a claim this batch cannot make.
|
||||
if got.GitCommit != "unknown" || got.GitDirty != nil {
|
||||
t.Fatalf("git state = %q / %v, want unknown and no dirty flag", got.GitCommit, got.GitDirty)
|
||||
}
|
||||
want := fmt.Sprintf("%x", sha256.Sum256([]byte("{}\n")))
|
||||
if got.ResultsSHA256 != want || got.TrajectoriesSHA256 != want {
|
||||
t.Fatalf("digests = %q / %q, want %q", got.ResultsSHA256, got.TrajectoriesSHA256, want)
|
||||
}
|
||||
}
|
||||
|
||||
// A dry batch pays nothing, so its cost fields must be absent rather than zero:
|
||||
// an unmeasured spend must not read as a measured one.
|
||||
func TestBatchManifestLeavesUnreportedUsageAbsent(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
seedArtifacts(t, root)
|
||||
all := []contextMetrics{{Task: "a", Arm: "index-off"}, {Task: "a", Arm: "index-half"}}
|
||||
if err := writeBatchManifest(root, batchInfo{Mode: "run-index", Planned: 2, Started: time.Now(), Dry: true}, all); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, raw := readManifest(t, root)
|
||||
if got.UsageReportedRuns != 0 {
|
||||
t.Fatalf("usage_reported_runs = %d, want 0 reported", got.UsageReportedRuns)
|
||||
}
|
||||
if got.Model != "scripted" {
|
||||
t.Fatalf("dry batch model = %q, want scripted", got.Model)
|
||||
}
|
||||
for _, absent := range []string{"usage_reported_requests", "prompt_tokens", "completion_tokens", "cache_hit_tokens"} {
|
||||
if strings.Contains(raw, absent) {
|
||||
t.Fatalf("a dry batch wrote %q:\n%s", absent, raw)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,69 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// boundaryRuns builds the input reportBoundaries reads: one scored run for an
|
||||
// index task's cue side, and one failed run for another index task's cue side.
|
||||
// The runs are constructed because the report is a pure function of the slice;
|
||||
// the batch path that produces them is covered by the failed-batch tests.
|
||||
func boundaryRuns(t *testing.T) (all []contextMetrics, scoredTask, failedTask string) {
|
||||
t.Helper()
|
||||
index := indexTasks()
|
||||
if len(index) < 2 {
|
||||
t.Fatalf("need at least two index tasks, have %d", len(index))
|
||||
}
|
||||
scored, failed := index[0], index[1]
|
||||
cue, noCue := boundaryPair(scored.CueTier)
|
||||
if cue == "" || noCue == "" {
|
||||
t.Fatalf("task %s has an empty boundary pair (%q, %q)", scored.ID, cue, noCue)
|
||||
}
|
||||
failedCue, _ := boundaryPair(failed.CueTier)
|
||||
if failedCue == "" {
|
||||
t.Fatalf("task %s has no cue side", failed.ID)
|
||||
}
|
||||
all = []contextMetrics{
|
||||
{Task: scored.ID, Arm: "index-" + cue, AnswerRecovered: true, SearchCalls: 1, RecallReturnedTokens: 318},
|
||||
{Task: scored.ID, Arm: "index-" + noCue, AnswerRecovered: false},
|
||||
{Task: failed.ID, Arm: "index-" + failedCue, Failed: true, FailureStage: stageNoEvidence},
|
||||
}
|
||||
return all, scored.ID, failed.ID
|
||||
}
|
||||
|
||||
// TestBoundaryReportExcludesFailedRuns: a run that never produced a transcript
|
||||
// must not appear in the cue-boundary table, which was the last reporting exit
|
||||
// that still folded it in as a zero measurement.
|
||||
func TestBoundaryReportExcludesFailedRuns(t *testing.T) {
|
||||
all, scoredTask, failedTask := boundaryRuns(t)
|
||||
out := captureStdout(t, func() { reportBoundaries(all) })
|
||||
if strings.Contains(out, failedTask) {
|
||||
t.Fatalf("the boundary table printed the failed task %q as a measurement:\n%s", failedTask, out)
|
||||
}
|
||||
if !strings.Contains(out, scoredTask) {
|
||||
t.Fatalf("the boundary table dropped the scored task %q, so the exclusion is not specific to failures:\n%s", scoredTask, out)
|
||||
}
|
||||
if !strings.Contains(out, "cue-present") || !strings.Contains(out, "cue-absent") {
|
||||
t.Fatalf("the pooled comparison disappeared:\n%s", out)
|
||||
}
|
||||
}
|
||||
|
||||
// TestBoundaryReportSaysNothingToCompareWhenEveryRunFailed: with no scored run
|
||||
// the report must say so, rather than printing a comparison of zeros.
|
||||
func TestBoundaryReportSaysNothingToCompareWhenEveryRunFailed(t *testing.T) {
|
||||
all, _, _ := boundaryRuns(t)
|
||||
failedOnly := make([]contextMetrics, 0, len(all))
|
||||
for _, m := range all {
|
||||
m.Failed = true
|
||||
m.FailureStage = stageNoEvidence
|
||||
failedOnly = append(failedOnly, m)
|
||||
}
|
||||
out := captureStdout(t, func() { reportBoundaries(failedOnly) })
|
||||
if !strings.Contains(out, "scored=0") || !strings.Contains(out, "failed=") {
|
||||
t.Fatalf("an all-failed batch did not report scored=0 with a failed count:\n%s", out)
|
||||
}
|
||||
if strings.Contains(out, "cue-present") || strings.Contains(out, "cue-absent") {
|
||||
t.Fatalf("an all-failed batch still printed a pooled comparison of zeros:\n%s", out)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,171 @@
|
||||
// capture.go — what the agent actually hands a provider.
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"reasonix/internal/state/sessionstore"
|
||||
"strings"
|
||||
|
||||
"reasonix/internal/assembly/boot"
|
||||
"reasonix/internal/contract/ablation"
|
||||
"reasonix/internal/contract/event"
|
||||
"reasonix/internal/contract/provider"
|
||||
"reasonix/internal/contract/tool"
|
||||
"reasonix/internal/runtime/agent"
|
||||
_ "reasonix/internal/tools/builtin"
|
||||
)
|
||||
|
||||
// A projection is what the agent believes; a provider.Request is what the model
|
||||
// is told. Rendering, sanitization and tool-schema construction sit between
|
||||
// them, so the arm's contract is checked on the request.
|
||||
|
||||
type captureProvider struct {
|
||||
requests []provider.Request
|
||||
}
|
||||
|
||||
func (p *captureProvider) Name() string { return "contextbench-capture" }
|
||||
|
||||
// Stream records the request and answers with a clean terminal response. An
|
||||
// empty answer would buy an empty-final retry and a second request, which is
|
||||
// not the one round this is here to inspect.
|
||||
func (p *captureProvider) Stream(_ context.Context, req provider.Request) (<-chan provider.Chunk, error) {
|
||||
p.requests = append(p.requests, cloneRequest(req))
|
||||
ch := make(chan provider.Chunk, 2)
|
||||
ch <- provider.Chunk{Type: provider.ChunkText, Text: "contextbench preflight complete"}
|
||||
ch <- provider.Chunk{Type: provider.ChunkDone}
|
||||
close(ch)
|
||||
return ch, nil
|
||||
}
|
||||
|
||||
func cloneRequest(req provider.Request) provider.Request {
|
||||
out := req
|
||||
out.Messages = append([]provider.Message(nil), req.Messages...)
|
||||
out.Tools = append([]provider.ToolSchema(nil), req.Tools...)
|
||||
return out
|
||||
}
|
||||
|
||||
// captureRequest rebuilds the session the way a resume does — transcript from
|
||||
// disk, projection from its sidecar, the shipped tool surface at this arm —
|
||||
// and returns the request the first round hands the provider.
|
||||
func captureRequest(f builtFixture, arm ablation.Set, prompt string) (provider.Request, error) {
|
||||
sess, err := sessionstore.LoadSession(f.Path)
|
||||
if err != nil {
|
||||
return provider.Request{}, fmt.Errorf("load session: %w", err)
|
||||
}
|
||||
reg := tool.NewRegistry()
|
||||
for _, t := range tool.Builtins() {
|
||||
reg.Add(t)
|
||||
}
|
||||
boot.ApplyUnifiedProviderToolSurface(reg, false, arm)
|
||||
|
||||
cap := &captureProvider{}
|
||||
a := agent.New(cap, reg, sess, agent.Options{
|
||||
ContextWindow: fixtureWindow, CompactRatio: 0.5, RecentKeep: 2,
|
||||
SessionPath: f.Path, KeepPolicy: agent.KeepErrors, Ablation: arm,
|
||||
}, event.Discard)
|
||||
a.LoadProjectionSidecar(f.Path)
|
||||
|
||||
if err := a.Run(context.Background(), prompt); err != nil {
|
||||
return provider.Request{}, fmt.Errorf("run: %w", err)
|
||||
}
|
||||
// The turn's own request is the one carrying tools. A fold on the way in
|
||||
// asks the same provider for a digest, and that request has none — taking
|
||||
// the first request blindly would inspect the summarizer's prompt.
|
||||
for _, req := range cap.requests {
|
||||
if len(req.Tools) > 0 {
|
||||
return req, nil
|
||||
}
|
||||
}
|
||||
return provider.Request{}, fmt.Errorf("no request carried tools (%d captured)", len(cap.requests))
|
||||
}
|
||||
|
||||
// surfaceField is one place a model could read a string, named so a leak
|
||||
// report says where rather than only that.
|
||||
type surfaceField struct {
|
||||
Path string
|
||||
Text string
|
||||
}
|
||||
|
||||
// providerSurface enumerates every provider-visible string in a request.
|
||||
// Enumerated rather than marshalled whole: a Message also carries local-only
|
||||
// fields, and scanning those would fail on bytes no model ever sees.
|
||||
func providerSurface(req provider.Request) []surfaceField {
|
||||
var out []surfaceField
|
||||
add := func(path, text string) {
|
||||
if strings.TrimSpace(text) != "" {
|
||||
out = append(out, surfaceField{Path: path, Text: text})
|
||||
}
|
||||
}
|
||||
for i, m := range req.Messages {
|
||||
add(fmt.Sprintf("messages[%d].content", i), m.Content)
|
||||
add(fmt.Sprintf("messages[%d].reasoning_content", i), m.ReasoningContent)
|
||||
add(fmt.Sprintf("messages[%d].name", i), m.Name)
|
||||
for j, tc := range m.ToolCalls {
|
||||
add(fmt.Sprintf("messages[%d].tool_calls[%d].name", i, j), tc.Name)
|
||||
add(fmt.Sprintf("messages[%d].tool_calls[%d].arguments", i, j), tc.Arguments)
|
||||
}
|
||||
for j, item := range m.ResponsesItems {
|
||||
add(fmt.Sprintf("messages[%d].responses_items[%d]", i, j), string(item))
|
||||
}
|
||||
}
|
||||
for i, t := range req.Tools {
|
||||
add(fmt.Sprintf("tools[%d].name", i), t.Name)
|
||||
add(fmt.Sprintf("tools[%d].description", i), t.Description)
|
||||
if len(t.Parameters) > 0 {
|
||||
b, err := json.Marshal(t.Parameters)
|
||||
if err == nil {
|
||||
add(fmt.Sprintf("tools[%d].parameters", i), string(b))
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// findInSurface reports every place a marker is readable.
|
||||
func findInSurface(req provider.Request, marker string) []string {
|
||||
var hits []string
|
||||
for _, f := range providerSurface(req) {
|
||||
if strings.Contains(f.Text, marker) {
|
||||
hits = append(hits, f.Path)
|
||||
}
|
||||
}
|
||||
return hits
|
||||
}
|
||||
|
||||
// recallToolSchema returns the recall tool as the provider sees it.
|
||||
func recallToolSchema(req provider.Request) (provider.ToolSchema, bool) {
|
||||
for _, t := range req.Tools {
|
||||
if t.Name == "recall" {
|
||||
return t, true
|
||||
}
|
||||
}
|
||||
return provider.ToolSchema{}, false
|
||||
}
|
||||
|
||||
// localOnlyLeaks reports host-side fields that should never survive the
|
||||
// provider boundary. A hit here is a boundary regression, marker or no marker.
|
||||
func localOnlyLeaks(req provider.Request) []string {
|
||||
var out []string
|
||||
for i, m := range req.Messages {
|
||||
switch {
|
||||
case m.RawContent != "":
|
||||
out = append(out, fmt.Sprintf("messages[%d].raw_content", i))
|
||||
case m.LocalOnly:
|
||||
out = append(out, fmt.Sprintf("messages[%d].local_only", i))
|
||||
case m.DecisionReceipt != nil || len(m.DecisionReceipts) > 0:
|
||||
out = append(out, fmt.Sprintf("messages[%d].decision_receipt", i))
|
||||
case m.ToolExecution != nil:
|
||||
out = append(out, fmt.Sprintf("messages[%d].tool_execution", i))
|
||||
case len(m.MemoryCitations) > 0:
|
||||
out = append(out, fmt.Sprintf("messages[%d].memory_citations", i))
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// searchWording is what a search-off arm must never be told. The arm exists to
|
||||
// reproduce the runtime before search; an arm offered a capability it does not
|
||||
// have is a different experiment.
|
||||
var searchWording = []string{"recall with a query", "search the whole folded region", "query still finds"}
|
||||
@@ -0,0 +1,671 @@
|
||||
// contextbench.go — what the model did with the affordance, read off structure.
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"regexp"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"reasonix/internal/base/tokencount"
|
||||
"reasonix/internal/contract/provider"
|
||||
)
|
||||
|
||||
// Every judgement here reads a tool call's name and its JSON arguments, or a
|
||||
// canonical position, and never the model's prose. "I will search the history"
|
||||
// is not a search; a recall call with a query is.
|
||||
|
||||
// failure stages, in funnel order. Exactly one is reported per run, except for
|
||||
// NoEvidence: that stage belongs to a run with no measurement at all.
|
||||
const (
|
||||
stageReadMissed = "NoSearch" // never asked, and never read the target either
|
||||
stageSearchMiss = "SearchMiss" // searched, target never came back
|
||||
stageHitNotRead = "HitNotRead" // target was returned and never opened
|
||||
stageAnswerWrong = "ReadButAnswerWrong" // opened it and still answered wrong
|
||||
stageRecovered = "Recovered"
|
||||
stageContaminated = "Contaminated"
|
||||
stageNoEvidence = "NoEvidence"
|
||||
|
||||
// Routing is judged by model round, not by the linear order of tool events:
|
||||
// two calls in one round are one parallel fan-out, and calling that
|
||||
// "workspace first" would report a choice the model never made.
|
||||
routeMemoryFirst = "MemoryFirst"
|
||||
routeWorkspaceFirst = "WorkspaceFirst"
|
||||
routeParallel = "Parallel"
|
||||
routeNeither = "Neither"
|
||||
stageCueRead = "DirectRead" // read the target from an index address, no search needed
|
||||
stageNoRetrieval = "NoRetrieval" // answered without touching recall at all
|
||||
toolRecall = "recall"
|
||||
)
|
||||
|
||||
// searchAttempt is one query the model wrote, and what it got back. Kept per
|
||||
// attempt rather than aggregated: the question is why a retrieval that the host
|
||||
// probe ranks first takes the model more than one try.
|
||||
type searchAttempt struct {
|
||||
Ordinal int `json:"ordinal"`
|
||||
Round int `json:"round"`
|
||||
Query string `json:"query"`
|
||||
TargetRank int `json:"target_rank"` // 0 = the target did not come back
|
||||
ResultCount int `json:"result_count"`
|
||||
// Reformulation classifies this query against the one before it, and
|
||||
// MissingProbeTerms are the host probe's terms this query left out. Both
|
||||
// are derived from token sets: no judge, and nothing read out of prose.
|
||||
Reformulation string `json:"reformulation,omitempty"`
|
||||
MissingProbeTerms []string `json:"missing_probe_terms,omitempty"`
|
||||
}
|
||||
|
||||
// contextMetrics is one run's behaviour.
|
||||
type contextMetrics struct {
|
||||
Task string `json:"task"`
|
||||
Arm string `json:"arm"`
|
||||
|
||||
SearchCalls int `json:"search_calls"`
|
||||
// SearchHits counts searches that returned anything; TargetSearchHits
|
||||
// counts searches that returned the target. The gap is query quality.
|
||||
SearchHits int `json:"search_hits"`
|
||||
TargetSearchHits int `json:"target_search_hits"`
|
||||
FirstTargetRank int `json:"first_target_rank"` // 0 = never returned
|
||||
// FirstHitCoverage: how much of the answer the first hit handed over.
|
||||
// rank=1 with coverage 1/3 is a window too short to answer with, not a
|
||||
// model that would not stop.
|
||||
FirstHitCoverage int `json:"first_hit_coverage"`
|
||||
AnswerValues int `json:"answer_values"`
|
||||
|
||||
ReadCalls int `json:"read_calls"`
|
||||
TargetRead bool `json:"target_read"`
|
||||
ReadAfterHit bool `json:"read_after_hit"`
|
||||
// RecallReadWithoutSearch: the index addressed the target, so no search was
|
||||
// needed. Not "direct read" — read_file is a read too, and in an audit the
|
||||
// two mean opposite things.
|
||||
RecallReadWithoutSearch bool `json:"recall_read_without_search"`
|
||||
SearchThenRecallRead bool `json:"search_then_recall_read"`
|
||||
|
||||
FirstSearchRound int `json:"first_search_round"`
|
||||
FirstReadRound int `json:"first_read_round"`
|
||||
RetrievalRounds int `json:"retrieval_rounds"`
|
||||
|
||||
// RecallReturnedTokens is what recall paged back in. The index saves
|
||||
// resident prefix; this is what is paid dynamically instead.
|
||||
RecallReturnedTokens int `json:"recall_returned_tokens"`
|
||||
|
||||
// Provider usage as reported on the stream, or empty for a dry run. The count
|
||||
// is completions that reported usage, so it is not a count of attempts.
|
||||
UsageReportedRequests int `json:"usage_reported_requests,omitempty"`
|
||||
PromptTokens int `json:"prompt_tokens,omitempty"`
|
||||
CompletionTokens int `json:"completion_tokens,omitempty"`
|
||||
CacheHitTokens int `json:"cache_hit_tokens,omitempty"`
|
||||
CacheMissTokens int `json:"cache_miss_tokens,omitempty"`
|
||||
UsageEstimated bool `json:"usage_estimated,omitempty"`
|
||||
|
||||
Searches []searchAttempt `json:"searches,omitempty"`
|
||||
// PostHitSearches are queries issued after the target had already been
|
||||
// returned: not a query problem, a result-to-read problem.
|
||||
PostHitSearches int `json:"post_hit_searches"`
|
||||
|
||||
AnswerRecovered bool `json:"answer_recovered"`
|
||||
// AnswerAmbiguous marks an answer that offered more than one candidate for
|
||||
// a value. Markers matching proves the right fact appears; it does not
|
||||
// prove it was submitted as the answer. Recorded, never acted on.
|
||||
AnswerAmbiguous bool `json:"answer_ambiguous,omitempty"`
|
||||
AmbiguousVar string `json:"ambiguous_var,omitempty"`
|
||||
MissingMarkers []string `json:"missing_markers,omitempty"`
|
||||
// FinalAnswer is kept bounded so a surprising run can be read back. A
|
||||
// metric nobody can audit is a claim, not a measurement.
|
||||
FinalAnswer string `json:"final_answer,omitempty"`
|
||||
FailureStage string `json:"failure_stage"`
|
||||
TimeoutReason string `json:"timeout_reason,omitempty"`
|
||||
|
||||
// Failed marks a run that ended before a transcript existed: it has no score
|
||||
// and no place in the statistics. The row stays in results.jsonl with its
|
||||
// task, arm and stage, so a failure is counted rather than dropped.
|
||||
Failed bool `json:"failed,omitempty"`
|
||||
Reason string `json:"reason,omitempty"`
|
||||
|
||||
// UnexpectedWorkTools are investigation tools used instead of recall.
|
||||
// Reaching for the workspace is behaviour, not noise: how a model routes
|
||||
// its uncertainty when memory is missing is part of what is measured.
|
||||
UnexpectedWorkTools []string `json:"unexpected_work_tools,omitempty"`
|
||||
EscapeCalls int `json:"escape_calls"`
|
||||
EscapeBeforeFirstRecall int `json:"escape_before_first_recall"`
|
||||
EscapeAfterFirstRecall int `json:"escape_after_first_recall"`
|
||||
// Contaminated: the answer was reachable through something other than
|
||||
// recall. Asserting this beats trusting the isolation, and a run that trips
|
||||
// it leaves the statistics rather than inflating one.
|
||||
Contaminated bool `json:"contaminated,omitempty"`
|
||||
LeakedVia string `json:"leaked_via,omitempty"`
|
||||
LeakedMarker string `json:"leaked_marker,omitempty"`
|
||||
LeakedArgs string `json:"leaked_args,omitempty"`
|
||||
LeakedOutput string `json:"leaked_output,omitempty"`
|
||||
// EscapeFoundAnswer separates "looked at the workspace" from "found the
|
||||
// answer there": different facts about the same call.
|
||||
EscapeFoundAnswer bool `json:"escape_found_answer,omitempty"`
|
||||
|
||||
CueVisible bool `json:"cue_visible"`
|
||||
|
||||
// trajectory is kept out of the metric line and written beside it, so a
|
||||
// future question can be asked of this batch instead of a new one.
|
||||
trajectory []provider.Message
|
||||
|
||||
// CueDirectRead: the cue was addressed, the first memory action was a
|
||||
// positions read, and it covered the target. Crediting any positions call
|
||||
// would hand the index a search's work.
|
||||
CueDirectRead bool `json:"cue_direct_read"`
|
||||
|
||||
// Sufficiency, not hit. A search that returns the target at rank 1 has not
|
||||
// necessarily returned the answer: the snippet is bounded. The stopping
|
||||
// boundary is the round where everything needed was readable.
|
||||
FirstSufficientRound int `json:"first_sufficient_round"`
|
||||
SufficientVia string `json:"sufficient_via"` // snippet | read | never
|
||||
RoundsAfterSufficient int `json:"rounds_after_sufficient"`
|
||||
SearchesAfterSufficient int `json:"searches_after_sufficient"`
|
||||
ReadsAfterSufficient int `json:"reads_after_sufficient"`
|
||||
EscapesAfterSufficient int `json:"escapes_after_sufficient"`
|
||||
RecallTokensAfterSufficient int `json:"recall_tokens_after_sufficient"`
|
||||
StoppingClass string `json:"stopping_class"`
|
||||
|
||||
// What happened after the answer was already in hand. A timeout that spent
|
||||
// seven rounds past its target is a stopping problem, and extending the
|
||||
// budget would only buy the loop more time.
|
||||
TargetFirstSeenRound int `json:"target_first_seen_round"`
|
||||
RoundsAfterTarget int `json:"rounds_after_target"`
|
||||
SearchesAfterTarget int `json:"searches_after_target"`
|
||||
RecallTokensAfterTarget int `json:"recall_tokens_after_target"`
|
||||
TargetHeldAtEnd bool `json:"target_held_at_end"`
|
||||
|
||||
// Routing is where the model looked first when its memory was missing.
|
||||
Routing string `json:"routing"`
|
||||
FirstRecallRound int `json:"first_recall_round"`
|
||||
FirstEscapeRound int `json:"first_escape_round"`
|
||||
}
|
||||
|
||||
// hitPosition matches the address recall prints for a search hit. Reading our
|
||||
// own rendered format, not guessing at the model's words.
|
||||
var hitPosition = regexp.MustCompile(`(?m)^#(\d+) `)
|
||||
|
||||
// scoreRun reads the messages one turn appended and reports what happened.
|
||||
// runScorer walks one turn's messages, holding the state a judgement needs:
|
||||
// which calls are open, whether the target has come back, and which search
|
||||
// each result belongs to.
|
||||
type runScorer struct {
|
||||
m contextMetrics
|
||||
inst fixtureInstance
|
||||
target int
|
||||
calls map[string]provider.ToolCall
|
||||
pending map[string]int
|
||||
round int
|
||||
sawTargetHit bool
|
||||
// seen accumulates everything recall has shown the model, so sufficiency is
|
||||
// judged on what it could read rather than on which call produced it.
|
||||
seen strings.Builder
|
||||
sufficient bool
|
||||
}
|
||||
|
||||
// scoreRun reads the messages one turn appended and reports what happened.
|
||||
func scoreRun(msgs []provider.Message, inst fixtureInstance, target int, arm string, cueVisible bool) contextMetrics {
|
||||
s := &runScorer{
|
||||
m: contextMetrics{Task: inst.Task.ID, Arm: arm, CueVisible: cueVisible},
|
||||
inst: inst, target: target,
|
||||
calls: map[string]provider.ToolCall{}, pending: map[string]int{},
|
||||
}
|
||||
for _, msg := range msgs {
|
||||
if len(msg.ToolCalls) > 0 {
|
||||
s.round++
|
||||
}
|
||||
for _, tc := range msg.ToolCalls {
|
||||
s.calls[tc.ID] = tc
|
||||
s.observeCall(tc)
|
||||
}
|
||||
if msg.Role == provider.RoleTool {
|
||||
s.observeResult(msg)
|
||||
}
|
||||
}
|
||||
return s.finish(cueVisible)
|
||||
}
|
||||
|
||||
// observeCall records one tool call: a recall search, a recall read, or a
|
||||
// reach for the workspace.
|
||||
func (s *runScorer) observeCall(tc provider.ToolCall) {
|
||||
m := &s.m
|
||||
if tc.Name != toolRecall {
|
||||
if !containsString(m.UnexpectedWorkTools, tc.Name) {
|
||||
m.UnexpectedWorkTools = append(m.UnexpectedWorkTools, tc.Name)
|
||||
}
|
||||
m.EscapeCalls++
|
||||
if s.sufficient {
|
||||
m.EscapesAfterSufficient++
|
||||
}
|
||||
if m.FirstEscapeRound == 0 {
|
||||
m.FirstEscapeRound = s.round
|
||||
}
|
||||
if m.SearchCalls+m.ReadCalls == 0 {
|
||||
m.EscapeBeforeFirstRecall++
|
||||
} else {
|
||||
m.EscapeAfterFirstRecall++
|
||||
}
|
||||
return
|
||||
}
|
||||
if m.FirstRecallRound == 0 {
|
||||
m.FirstRecallRound = s.round
|
||||
}
|
||||
query, positions := recallArgs(tc.Arguments)
|
||||
switch {
|
||||
case query != "":
|
||||
m.SearchCalls++
|
||||
if m.FirstSearchRound == 0 {
|
||||
m.FirstSearchRound = s.round
|
||||
}
|
||||
if s.sawTargetHit {
|
||||
m.PostHitSearches++
|
||||
m.SearchesAfterTarget++
|
||||
}
|
||||
if s.sufficient {
|
||||
m.SearchesAfterSufficient++
|
||||
}
|
||||
m.Searches = append(m.Searches, searchAttempt{Ordinal: m.SearchCalls, Round: s.round, Query: query})
|
||||
s.pending[tc.ID] = len(m.Searches) - 1
|
||||
case len(positions) > 0:
|
||||
m.ReadCalls++
|
||||
if s.sufficient {
|
||||
m.ReadsAfterSufficient++
|
||||
}
|
||||
if m.FirstReadRound == 0 {
|
||||
m.FirstReadRound = s.round
|
||||
}
|
||||
// A read covers the target when it names its position: recall returns
|
||||
// the tool results under the caller's own address.
|
||||
if containsInt(positions, s.target) {
|
||||
m.TargetRead = true
|
||||
switch {
|
||||
case s.sawTargetHit:
|
||||
m.ReadAfterHit, m.SearchThenRecallRead = true, true
|
||||
case m.SearchCalls == 0:
|
||||
m.RecallReadWithoutSearch = true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// observeResult reads what a call returned: contamination, page-in cost, and
|
||||
// whether this search brought the target back.
|
||||
func (s *runScorer) observeResult(msg provider.Message) {
|
||||
m := &s.m
|
||||
call, ok := s.calls[msg.ToolCallID]
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
if call.Name != toolRecall {
|
||||
// Codenames only: a three-digit epoch appears in any long file by
|
||||
// chance, and one did. A model holding the codename holds the numbers
|
||||
// beside it anyway.
|
||||
for _, marker := range s.inst.AnswerMarkers {
|
||||
if len(marker) >= 6 && strings.Contains(msg.Content, marker) {
|
||||
m.Contaminated, m.LeakedVia, m.LeakedMarker = true, call.Name, marker
|
||||
m.EscapeFoundAnswer = true
|
||||
m.LeakedArgs, m.LeakedOutput = truncate(call.Arguments, 200), truncate(msg.Content, 300)
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
cost := tokencount.Text(msg.Content)
|
||||
m.RecallReturnedTokens += cost
|
||||
if s.sawTargetHit {
|
||||
m.RecallTokensAfterTarget += cost
|
||||
}
|
||||
if s.sufficient {
|
||||
m.RecallTokensAfterSufficient += cost
|
||||
}
|
||||
// Everything recall showed the model, judged as a whole: the answer may
|
||||
// arrive across a snippet and a read, and either way it is readable.
|
||||
s.seen.WriteString(msg.Content)
|
||||
s.seen.WriteString("\n")
|
||||
if !s.sufficient && allMarkersIn(s.seen.String(), s.inst.AnswerMarkers) {
|
||||
s.sufficient = true
|
||||
m.FirstSufficientRound = s.round
|
||||
query, _ := recallArgs(call.Arguments)
|
||||
m.SufficientVia = "read"
|
||||
if query != "" {
|
||||
m.SufficientVia = "snippet"
|
||||
}
|
||||
}
|
||||
if query, _ := recallArgs(call.Arguments); query == "" {
|
||||
return
|
||||
}
|
||||
positions := searchHitPositions(msg.Content)
|
||||
if len(positions) > 0 {
|
||||
m.SearchHits++
|
||||
}
|
||||
rank := indexOfInt(positions, s.target)
|
||||
if i, ok := s.pending[msg.ToolCallID]; ok {
|
||||
m.Searches[i].TargetRank, m.Searches[i].ResultCount = rank, len(positions)
|
||||
}
|
||||
if rank == 0 {
|
||||
return
|
||||
}
|
||||
m.TargetSearchHits++
|
||||
if !s.sawTargetHit {
|
||||
m.TargetFirstSeenRound = s.round
|
||||
m.FirstHitCoverage = answerCoverage(msg.Content, s.inst.AnswerMarkers)
|
||||
}
|
||||
s.sawTargetHit = true
|
||||
if m.FirstTargetRank == 0 {
|
||||
m.FirstTargetRank = rank
|
||||
}
|
||||
}
|
||||
|
||||
// finish derives what only the whole walk can say.
|
||||
func (s *runScorer) finish(cueVisible bool) contextMetrics {
|
||||
m := s.m
|
||||
m.RetrievalRounds = m.SearchCalls + m.ReadCalls
|
||||
m.TargetHeldAtEnd = s.sawTargetHit || m.TargetRead
|
||||
if m.TargetFirstSeenRound > 0 {
|
||||
m.RoundsAfterTarget = s.round - m.TargetFirstSeenRound
|
||||
}
|
||||
m.Routing = routing(m.FirstRecallRound, m.FirstEscapeRound)
|
||||
m.CueDirectRead = cueVisible && m.RecallReadWithoutSearch
|
||||
if m.SufficientVia == "" {
|
||||
m.SufficientVia = "never"
|
||||
}
|
||||
if m.FirstSufficientRound > 0 {
|
||||
m.RoundsAfterSufficient = s.round - m.FirstSufficientRound
|
||||
}
|
||||
m.AnswerValues = len(s.inst.AnswerMarkers)
|
||||
m.StoppingClass = stoppingClass(m)
|
||||
classifySearches(&m, s.inst)
|
||||
return m
|
||||
}
|
||||
|
||||
// scoreAnswer grades the final text on the task's markers. Nonces, matched
|
||||
// exactly: no judge model, and nothing a general answer could satisfy.
|
||||
func (m *contextMetrics) scoreAnswer(final string, inst fixtureInstance) {
|
||||
lower := strings.ToLower(final)
|
||||
for _, marker := range inst.AnswerMarkers {
|
||||
if !strings.Contains(lower, strings.ToLower(marker)) {
|
||||
m.MissingMarkers = append(m.MissingMarkers, marker)
|
||||
}
|
||||
}
|
||||
m.AnswerRecovered = len(m.MissingMarkers) == 0
|
||||
m.AnswerAmbiguous, m.AmbiguousVar = ambiguousAnswer(final, inst)
|
||||
m.FailureStage = m.stage()
|
||||
if len(final) > 1200 {
|
||||
final = final[:1200] + "…"
|
||||
}
|
||||
m.FinalAnswer = final
|
||||
}
|
||||
|
||||
// stage places the run in the funnel. One label, chosen at the first step that
|
||||
// did not happen, so a table of stages reads as where the retrieval broke.
|
||||
func (m contextMetrics) stage() string {
|
||||
switch {
|
||||
case m.Contaminated:
|
||||
return stageContaminated
|
||||
case m.AnswerRecovered && m.RecallReadWithoutSearch:
|
||||
return stageCueRead
|
||||
case m.AnswerRecovered:
|
||||
return stageRecovered
|
||||
case m.TargetRead:
|
||||
return stageAnswerWrong
|
||||
case m.TargetSearchHits > 0:
|
||||
return stageHitNotRead
|
||||
case m.SearchCalls > 0:
|
||||
return stageSearchMiss
|
||||
case m.ReadCalls > 0:
|
||||
return stageReadMissed
|
||||
default:
|
||||
return stageNoRetrieval
|
||||
}
|
||||
}
|
||||
|
||||
// recallArgs pulls the two fields that say which operation a recall call is.
|
||||
func recallArgs(args string) (query string, positions []int) {
|
||||
var parsed struct {
|
||||
Query string `json:"query"`
|
||||
Positions []int `json:"positions"`
|
||||
}
|
||||
if err := json.Unmarshal([]byte(args), &parsed); err != nil {
|
||||
return "", nil
|
||||
}
|
||||
return strings.TrimSpace(parsed.Query), parsed.Positions
|
||||
}
|
||||
|
||||
// searchHitPositions reads the addresses out of a search result, in rank order.
|
||||
func searchHitPositions(text string) []int {
|
||||
var out []int
|
||||
for _, match := range hitPosition.FindAllStringSubmatch(text, -1) {
|
||||
if n, err := strconv.Atoi(match[1]); err == nil {
|
||||
out = append(out, n)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func containsInt(list []int, want int) bool { return slices.Contains(list, want) }
|
||||
|
||||
// indexOfInt returns a 1-based rank, or 0 when absent.
|
||||
func indexOfInt(list []int, want int) int {
|
||||
for i, v := range list {
|
||||
if v == want {
|
||||
return i + 1
|
||||
}
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func containsString(list []string, want string) bool { return slices.Contains(list, want) }
|
||||
|
||||
// funnel summarizes one arm's runs the way the experiment is read.
|
||||
type funnel struct {
|
||||
Arm string
|
||||
Runs int
|
||||
Scored, Contaminated, Errored int
|
||||
Searched, TargetHit, TargetRead, Answered int
|
||||
CueReads int
|
||||
SearchCalls, ReadCalls, RecallTokens int
|
||||
Escapes, EscapeCalls int
|
||||
Stages, Routes map[string]int
|
||||
}
|
||||
|
||||
func summarize(arm string, runs []contextMetrics) funnel {
|
||||
f := funnel{Arm: arm, Runs: len(runs), Stages: map[string]int{}, Routes: map[string]int{}}
|
||||
for _, r := range runs {
|
||||
// A run that never produced a transcript is not a scored run and has no
|
||||
// stage to place: counting it would put an empty name in the funnel, the
|
||||
// stage table and the routes.
|
||||
if r.Failed {
|
||||
continue
|
||||
}
|
||||
if r.Contaminated {
|
||||
f.Contaminated++
|
||||
f.Stages[r.FailureStage]++
|
||||
continue
|
||||
}
|
||||
if r.FailureStage == "RunError" {
|
||||
f.Errored++
|
||||
f.Stages[r.FailureStage]++
|
||||
continue
|
||||
}
|
||||
f.Scored++
|
||||
if r.SearchCalls > 0 {
|
||||
f.Searched++
|
||||
}
|
||||
if r.TargetSearchHits > 0 {
|
||||
f.TargetHit++
|
||||
}
|
||||
if r.TargetRead {
|
||||
f.TargetRead++
|
||||
}
|
||||
if r.AnswerRecovered {
|
||||
f.Answered++
|
||||
}
|
||||
if r.RecallReadWithoutSearch {
|
||||
f.CueReads++
|
||||
}
|
||||
if len(r.UnexpectedWorkTools) > 0 {
|
||||
f.Escapes++
|
||||
}
|
||||
f.EscapeCalls += r.EscapeCalls
|
||||
f.SearchCalls += r.SearchCalls
|
||||
f.ReadCalls += r.ReadCalls
|
||||
f.RecallTokens += r.RecallReturnedTokens
|
||||
f.Stages[r.FailureStage]++
|
||||
if r.FailureStage != "RunError" {
|
||||
f.Routes[r.Routing]++
|
||||
}
|
||||
}
|
||||
return f
|
||||
}
|
||||
|
||||
func (f funnel) line() string {
|
||||
per := func(n int) float64 {
|
||||
if f.Scored == 0 {
|
||||
return 0
|
||||
}
|
||||
return float64(n) / float64(f.Scored)
|
||||
}
|
||||
line := fmt.Sprintf("%-14s runs=%-3d answered=%d/%d searched=%d target-hit=%d cue-read=%d search/task=%.2f recall-tok/task=%.0f escape/task=%.2f",
|
||||
f.Arm, f.Scored, f.Answered, f.Scored, f.Searched, f.TargetHit, f.CueReads,
|
||||
per(f.SearchCalls), per(f.RecallTokens), per(f.EscapeCalls))
|
||||
if f.Contaminated > 0 {
|
||||
line += fmt.Sprintf(" CONTAMINATED=%d", f.Contaminated)
|
||||
}
|
||||
if f.Errored > 0 {
|
||||
line += fmt.Sprintf(" ERRORED=%d", f.Errored)
|
||||
}
|
||||
return line
|
||||
}
|
||||
|
||||
func writeJSONLine(w io.Writer, v any) error {
|
||||
b, err := json.Marshal(v)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = fmt.Fprintln(w, string(b))
|
||||
return err
|
||||
}
|
||||
|
||||
func truncate(s string, n int) string {
|
||||
if len(s) <= n {
|
||||
return s
|
||||
}
|
||||
return s[:n] + "…"
|
||||
}
|
||||
|
||||
// routing says where the model looked first, comparing rounds rather than
|
||||
// event order. Same round is a parallel fan-out, not a preference.
|
||||
func routing(recallRound, escapeRound int) string {
|
||||
switch {
|
||||
case recallRound == 0 && escapeRound == 0:
|
||||
return routeNeither
|
||||
case recallRound == 0:
|
||||
return routeWorkspaceFirst
|
||||
case escapeRound == 0:
|
||||
return routeMemoryFirst
|
||||
case recallRound < escapeRound:
|
||||
return routeMemoryFirst
|
||||
case escapeRound < recallRound:
|
||||
return routeWorkspaceFirst
|
||||
default:
|
||||
return routeParallel
|
||||
}
|
||||
}
|
||||
|
||||
// ambiguousAnswer reports an answer naming more than one candidate of a
|
||||
// codename's shape. Only codenames are checked: they carry a stem this run
|
||||
// declared, so two distinct "fence-…" values in one answer is one answer too
|
||||
// many. Numbers have no such shape and are left alone.
|
||||
func ambiguousAnswer(final string, inst fixtureInstance) (bool, string) {
|
||||
for _, spec := range inst.Task.Vars {
|
||||
if spec.Kind != varCodename {
|
||||
continue
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
for _, match := range regexp.MustCompile(regexp.QuoteMeta(spec.Word)+`-[a-z0-9]{3,}`).FindAllString(final, -1) {
|
||||
seen[match] = true
|
||||
}
|
||||
if len(seen) > 1 {
|
||||
return true, spec.Name
|
||||
}
|
||||
}
|
||||
return false, ""
|
||||
}
|
||||
|
||||
func allMarkersIn(text string, markers []string) bool {
|
||||
if len(markers) == 0 {
|
||||
return false
|
||||
}
|
||||
lower := strings.ToLower(text)
|
||||
for _, marker := range markers {
|
||||
if !strings.Contains(lower, strings.ToLower(marker)) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// Stopping classes. The boundary is sufficiency, not the target coming back:
|
||||
// a rank-1 hit whose snippet was too short to answer leaves a read entirely
|
||||
// justified, and counting that as waste would blame the model for the bound.
|
||||
const (
|
||||
stopSnippet = "SnippetStop" // enough arrived, and the turn ended
|
||||
stopSnippetRead = "SnippetThenRead" // the snippet was short; the read finished it
|
||||
stopDefensiveRead = "DefensiveRead" // enough had arrived, one read confirmed it
|
||||
stopPostRetrieval = "PostSufficientRetrieval" // enough had arrived, and retrieval continued
|
||||
stopRunaway = "PostSufficientRunaway" // it continued for rounds, or ran out of budget
|
||||
stopNeverEnough = "NeverSufficient"
|
||||
)
|
||||
|
||||
// runawayRounds is where continuing stops looking like confirmation.
|
||||
const runawayRounds = 3
|
||||
|
||||
func stoppingClass(m contextMetrics) string {
|
||||
if m.FirstSufficientRound == 0 {
|
||||
return stopNeverEnough
|
||||
}
|
||||
after := m.SearchesAfterSufficient + m.ReadsAfterSufficient + m.EscapesAfterSufficient
|
||||
switch {
|
||||
case after == 0 && m.SufficientVia == "snippet":
|
||||
return stopSnippet
|
||||
case after == 0:
|
||||
return stopSnippetRead
|
||||
case m.RoundsAfterSufficient >= runawayRounds || m.FailureStage == "RunError":
|
||||
return stopRunaway
|
||||
case after == 1 && m.ReadsAfterSufficient == 1:
|
||||
return stopDefensiveRead
|
||||
default:
|
||||
return stopPostRetrieval
|
||||
}
|
||||
}
|
||||
|
||||
// reportStopping is the funnel that matters once retrieval succeeds: what the
|
||||
// turn did after it already had everything it needed.
|
||||
func reportStopping(all []contextMetrics) string {
|
||||
classes := map[string]int{}
|
||||
var extraRounds, extraTokens []float64
|
||||
sufficient, answeredNext := 0, 0
|
||||
for _, m := range all {
|
||||
if m.Failed || m.Contaminated {
|
||||
continue
|
||||
}
|
||||
classes[m.StoppingClass]++
|
||||
if m.FirstSufficientRound == 0 {
|
||||
continue
|
||||
}
|
||||
sufficient++
|
||||
extraRounds = append(extraRounds, float64(m.RoundsAfterSufficient))
|
||||
extraTokens = append(extraTokens, float64(m.RecallTokensAfterSufficient))
|
||||
if m.RoundsAfterSufficient <= 1 {
|
||||
answeredNext++
|
||||
}
|
||||
}
|
||||
var b strings.Builder
|
||||
b.WriteString("\n## After the evidence was already enough\n")
|
||||
fmt.Fprintf(&b, " sufficient in %d runs; ended within one round in %d\n", sufficient, answeredNext)
|
||||
fmt.Fprintf(&b, " extra rounds mean %.1f median %.1f p90 %.0f max %.0f\n",
|
||||
mean(extraRounds), median(extraRounds), percentile(extraRounds, 0.9), maxOf(extraRounds))
|
||||
fmt.Fprintf(&b, " extra recall tokens mean %.0f median %.0f p90 %.0f max %.0f\n",
|
||||
mean(extraTokens), median(extraTokens), percentile(extraTokens, 0.9), maxOf(extraTokens))
|
||||
b.WriteString(" " + countsLine(classes) + "\n")
|
||||
return b.String()
|
||||
}
|
||||
@@ -0,0 +1,337 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"reasonix/internal/contract/provider"
|
||||
)
|
||||
|
||||
// The scorer decides what the pilot concludes, so it is held to the same rule
|
||||
// the corpus is: judgements read structure. A model that says it will search
|
||||
// has not searched; a recall call carrying a query has.
|
||||
|
||||
const scorerTarget = 42
|
||||
|
||||
func recallCall(id, args string) provider.Message {
|
||||
return provider.Message{Role: provider.RoleAssistant, ToolCalls: []provider.ToolCall{
|
||||
{ID: id, Name: "recall", Arguments: args},
|
||||
}}
|
||||
}
|
||||
|
||||
func toolResult(id, text string) provider.Message {
|
||||
return provider.Message{Role: provider.RoleTool, ToolCallID: id, Name: "recall", Content: text}
|
||||
}
|
||||
|
||||
func hitBlock(positions ...int) string {
|
||||
var b strings.Builder
|
||||
b.WriteString("Folded context matching \"x\":\n")
|
||||
for _, p := range positions {
|
||||
fmt.Fprintf(&b, "\n#%d assistant_text\nsome snippet here\n", p)
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
func scorerTask() fixtureInstance {
|
||||
return fixtureInstance{Task: contextTask{ID: "t"}, AnswerMarkers: []string{"cobalt-lark-17"}}
|
||||
}
|
||||
|
||||
func score(t *testing.T, msgs []provider.Message, final string) contextMetrics {
|
||||
t.Helper()
|
||||
m := scoreRun(msgs, scorerTask(), scorerTarget, "arm", false)
|
||||
m.scoreAnswer(final, scorerTask())
|
||||
return m
|
||||
}
|
||||
|
||||
// The whole funnel, walked end to end.
|
||||
func TestScorerReadsTheFullRetrievalChain(t *testing.T) {
|
||||
m := score(t, []provider.Message{
|
||||
recallCall("c1", `{"query":"retry boundary"}`),
|
||||
toolResult("c1", hitBlock(7, scorerTarget)),
|
||||
recallCall("c2", fmt.Sprintf(`{"positions":[%d]}`, scorerTarget)),
|
||||
toolResult("c2", "#42\n[assistant]\nthe ownership token is cobalt-lark-17"),
|
||||
{Role: provider.RoleAssistant, Content: "The token was cobalt-lark-17."},
|
||||
}, "The token was cobalt-lark-17.")
|
||||
|
||||
if m.SearchCalls != 1 || m.ReadCalls != 1 {
|
||||
t.Errorf("calls = %d search / %d read, want 1/1", m.SearchCalls, m.ReadCalls)
|
||||
}
|
||||
if m.TargetSearchHits != 1 || m.FirstTargetRank != 2 {
|
||||
t.Errorf("target hit %d at rank %d, want 1 at rank 2", m.TargetSearchHits, m.FirstTargetRank)
|
||||
}
|
||||
if !m.TargetRead || !m.ReadAfterHit || m.RecallReadWithoutSearch {
|
||||
t.Errorf("read flags = read %v afterHit %v direct %v", m.TargetRead, m.ReadAfterHit, m.RecallReadWithoutSearch)
|
||||
}
|
||||
if !m.AnswerRecovered || m.FailureStage != stageRecovered {
|
||||
t.Errorf("stage = %q recovered=%v, want Recovered", m.FailureStage, m.AnswerRecovered)
|
||||
}
|
||||
if m.RecallReturnedTokens == 0 {
|
||||
t.Error("recall returned tokens were not counted")
|
||||
}
|
||||
}
|
||||
|
||||
// An index arm's claim: the address was already on screen, so no search was
|
||||
// needed. That is a different outcome from finding it, and gets its own stage.
|
||||
func TestScorerSeparatesCueReadFromSearch(t *testing.T) {
|
||||
m := score(t, []provider.Message{
|
||||
recallCall("c1", fmt.Sprintf(`{"positions":[%d]}`, scorerTarget)),
|
||||
toolResult("c1", "#42\nthe ownership token is cobalt-lark-17"),
|
||||
{Role: provider.RoleAssistant, Content: "cobalt-lark-17"},
|
||||
}, "cobalt-lark-17")
|
||||
|
||||
if m.SearchCalls != 0 || !m.RecallReadWithoutSearch {
|
||||
t.Errorf("search=%d direct=%v, want 0 and true", m.SearchCalls, m.RecallReadWithoutSearch)
|
||||
}
|
||||
if m.FailureStage != stageCueRead {
|
||||
t.Errorf("stage = %q, want %q", m.FailureStage, stageCueRead)
|
||||
}
|
||||
}
|
||||
|
||||
// Every way the chain can break gets its own label, so a failed run says where
|
||||
// rather than only that.
|
||||
func TestScorerNamesWhereTheChainBroke(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
msgs []provider.Message
|
||||
final string
|
||||
want string
|
||||
}{
|
||||
{
|
||||
name: "never touched recall",
|
||||
msgs: []provider.Message{{Role: provider.RoleAssistant, Content: "I do not recall that."}},
|
||||
final: "I do not recall that.",
|
||||
want: stageNoRetrieval,
|
||||
},
|
||||
{
|
||||
name: "searched and missed",
|
||||
msgs: []provider.Message{
|
||||
recallCall("c1", `{"query":"unrelated words"}`),
|
||||
toolResult("c1", hitBlock(3, 9)),
|
||||
},
|
||||
final: "not found",
|
||||
want: stageSearchMiss,
|
||||
},
|
||||
{
|
||||
name: "found it and never opened it",
|
||||
msgs: []provider.Message{
|
||||
recallCall("c1", `{"query":"retry boundary"}`),
|
||||
toolResult("c1", hitBlock(scorerTarget)),
|
||||
},
|
||||
final: "probably something about retries",
|
||||
want: stageHitNotRead,
|
||||
},
|
||||
{
|
||||
name: "read it and answered wrong",
|
||||
msgs: []provider.Message{
|
||||
recallCall("c1", fmt.Sprintf(`{"positions":[%d]}`, scorerTarget)),
|
||||
toolResult("c1", "#42\nthe token is elsewhere"),
|
||||
},
|
||||
final: "I think it was amber-9",
|
||||
want: stageAnswerWrong,
|
||||
},
|
||||
{
|
||||
name: "read the wrong position",
|
||||
msgs: []provider.Message{
|
||||
recallCall("c1", `{"positions":[7]}`),
|
||||
toolResult("c1", "#7\nnothing useful"),
|
||||
},
|
||||
final: "unclear",
|
||||
want: stageReadMissed,
|
||||
},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
if got := score(t, tc.msgs, tc.final).FailureStage; got != tc.want {
|
||||
t.Errorf("stage = %q, want %q", got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// The answer lives only in folded history, so reaching for the workspace is a
|
||||
// strategy change worth seeing — including when it happens to work.
|
||||
func TestScorerRecordsWorkspaceEscapes(t *testing.T) {
|
||||
m := score(t, []provider.Message{
|
||||
{Role: provider.RoleAssistant, ToolCalls: []provider.ToolCall{
|
||||
{ID: "g1", Name: "bash", Arguments: `{"command":"rg cobalt-lark"}`},
|
||||
{ID: "g2", Name: "read_file", Arguments: `{"path":"notes.md"}`},
|
||||
}},
|
||||
{Role: provider.RoleAssistant, Content: "cobalt-lark-17"},
|
||||
}, "cobalt-lark-17")
|
||||
|
||||
if len(m.UnexpectedWorkTools) != 2 {
|
||||
t.Fatalf("escapes = %v, want bash and read_file", m.UnexpectedWorkTools)
|
||||
}
|
||||
if m.SearchCalls != 0 || m.ReadCalls != 0 {
|
||||
t.Errorf("a workspace tool was counted as recall: search=%d read=%d", m.SearchCalls, m.ReadCalls)
|
||||
}
|
||||
// It answered, so the run is Recovered — the escape is reported beside the
|
||||
// outcome rather than overriding it.
|
||||
if !m.AnswerRecovered {
|
||||
t.Error("a correct answer was not scored as recovered")
|
||||
}
|
||||
}
|
||||
|
||||
// Narration is not action.
|
||||
func TestScorerIgnoresWhatTheModelSaysItWillDo(t *testing.T) {
|
||||
m := score(t, []provider.Message{
|
||||
{Role: provider.RoleAssistant, Content: "Let me search the folded context with recall for the retry boundary."},
|
||||
}, "Let me search the folded context with recall for the retry boundary.")
|
||||
if m.SearchCalls != 0 || m.FailureStage != stageNoRetrieval {
|
||||
t.Errorf("prose was scored as a search: calls=%d stage=%q", m.SearchCalls, m.FailureStage)
|
||||
}
|
||||
}
|
||||
|
||||
// Every marker must land, so a partially right answer is not a recovery.
|
||||
func TestScorerRequiresEveryMarker(t *testing.T) {
|
||||
task := fixtureInstance{Task: contextTask{ID: "t"}, AnswerMarkers: []string{"comet-42", "per-lineage"}}
|
||||
m := scoreRun(nil, task, scorerTarget, "arm", false)
|
||||
m.scoreAnswer("the salt was comet-42", task)
|
||||
if m.AnswerRecovered {
|
||||
t.Error("a half answer scored as recovered")
|
||||
}
|
||||
if len(m.MissingMarkers) != 1 || m.MissingMarkers[0] != "per-lineage" {
|
||||
t.Errorf("missing = %v, want per-lineage", m.MissingMarkers)
|
||||
}
|
||||
}
|
||||
|
||||
// An answer reachable through anything but recall invalidates the run. The
|
||||
// workspace is isolated so this should never fire; asserting it beats trusting
|
||||
// it, and a tripped run leaves the statistics rather than inflating one.
|
||||
func TestScorerInvalidatesRunsThatLeakedThroughAnotherTool(t *testing.T) {
|
||||
m := score(t, []provider.Message{
|
||||
{Role: provider.RoleAssistant, ToolCalls: []provider.ToolCall{
|
||||
{ID: "g1", Name: "bash", Arguments: `{"command":"rg -r ."}`},
|
||||
}},
|
||||
{Role: provider.RoleTool, ToolCallID: "g1", Name: "bash", Content: "corpus.go: the token is cobalt-lark-17"},
|
||||
{Role: provider.RoleAssistant, Content: "cobalt-lark-17"},
|
||||
}, "cobalt-lark-17")
|
||||
|
||||
if !m.Contaminated || m.LeakedVia != "bash" || m.LeakedMarker != "cobalt-lark-17" {
|
||||
t.Fatalf("leak not caught: contaminated=%v via=%q marker=%q", m.Contaminated, m.LeakedVia, m.LeakedMarker)
|
||||
}
|
||||
if m.FailureStage != stageContaminated {
|
||||
t.Errorf("stage = %q, want %q even though every marker matched", m.FailureStage, stageContaminated)
|
||||
}
|
||||
f := summarize("arm", []contextMetrics{m})
|
||||
if f.Scored != 0 || f.Answered != 0 || f.Contaminated != 1 {
|
||||
t.Errorf("a contaminated run entered the statistics: %+v", f)
|
||||
}
|
||||
}
|
||||
|
||||
// Where an escape happened matters: before any recall is a different strategy
|
||||
// from after one came back short.
|
||||
func TestScorerPlacesEscapesRelativeToRecall(t *testing.T) {
|
||||
m := score(t, []provider.Message{
|
||||
{Role: provider.RoleAssistant, ToolCalls: []provider.ToolCall{
|
||||
{ID: "g1", Name: "bash", Arguments: `{"command":"ls"}`},
|
||||
}},
|
||||
{Role: provider.RoleTool, ToolCallID: "g1", Name: "bash", Content: "README.md"},
|
||||
recallCall("c1", `{"query":"retry boundary"}`),
|
||||
toolResult("c1", hitBlock(scorerTarget)),
|
||||
{Role: provider.RoleAssistant, ToolCalls: []provider.ToolCall{
|
||||
{ID: "g2", Name: "read_file", Arguments: `{"path":"README.md"}`},
|
||||
}},
|
||||
{Role: provider.RoleTool, ToolCallID: "g2", Name: "read_file", Content: "nothing here"},
|
||||
}, "unclear")
|
||||
|
||||
if m.EscapeCalls != 2 || m.EscapeBeforeFirstRecall != 1 || m.EscapeAfterFirstRecall != 1 {
|
||||
t.Errorf("escapes = %d (%d before, %d after), want 2 (1, 1)",
|
||||
m.EscapeCalls, m.EscapeBeforeFirstRecall, m.EscapeAfterFirstRecall)
|
||||
}
|
||||
}
|
||||
|
||||
// Routing is judged by round. Two calls in one round are a fan-out, and
|
||||
// reporting that as a preference would invent a decision the model never made.
|
||||
func TestRoutingIsJudgedByRoundNotEventOrder(t *testing.T) {
|
||||
parallel := []provider.Message{
|
||||
{Role: provider.RoleAssistant, ToolCalls: []provider.ToolCall{
|
||||
{ID: "g1", Name: "bash", Arguments: `{"command":"ls"}`},
|
||||
{ID: "c1", Name: "recall", Arguments: `{"query":"retry boundary"}`},
|
||||
}},
|
||||
{Role: provider.RoleTool, ToolCallID: "g1", Name: "bash", Content: "README.md"},
|
||||
toolResult("c1", hitBlock(scorerTarget)),
|
||||
}
|
||||
if got := score(t, parallel, "unclear").Routing; got != routeParallel {
|
||||
t.Errorf("routing = %q for one round holding both, want %q", got, routeParallel)
|
||||
}
|
||||
|
||||
workspaceFirst := []provider.Message{
|
||||
{Role: provider.RoleAssistant, ToolCalls: []provider.ToolCall{
|
||||
{ID: "g1", Name: "bash", Arguments: `{"command":"ls"}`},
|
||||
}},
|
||||
{Role: provider.RoleTool, ToolCallID: "g1", Name: "bash", Content: "README.md"},
|
||||
recallCall("c1", `{"query":"retry boundary"}`),
|
||||
toolResult("c1", hitBlock(scorerTarget)),
|
||||
}
|
||||
if got := score(t, workspaceFirst, "unclear").Routing; got != routeWorkspaceFirst {
|
||||
t.Errorf("routing = %q for workspace in an earlier round, want %q", got, routeWorkspaceFirst)
|
||||
}
|
||||
|
||||
memoryFirst := []provider.Message{
|
||||
recallCall("c1", `{"query":"retry boundary"}`),
|
||||
toolResult("c1", hitBlock(scorerTarget)),
|
||||
{Role: provider.RoleAssistant, ToolCalls: []provider.ToolCall{
|
||||
{ID: "g1", Name: "bash", Arguments: `{"command":"ls"}`},
|
||||
}},
|
||||
{Role: provider.RoleTool, ToolCallID: "g1", Name: "bash", Content: "README.md"},
|
||||
}
|
||||
if got := score(t, memoryFirst, "unclear").Routing; got != routeMemoryFirst {
|
||||
t.Errorf("routing = %q for recall in an earlier round, want %q", got, routeMemoryFirst)
|
||||
}
|
||||
|
||||
if got := score(t, nil, "no idea").Routing; got != routeNeither {
|
||||
t.Errorf("routing = %q with no tools at all, want %q", got, routeNeither)
|
||||
}
|
||||
}
|
||||
|
||||
// Markers matching proves the right fact appears somewhere. An answer that
|
||||
// hedges between two candidates contains it and has not submitted it.
|
||||
func TestScorerFlagsAnAnswerThatNamesTwoCandidates(t *testing.T) {
|
||||
inst := fixtureInstance{
|
||||
Task: contextTask{ID: "t", Vars: []varSpec{code("marker", "fence")}},
|
||||
AnswerMarkers: []string{"fence-qmwnv"},
|
||||
}
|
||||
m := scoreRun(nil, inst, 0, "arm", false)
|
||||
m.scoreAnswer("It was fence-qmwnv, though it may have been fence-abcde.", inst)
|
||||
if !m.AnswerRecovered {
|
||||
t.Error("the marker is present; recovery should still read true")
|
||||
}
|
||||
if !m.AnswerAmbiguous || m.AmbiguousVar != "marker" {
|
||||
t.Errorf("ambiguity not flagged: %v %q", m.AnswerAmbiguous, m.AmbiguousVar)
|
||||
}
|
||||
|
||||
clean := scoreRun(nil, inst, 0, "arm", false)
|
||||
clean.scoreAnswer("The marker was fence-qmwnv.", inst)
|
||||
if clean.AnswerAmbiguous {
|
||||
t.Error("a single candidate was called ambiguous")
|
||||
}
|
||||
}
|
||||
|
||||
// The index arm's claim is narrow: the cue was addressed, the first memory
|
||||
// action was a positions read, and it covered the target. A positions call
|
||||
// after a search located the target credits the index for the search's work.
|
||||
func TestCueDirectReadRequiresTheCueAndTheFirstAction(t *testing.T) {
|
||||
direct := []provider.Message{
|
||||
recallCall("c1", fmt.Sprintf(`{"positions":[%d]}`, scorerTarget)),
|
||||
toolResult("c1", "#42\nthe token is cobalt-lark-17"),
|
||||
}
|
||||
m := scoreRun(direct, scorerTask(), scorerTarget, "arm", true)
|
||||
if !m.CueDirectRead {
|
||||
t.Error("a first-action read with the cue visible was not credited")
|
||||
}
|
||||
|
||||
// Same read, but no cue was on screen to give the address.
|
||||
if scoreRun(direct, scorerTask(), scorerTarget, "arm", false).CueDirectRead {
|
||||
t.Error("a read was credited to a cue that was not visible")
|
||||
}
|
||||
|
||||
// Same read, but a search found the address first.
|
||||
afterSearch := append([]provider.Message{
|
||||
recallCall("c0", `{"query":"retry boundary"}`),
|
||||
toolResult("c0", hitBlock(scorerTarget)),
|
||||
}, direct...)
|
||||
if scoreRun(afterSearch, scorerTask(), scorerTarget, "arm", true).CueDirectRead {
|
||||
t.Error("a read after a search was credited to the index")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,469 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"math/rand"
|
||||
"reasonix/internal/state/sessionstore"
|
||||
|
||||
"reasonix/internal/contract/provider"
|
||||
)
|
||||
|
||||
// Synthetic history on purpose: an answer that exists in the workspace is
|
||||
// solvable by reading the workspace, which measures grep. Every answer is a
|
||||
// placeholder here and a value only inside one run.
|
||||
|
||||
// experiment names which question a task belongs to.
|
||||
const (
|
||||
experimentSearch = "search" // does search reach what no index addresses?
|
||||
experimentIndex = "index" // what is an ambient address line worth?
|
||||
)
|
||||
|
||||
// cueTier is the narrowest fold-index scale at which an index task's cue is
|
||||
// still addressed. Below it the model has to find the call without a hint.
|
||||
const (
|
||||
tierQuarter = "quarter" // visible at default, half, quarter
|
||||
tierHalf = "half" // visible at default, half
|
||||
tierDefault = "default" // visible at default only
|
||||
)
|
||||
|
||||
// role places an index task in the efficiency substrate or the stopping corpus.
|
||||
const (
|
||||
roleEfficiency = "efficiency"
|
||||
roleStopping = "stopping"
|
||||
)
|
||||
|
||||
// plantKind is how a task's target enters the transcript.
|
||||
const (
|
||||
plantAssistant = "assistant" // the model's own account: never index-addressed
|
||||
plantCall = "call" // a tool call and its result: always index-addressed
|
||||
)
|
||||
|
||||
// contextTask is one recall question as a template. Nothing here is an answer.
|
||||
type contextTask struct {
|
||||
ID string
|
||||
Experiment string
|
||||
TargetKind string
|
||||
Prompt string
|
||||
ProbeQuery string
|
||||
// Vars declare the placeholders. AnswerVars name the ones a correct answer
|
||||
// must contain, CueVar the one an index line carries.
|
||||
Vars []varSpec
|
||||
AnswerVars []string
|
||||
CueVar string
|
||||
CueTier string
|
||||
// Role is roleEfficiency or roleStopping; it decides the batch a task enters.
|
||||
Role string
|
||||
// PlantAfterGen ages the target: mergeFoldIndex trims from its oldest end,
|
||||
// so what follows a cue decides which budget addresses it. Frozen, because
|
||||
// a run that searched for it would let the tested code move its goalposts.
|
||||
PlantAfterGen int
|
||||
// Plant is how the target enters, and the rest is what it says.
|
||||
Plant string
|
||||
CallTool string
|
||||
CallArgs string
|
||||
CallID string
|
||||
PlantBody string
|
||||
}
|
||||
|
||||
// fixtureInstance is one task with values. Scoring binds to this, never to the
|
||||
// corpus: the corpus has no answers to bind to.
|
||||
type fixtureInstance struct {
|
||||
Task contextTask
|
||||
Vars fixtureVars
|
||||
Prompt string
|
||||
ProbeQuery string
|
||||
AnswerMarkers []string
|
||||
CueMarker string
|
||||
body string
|
||||
callArgs string
|
||||
}
|
||||
|
||||
// instantiateTask resolves one task against a generator.
|
||||
func instantiateTask(t contextTask, rng *rand.Rand) (fixtureInstance, error) {
|
||||
vars := newVars(t.Vars, rng)
|
||||
inst := fixtureInstance{Task: t, Vars: vars}
|
||||
var err error
|
||||
for _, pair := range []struct {
|
||||
dst *string
|
||||
text string
|
||||
}{
|
||||
{&inst.Prompt, t.Prompt},
|
||||
{&inst.ProbeQuery, t.ProbeQuery},
|
||||
{&inst.body, t.PlantBody},
|
||||
{&inst.callArgs, t.CallArgs},
|
||||
} {
|
||||
if *pair.dst, err = instantiate(pair.text, vars); err != nil {
|
||||
return fixtureInstance{}, fmt.Errorf("%s: %w", t.ID, err)
|
||||
}
|
||||
}
|
||||
for _, name := range t.AnswerVars {
|
||||
value, ok := vars[name]
|
||||
if !ok {
|
||||
return fixtureInstance{}, fmt.Errorf("%s: answer var %q is not declared", t.ID, name)
|
||||
}
|
||||
inst.AnswerMarkers = append(inst.AnswerMarkers, value)
|
||||
}
|
||||
if t.CueVar != "" {
|
||||
inst.CueMarker = vars[t.CueVar]
|
||||
}
|
||||
return inst, nil
|
||||
}
|
||||
|
||||
// plant appends the instantiated target and returns its canonical position.
|
||||
func (f fixtureInstance) plant(s *sessionstore.Session) int {
|
||||
at := len(s.Messages)
|
||||
if f.Task.Plant == plantAssistant {
|
||||
s.Add(provider.Message{Role: provider.RoleAssistant, Content: f.body})
|
||||
return at
|
||||
}
|
||||
s.Add(provider.Message{Role: provider.RoleAssistant, ToolCalls: []provider.ToolCall{
|
||||
{ID: f.Task.CallID, Name: f.Task.CallTool, Arguments: f.callArgs},
|
||||
}})
|
||||
s.Add(provider.Message{Role: provider.RoleTool, ToolCallID: f.Task.CallID, Name: f.Task.CallTool, Content: f.body})
|
||||
return at
|
||||
}
|
||||
|
||||
func code(name, word string) varSpec { return varSpec{Name: name, Kind: varCodename, Word: word} }
|
||||
|
||||
func num(name string, lo, hi int) varSpec {
|
||||
return varSpec{Name: name, Kind: varInt, Min: lo, Max: hi}
|
||||
}
|
||||
|
||||
// efficiencyTasks is the substrate a study measures on. Every task in it
|
||||
// qualifies under the corpus bar; the stopping corpus is deliberately excluded,
|
||||
// because a task carrying post-sufficient stopping behaviour swamps the effect
|
||||
// an efficiency batch exists to measure.
|
||||
func efficiencyTasks() []contextTask {
|
||||
var out []contextTask
|
||||
for _, t := range indexTasks() {
|
||||
if t.Role == roleEfficiency {
|
||||
out = append(out, t)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func searchTasks() []contextTask {
|
||||
return []contextTask{
|
||||
{
|
||||
ID: "r01-retry-boundary", Experiment: experimentSearch, TargetKind: "assistant_reasoning",
|
||||
Prompt: "What ownership token did we settle on for the retry boundary, and under what tag did we file the reason for not moving retries above framing?",
|
||||
ProbeQuery: "retry boundary ownership token framing",
|
||||
Vars: []varSpec{code("token", "cobalt"), code("reason", "stream")},
|
||||
AnswerVars: []string{"token", "reason"},
|
||||
Plant: plantAssistant,
|
||||
PlantBody: "The retry boundary remains at ingress. The ownership token is {{token}}. " +
|
||||
"Moving retries above framing can duplicate a partially received stream, which we filed as {{reason}}.",
|
||||
},
|
||||
{
|
||||
ID: "r02-checkpoint-anchor", Experiment: experimentSearch, TargetKind: "assistant_reasoning",
|
||||
Prompt: "What was the codename of the checkpoint anchor policy, and what tag did we give the reason for not anchoring against digest positions?",
|
||||
ProbeQuery: "checkpoint anchor policy digest positions",
|
||||
Vars: []varSpec{code("policy", "northglass"), code("reason", "genlocal")},
|
||||
AnswerVars: []string{"policy", "reason"},
|
||||
Plant: plantAssistant,
|
||||
PlantBody: "The checkpoint anchor policy is {{policy}}. Digest positions are generation-local — " +
|
||||
"filed as {{reason}} — so recovery must anchor against canonical state.",
|
||||
},
|
||||
{
|
||||
ID: "r03-recovery-cjk", Experiment: experimentSearch, TargetKind: "assistant_reasoning_cjk",
|
||||
Prompt: "我们之前定的验证失败恢复策略代号叫什么?当时把“失败 digest 不能变成 baseline”这条登记在哪个标记下?",
|
||||
ProbeQuery: "恢复策略 baseline digest 验证失败",
|
||||
Vars: []varSpec{code("policy", "songzhen"), code("rule", "trustbase")},
|
||||
AnswerVars: []string{"policy", "rule"},
|
||||
Plant: plantAssistant,
|
||||
PlantBody: "恢复策略代号是 {{policy}}。验证失败以后必须保留旧基线,不能把新的 digest 晋升为 baseline," +
|
||||
"因为失败证据不能成为后续比较的可信基准;这条规则登记为 {{rule}}。",
|
||||
},
|
||||
{
|
||||
ID: "r04-lease-fence", Experiment: experimentSearch, TargetKind: "assistant_reasoning",
|
||||
Prompt: "那次 lease fence 检查报告的诊断标记是什么?expected 和 observed epoch 分别是多少?",
|
||||
ProbeQuery: "lease fence writer epoch mismatch",
|
||||
Vars: []varSpec{
|
||||
code("marker", "fence"), num("expected", 400, 900), num("observed", 100, 399),
|
||||
},
|
||||
AnswerVars: []string{"marker", "expected", "observed"},
|
||||
Plant: plantAssistant,
|
||||
PlantBody: "The lease fence check reported {{marker}}: expected writer epoch {{expected}}, " +
|
||||
"observed {{observed}}. We left the fence in place.",
|
||||
},
|
||||
{
|
||||
ID: "r05-schema-guard", Experiment: experimentSearch, TargetKind: "assistant_reasoning",
|
||||
Prompt: "当时 schema guard 报告的标记是什么,超预算多少 token?",
|
||||
ProbeQuery: "schema guard provider visible surface budget",
|
||||
Vars: []varSpec{code("marker", "amber"), num("over", 120, 480)},
|
||||
AnswerVars: []string{"marker", "over"},
|
||||
Plant: plantAssistant,
|
||||
PlantBody: "The schema guard reported {{marker}}: the provider-visible tool schema exceeded " +
|
||||
"the allowed surface by {{over}} tokens. We trimmed a description instead.",
|
||||
},
|
||||
{
|
||||
ID: "r06-shadow-probe", Experiment: experimentSearch, TargetKind: "assistant_reasoning",
|
||||
Prompt: "我们那次 shadow probe 的 marker、p95 divergence 和 generation 分别是多少?",
|
||||
ProbeQuery: "shadow probe divergence generation marker",
|
||||
Vars: []varSpec{
|
||||
code("marker", "violet"),
|
||||
{Name: "divergence", Kind: varDecimal, Min: 110, Max: 890},
|
||||
num("generation", 14, 60),
|
||||
},
|
||||
AnswerVars: []string{"marker", "divergence", "generation"},
|
||||
Plant: plantAssistant,
|
||||
PlantBody: "The shadow probe came back as marker={{marker}} with a p95 divergence of " +
|
||||
"{{divergence}} at generation={{generation}}, which we judged within tolerance.",
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// efficiencyIndexTasks is what a study measures on: two tasks per cue tier.
|
||||
func efficiencyIndexTasks() []contextTask {
|
||||
out := indexQuarterTasks()
|
||||
out = append(out, indexHalfTasks()...)
|
||||
out = append(out, indexDefaultTasks()...)
|
||||
return out
|
||||
}
|
||||
|
||||
// indexQuarterTasks holds the quarter-tier literals.
|
||||
func indexQuarterTasks() []contextTask {
|
||||
return []contextTask{
|
||||
{
|
||||
ID: "i37-dispatchlag", Experiment: experimentIndex, TargetKind: "tool_input", Role: roleEfficiency,
|
||||
Prompt: "早前会话里执行过 ./scripts/dispatchlag,它打印出的 dispatch lag epoch 是多少?",
|
||||
ProbeQuery: "dispatchlag dispatch lag epoch",
|
||||
Vars: []varSpec{code("cue", "dispatchlag"), code("epoch", "cobalt")},
|
||||
AnswerVars: []string{"epoch"}, CueVar: "cue", CueTier: tierQuarter,
|
||||
PlantAfterGen: 4,
|
||||
Plant: plantCall, CallID: "p37", CallTool: "bash",
|
||||
CallArgs: `{"command":"./scripts/dispatchlag {{cue}}"}`,
|
||||
PlantBody: "dispatch lag epoch = {{epoch}}",
|
||||
},
|
||||
{
|
||||
ID: "i118-fencelag", Experiment: experimentIndex, TargetKind: "tool_input", Role: roleEfficiency,
|
||||
Prompt: "早前会话里运行过 ./scripts/fencelag,它打印出的 fence lag 是多少?",
|
||||
ProbeQuery: "fencelag fence lag",
|
||||
Vars: []varSpec{code("cue", "fencelag"), code("v", "slate")},
|
||||
AnswerVars: []string{"v"}, CueVar: "cue", CueTier: tierQuarter,
|
||||
PlantAfterGen: 4,
|
||||
Plant: plantCall, CallID: "x118", CallTool: "bash",
|
||||
CallArgs: `{"command":"./scripts/fencelag {{cue}}"}`,
|
||||
PlantBody: "fence lag = {{v}}",
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// indexHalfTasks holds the half-tier literals.
|
||||
func indexHalfTasks() []contextTask {
|
||||
return []contextTask{
|
||||
{
|
||||
ID: "i70-ingestfloor", Experiment: experimentIndex, TargetKind: "tool_input", Role: roleEfficiency,
|
||||
Prompt: "早前会话里执行过 ./scripts/ingestfloor,它打印出的 route quota 是多少?",
|
||||
ProbeQuery: "ingestfloor quota",
|
||||
Vars: []varSpec{code("cue", "ingestfloor"), num("quota", 1200, 9800)},
|
||||
AnswerVars: []string{"quota"}, CueVar: "cue", CueTier: tierHalf,
|
||||
PlantAfterGen: 2,
|
||||
Plant: plantCall, CallID: "q70", CallTool: "bash",
|
||||
CallArgs: `{"command":"./scripts/ingestfloor {{cue}}"}`,
|
||||
PlantBody: "route quota = {{quota}}",
|
||||
},
|
||||
{
|
||||
ID: "i106-budgetseal", Experiment: experimentIndex, TargetKind: "tool_input", Role: roleEfficiency,
|
||||
Prompt: "早前会话里执行过 ./scripts/budgetseal,它打印出的 route quota 是多少?",
|
||||
ProbeQuery: "budgetseal route quota",
|
||||
Vars: []varSpec{code("cue", "budgetseal"), code("quota", "amber")},
|
||||
AnswerVars: []string{"quota"}, CueVar: "cue", CueTier: tierHalf,
|
||||
PlantAfterGen: 2,
|
||||
Plant: plantCall, CallID: "h106", CallTool: "bash",
|
||||
CallArgs: `{"command":"./scripts/budgetseal {{cue}}"}`,
|
||||
PlantBody: "route quota = {{quota}}",
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// indexDefaultTasks holds the default-tier literals.
|
||||
func indexDefaultTasks() []contextTask {
|
||||
return []contextTask{
|
||||
{
|
||||
ID: "i47-tenantlease", Experiment: experimentIndex, TargetKind: "tool_input", Role: roleEfficiency,
|
||||
Prompt: "早前会话里执行过 ./scripts/tenantlease,它打印出的 tenant lease(秒)是多少?",
|
||||
ProbeQuery: "tenantlease tenant lease",
|
||||
Vars: []varSpec{code("cue", "tenantlease"), {Name: "lease", Kind: varDuration, Min: 15, Max: 300}},
|
||||
AnswerVars: []string{"lease"}, CueVar: "cue", CueTier: tierDefault,
|
||||
PlantAfterGen: 0,
|
||||
Plant: plantCall, CallID: "p47", CallTool: "bash",
|
||||
CallArgs: `{"command":"./scripts/tenantlease {{cue}}"}`,
|
||||
PlantBody: "tenant lease = {{lease}}",
|
||||
},
|
||||
{
|
||||
ID: "i112-lagquota", Experiment: experimentIndex, TargetKind: "tool_input", Role: roleEfficiency,
|
||||
Prompt: "早前会话里执行过 ./scripts/lagquota,它打印出的 lagquota lease 是多少?",
|
||||
ProbeQuery: "lagquota lagquota lease",
|
||||
Vars: []varSpec{code("cue", "lagquota"), {Name: "lease", Kind: varDuration, Min: 20, Max: 420}},
|
||||
AnswerVars: []string{"lease"}, CueVar: "cue", CueTier: tierDefault,
|
||||
PlantAfterGen: 0,
|
||||
Plant: plantCall, CallID: "d112", CallTool: "bash",
|
||||
CallArgs: `{"command":"./scripts/lagquota {{cue}}"}`,
|
||||
PlantBody: "lagquota lease = {{lease}}",
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// stoppingIndexTasks is the frozen study's six tasks, kept out of efficiency
|
||||
// batches and measurable on their own with -task <id>.
|
||||
func stoppingIndexTasks() []contextTask {
|
||||
return []contextTask{
|
||||
{
|
||||
ID: "i01-transport-fallback", Experiment: experimentIndex, TargetKind: "tool_input", Role: roleStopping,
|
||||
Prompt: "transport fallback probe 当时报告的 selector 是什么?",
|
||||
ProbeQuery: "transport fallback probe selector",
|
||||
Vars: []varSpec{code("cue", "tfprobe"), code("selector", "fern")},
|
||||
AnswerVars: []string{"selector"}, CueVar: "cue", CueTier: tierQuarter,
|
||||
PlantAfterGen: 4,
|
||||
Plant: plantCall, CallID: "tf1", CallTool: "bash",
|
||||
CallArgs: `{"command":"./scripts/probe {{cue}}"}`,
|
||||
PlantBody: "fallback selector = {{selector}}",
|
||||
},
|
||||
{
|
||||
ID: "i02-scheduler-handoff", Experiment: experimentIndex, TargetKind: "tool_input", Role: roleStopping,
|
||||
Prompt: "scheduler handoff 当时记录的 grace 和 marker 是什么?",
|
||||
ProbeQuery: "scheduler handoff grace marker",
|
||||
Vars: []varSpec{
|
||||
code("cue", "schedhandoff"), code("marker", "opal"),
|
||||
{Name: "grace", Kind: varDuration, Min: 18, Max: 90},
|
||||
},
|
||||
AnswerVars: []string{"grace", "marker"}, CueVar: "cue", CueTier: tierQuarter,
|
||||
PlantAfterGen: 4,
|
||||
Plant: plantCall, CallID: "sh1", CallTool: "read_file",
|
||||
CallArgs: `{"path":"scratch/{{cue}}.txt"}`,
|
||||
PlantBody: "handoff grace = {{grace}}\nmarker = {{marker}}",
|
||||
},
|
||||
{
|
||||
ID: "i03-coalescing", Experiment: experimentIndex, TargetKind: "tool_input", Role: roleStopping,
|
||||
Prompt: "job completion 的 coalescing probe 最后记录的 window 和 marker 是多少?",
|
||||
ProbeQuery: "coalescing policy completion window",
|
||||
Vars: []varSpec{
|
||||
code("cue", "coalescing"), code("marker", "cedar"),
|
||||
{Name: "window", Kind: varDuration, Min: 40, Max: 140},
|
||||
},
|
||||
AnswerVars: []string{"window", "marker"}, CueVar: "cue", CueTier: tierHalf,
|
||||
PlantAfterGen: 2,
|
||||
Plant: plantCall, CallID: "cp1", CallTool: "bash",
|
||||
CallArgs: `{"command":"grep {{cue}} scratch/job-notes.log"}`,
|
||||
PlantBody: "completion coalesce window = {{window}}\nmarker = {{marker}}",
|
||||
},
|
||||
{
|
||||
ID: "i04-recovery-fence", Experiment: experimentIndex, TargetKind: "tool_input", Role: roleStopping,
|
||||
Prompt: "recovery fence 那次检查报告的 marker 和两个 epoch 是什么?",
|
||||
ProbeQuery: "recovery fence epoch check",
|
||||
Vars: []varSpec{
|
||||
code("cue", "recfence"), code("marker", "quartz"),
|
||||
num("expected", 400, 900), num("observed", 100, 399),
|
||||
},
|
||||
AnswerVars: []string{"marker", "expected", "observed"}, CueVar: "cue", CueTier: tierHalf,
|
||||
PlantAfterGen: 2,
|
||||
Plant: plantCall, CallID: "rf1", CallTool: "bash",
|
||||
CallArgs: `{"command":"./scripts/check {{cue}}"}`,
|
||||
PlantBody: "{{marker}}\nexpected epoch {{expected}}\nobserved epoch {{observed}}",
|
||||
},
|
||||
{
|
||||
ID: "i05-cache-probe", Experiment: experimentIndex, TargetKind: "tool_input", Role: roleStopping,
|
||||
Prompt: "cache probe 当时记录的 prefix salt 和 scope 是什么?",
|
||||
ProbeQuery: "cache probe prefix salt scope",
|
||||
Vars: []varSpec{code("cue", "cacheprobe"), code("salt", "comet"), code("scope", "lineage")},
|
||||
AnswerVars: []string{"salt", "scope"}, CueVar: "cue", CueTier: tierDefault,
|
||||
PlantAfterGen: 0,
|
||||
Plant: plantCall, CallID: "cb1", CallTool: "read_file",
|
||||
CallArgs: `{"path":"scratch/{{cue}}.txt"}`,
|
||||
PlantBody: "prefix salt = {{salt}}\nscope = {{scope}}",
|
||||
},
|
||||
{
|
||||
ID: "i06-dispatch-handoff", Experiment: experimentIndex, TargetKind: "tool_input", Role: roleStopping,
|
||||
Prompt: "dispatch handoff probe 当时确认的 mode 和 marker 是什么?",
|
||||
ProbeQuery: "dispatch handoff probe mode",
|
||||
Vars: []varSpec{code("cue", "dispatchhandoff"), code("mode", "trigger"), code("marker", "iris")},
|
||||
AnswerVars: []string{"mode", "marker"}, CueVar: "cue", CueTier: tierDefault,
|
||||
PlantAfterGen: 0,
|
||||
Plant: plantCall, CallID: "dh1", CallTool: "bash",
|
||||
CallArgs: `{"command":"./scripts/probe {{cue}}"}`,
|
||||
PlantBody: "handoff mode = {{mode}}\nmarker = {{marker}}",
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// indexTasks is the whole index corpus, the efficiency substrate first.
|
||||
func indexTasks() []contextTask {
|
||||
out := efficiencyIndexTasks()
|
||||
return append(out, stoppingIndexTasks()...)
|
||||
}
|
||||
|
||||
func allTasks() []contextTask {
|
||||
return append(searchTasks(), indexTasks()...)
|
||||
}
|
||||
|
||||
func taskByID(id string) (contextTask, bool) {
|
||||
for _, t := range allTasks() {
|
||||
if t.ID == id {
|
||||
return t, true
|
||||
}
|
||||
}
|
||||
return contextTask{}, false
|
||||
}
|
||||
|
||||
// tierScales is the fixture's contract: which arms must address a cue and
|
||||
// which must not. A policy change that moves a cue across a boundary fails
|
||||
// preflight rather than turning two arms into one experiment.
|
||||
func tierScales(tier string) (visible, hidden []string) {
|
||||
switch tier {
|
||||
case tierQuarter:
|
||||
return []string{"default", "half", "quarter"}, []string{"off"}
|
||||
case tierHalf:
|
||||
return []string{"default", "half"}, []string{"quarter", "off"}
|
||||
case tierDefault:
|
||||
return []string{"default"}, []string{"half", "quarter", "off"}
|
||||
default:
|
||||
return nil, nil
|
||||
}
|
||||
}
|
||||
|
||||
// boundaryPair is the one comparison an index task is built for: the narrowest
|
||||
// scale that still addresses its cue, and the next one down. Holding the
|
||||
// question fixed and moving only the affordance is what four arm averages
|
||||
// cannot do, since each of those mixes six different questions.
|
||||
func boundaryPair(tier string) (cue, noCue string) {
|
||||
switch tier {
|
||||
case tierQuarter:
|
||||
return "quarter", "off"
|
||||
case tierHalf:
|
||||
return "half", "quarter"
|
||||
case tierDefault:
|
||||
return "default", "half"
|
||||
default:
|
||||
return "", ""
|
||||
}
|
||||
}
|
||||
|
||||
func validateCorpus() error {
|
||||
seen := map[string]bool{}
|
||||
for _, t := range allTasks() {
|
||||
if seen[t.ID] {
|
||||
return fmt.Errorf("duplicate task id %q", t.ID)
|
||||
}
|
||||
seen[t.ID] = true
|
||||
if len(t.AnswerVars) == 0 || t.ProbeQuery == "" || t.Prompt == "" || t.PlantBody == "" {
|
||||
return fmt.Errorf("%s: a task needs a prompt, a probe query, a body and answer vars", t.ID)
|
||||
}
|
||||
if t.Experiment == experimentIndex {
|
||||
if t.CueVar == "" {
|
||||
return fmt.Errorf("%s: an index task needs a cue var", t.ID)
|
||||
}
|
||||
if t.Role != roleEfficiency && t.Role != roleStopping {
|
||||
return fmt.Errorf("%s: an index task needs a role (%s or %s)", t.ID, roleEfficiency, roleStopping)
|
||||
}
|
||||
if v, _ := tierScales(t.CueTier); v == nil {
|
||||
return fmt.Errorf("%s: unknown cue tier %q", t.ID, t.CueTier)
|
||||
}
|
||||
}
|
||||
if _, err := instantiateTask(t, seededRand(t.ID, "validate")); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,243 @@
|
||||
// dose.go — does the cost step where the cue disappears?
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"math"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Not "a bigger index is better" — CueDirectRead says the addresses are rarely
|
||||
// used. The hypothesis is that a visible action cue lowers retrieval cost,
|
||||
// which predicts a step at each task's own boundary, not a slope.
|
||||
|
||||
// scaleOrder is the dose axis, widest first.
|
||||
var scaleOrder = []string{"default", "half", "quarter", "off"}
|
||||
|
||||
// boundaryStep is the index into the three transitions where a tier's cue
|
||||
// disappears: default->half is 0, half->quarter is 1, quarter->off is 2.
|
||||
func boundaryStep(tier string) int {
|
||||
switch tier {
|
||||
case tierDefault:
|
||||
return 0
|
||||
case tierHalf:
|
||||
return 1
|
||||
case tierQuarter:
|
||||
return 2
|
||||
default:
|
||||
return -1
|
||||
}
|
||||
}
|
||||
|
||||
// doseCurve is one task's readings across the four budgets.
|
||||
type doseCurve struct {
|
||||
Task string
|
||||
Tier string
|
||||
Metric string
|
||||
Values [4]float64
|
||||
// Missing marks arms that produced nothing. Printing those as zero draws a
|
||||
// curve falling to the floor where in fact it was never measured.
|
||||
Missing [4]bool
|
||||
Present bool
|
||||
// PreMean averages the arms where the cue is visible, PostMean the rest.
|
||||
PreMean, PostMean float64
|
||||
BoundaryDelta float64
|
||||
// BoundaryAligned: the largest transition is the one where this task's cue
|
||||
// disappears. A change elsewhere is variance shaped like an effect.
|
||||
BoundaryAligned bool
|
||||
}
|
||||
|
||||
func buildCurve(task, tier, metric string, byScale map[string]float64) doseCurve {
|
||||
c := doseCurve{Task: task, Tier: tier, Metric: metric, Present: true}
|
||||
for i, scale := range scaleOrder {
|
||||
v, ok := byScale[scale]
|
||||
if !ok {
|
||||
c.Present, c.Missing[i] = false, true
|
||||
}
|
||||
c.Values[i] = v
|
||||
}
|
||||
step := boundaryStep(tier)
|
||||
if !c.Present || step < 0 {
|
||||
return c
|
||||
}
|
||||
visible := step + 1 // arms before the boundary still show the cue
|
||||
c.PreMean = mean(c.Values[:visible])
|
||||
c.PostMean = mean(c.Values[visible:])
|
||||
c.BoundaryDelta = c.PostMean - c.PreMean
|
||||
|
||||
transitions := [3]float64{
|
||||
math.Abs(c.Values[1] - c.Values[0]),
|
||||
math.Abs(c.Values[2] - c.Values[1]),
|
||||
math.Abs(c.Values[3] - c.Values[2]),
|
||||
}
|
||||
largest, at := 0.0, -1
|
||||
for i, t := range transitions {
|
||||
if t > largest {
|
||||
largest, at = t, i
|
||||
}
|
||||
}
|
||||
// A flat curve has no largest transition to align with anything.
|
||||
c.BoundaryAligned = largest > 0 && at == step
|
||||
return c
|
||||
}
|
||||
|
||||
func mean(xs []float64) float64 {
|
||||
if len(xs) == 0 {
|
||||
return 0
|
||||
}
|
||||
sum := 0.0
|
||||
for _, x := range xs {
|
||||
sum += x
|
||||
}
|
||||
return sum / float64(len(xs))
|
||||
}
|
||||
|
||||
func median(xs []float64) float64 {
|
||||
if len(xs) == 0 {
|
||||
return 0
|
||||
}
|
||||
s := append([]float64(nil), xs...)
|
||||
sort.Float64s(s)
|
||||
mid := len(s) / 2
|
||||
if len(s)%2 == 1 {
|
||||
return s[mid]
|
||||
}
|
||||
return (s[mid-1] + s[mid]) / 2
|
||||
}
|
||||
|
||||
// doseMetrics are the readings a curve is drawn for.
|
||||
var doseMetrics = []struct {
|
||||
name string
|
||||
of func(contextMetrics) float64
|
||||
}{
|
||||
{"searches", func(m contextMetrics) float64 { return float64(m.SearchCalls) }},
|
||||
{"recall-tokens", func(m contextMetrics) float64 { return float64(m.RecallReturnedTokens) }},
|
||||
{"tool-rounds", func(m contextMetrics) float64 { return float64(m.RetrievalRounds + m.EscapeCalls) }},
|
||||
{"escapes", func(m contextMetrics) float64 { return float64(m.EscapeCalls) }},
|
||||
}
|
||||
|
||||
// reportDoseResponse draws each task's curve and reports where the steps fell.
|
||||
func reportDoseResponse(all []contextMetrics) string {
|
||||
valid := map[string]map[string]contextMetrics{}
|
||||
for _, m := range all {
|
||||
if m.Failed || m.Contaminated || m.FailureStage == "RunError" {
|
||||
continue
|
||||
}
|
||||
scale := strings.TrimPrefix(m.Arm, "index-")
|
||||
if valid[m.Task] == nil {
|
||||
valid[m.Task] = map[string]contextMetrics{}
|
||||
}
|
||||
valid[m.Task][scale] = m
|
||||
}
|
||||
|
||||
var b strings.Builder
|
||||
b.WriteString("\n## Dose-response, per task\n")
|
||||
b.WriteString(" The arrow marks where this task's cue stops being addressed.\n")
|
||||
aligned := map[string]int{}
|
||||
counted := map[string]int{}
|
||||
deltas := map[string][]float64{}
|
||||
|
||||
for _, t := range indexTasks() {
|
||||
arms := valid[t.ID]
|
||||
if len(arms) == 0 {
|
||||
continue
|
||||
}
|
||||
step := boundaryStep(t.CueTier)
|
||||
fmt.Fprintf(&b, "\n %s (cue survives to %s)\n", t.ID, t.CueTier)
|
||||
fmt.Fprintf(&b, " %-14s%s\n", "", scaleHeader(step))
|
||||
for _, metric := range doseMetrics {
|
||||
byScale := map[string]float64{}
|
||||
for scale, m := range arms {
|
||||
byScale[scale] = metric.of(m)
|
||||
}
|
||||
c := buildCurve(t.ID, t.CueTier, metric.name, byScale)
|
||||
mark := " "
|
||||
if c.BoundaryAligned {
|
||||
mark = "*"
|
||||
aligned[metric.name]++
|
||||
}
|
||||
if c.Present {
|
||||
counted[metric.name]++
|
||||
deltas[metric.name] = append(deltas[metric.name], c.BoundaryDelta)
|
||||
}
|
||||
delta := fmt.Sprintf("Δ%+8.1f", c.BoundaryDelta)
|
||||
if !c.Present {
|
||||
delta = " no curve"
|
||||
}
|
||||
fmt.Fprintf(&b, " %-14s%s %s %s\n", metric.name, curveCells(c), delta, mark)
|
||||
}
|
||||
}
|
||||
|
||||
b.WriteString("\n## Where the step fell (* = largest transition is at the cue boundary)\n")
|
||||
for _, metric := range doseMetrics {
|
||||
d := deltas[metric.name]
|
||||
sensitivity := ""
|
||||
if len(d) > 1 {
|
||||
sensitivity = fmt.Sprintf(" without the largest: %+.1f", meanWithoutLargest(d))
|
||||
}
|
||||
fmt.Fprintf(&b, " %-14s boundary-aligned %d/%d mean Δ%+.1f median Δ%+.1f%s\n",
|
||||
metric.name, aligned[metric.name], counted[metric.name], mean(d), median(d), sensitivity)
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// meanWithoutLargest is the robustness check: the largest effect is reported
|
||||
// and then set aside, so a conclusion never rests on one task alone.
|
||||
func meanWithoutLargest(d []float64) float64 {
|
||||
if len(d) < 2 {
|
||||
return mean(d)
|
||||
}
|
||||
worst, at := 0.0, 0
|
||||
for i, v := range d {
|
||||
if math.Abs(v) > worst {
|
||||
worst, at = math.Abs(v), i
|
||||
}
|
||||
}
|
||||
rest := append(append([]float64(nil), d[:at]...), d[at+1:]...)
|
||||
return mean(rest)
|
||||
}
|
||||
|
||||
func scaleHeader(step int) string {
|
||||
var b strings.Builder
|
||||
for i, s := range scaleOrder {
|
||||
if i == step+1 {
|
||||
s = "↓" + s
|
||||
}
|
||||
fmt.Fprintf(&b, "%8s", s)
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// curveCells prints a reading per arm, and a dash where an arm produced none.
|
||||
func curveCells(c doseCurve) string {
|
||||
var b strings.Builder
|
||||
for i, v := range c.Values {
|
||||
if c.Missing[i] {
|
||||
fmt.Fprintf(&b, "%8s", "-")
|
||||
continue
|
||||
}
|
||||
fmt.Fprintf(&b, "%8.0f", v)
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
func percentile(xs []float64, q float64) float64 {
|
||||
if len(xs) == 0 {
|
||||
return 0
|
||||
}
|
||||
s := append([]float64(nil), xs...)
|
||||
sort.Float64s(s)
|
||||
i := int(float64(len(s)-1) * q)
|
||||
return s[i]
|
||||
}
|
||||
|
||||
func maxOf(xs []float64) float64 {
|
||||
out := 0.0
|
||||
for _, x := range xs {
|
||||
if x > out {
|
||||
out = x
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,306 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// blockTask makes one task fail before its first model turn: its own directory
|
||||
// is a regular file, so buildFixture cannot create the session under it. It is
|
||||
// the same early exit as a failed fixture build, session load, or
|
||||
// answer-on-disk guard, and it needs no provider.
|
||||
func blockTask(t *testing.T, root, id string) {
|
||||
t.Helper()
|
||||
if err := os.WriteFile(filepath.Join(root, id), []byte("not a directory\n"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// mixedTaskPair is one task whose runs all fail and one whose runs all succeed.
|
||||
// Both are drawn from the corpus, so the batch is the real pipeline.
|
||||
func mixedTaskPair() (blocked, ok contextTask) {
|
||||
blocked, ok = indexTasks()[0], indexTasks()[1]
|
||||
for _, t := range indexTasks() {
|
||||
if t.Role == roleEfficiency && len(armsFor(t)) == 2 {
|
||||
ok = t
|
||||
break
|
||||
}
|
||||
}
|
||||
return blocked, ok
|
||||
}
|
||||
|
||||
// runBlockedExperiment runs one dry index batch over the given tasks, with the
|
||||
// named task failing before any model turn, and returns the batch directory, the
|
||||
// runs, the manifest, and everything the run printed.
|
||||
func runBlockedExperiment(t *testing.T, tasks []contextTask, blocked ...string) (string, []contextMetrics, batchManifest, string) {
|
||||
t.Helper()
|
||||
root := t.TempDir()
|
||||
for _, id := range blocked {
|
||||
blockTask(t, root, id)
|
||||
}
|
||||
out := captureStdout(t, func() {
|
||||
if code := runExperiment("run-index", experimentIndex, root, true, tasks); code != 0 {
|
||||
t.Errorf("runExperiment exit = %d, want 0 (the batch finished)", code)
|
||||
}
|
||||
})
|
||||
runs, _ := readResultLines(t, root)
|
||||
manifest, _ := readManifest(t, root)
|
||||
return root, runs, manifest, out
|
||||
}
|
||||
|
||||
// captureStdout reads everything the function prints. os.Pipe is unbuffered, so
|
||||
// the reader must run beside the call.
|
||||
func captureStdout(t *testing.T, f func()) string {
|
||||
t.Helper()
|
||||
r, w, err := os.Pipe()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
old := os.Stdout
|
||||
os.Stdout = w
|
||||
done := make(chan string, 1)
|
||||
go func() {
|
||||
var b strings.Builder
|
||||
buf := make([]byte, 4096)
|
||||
for {
|
||||
n, err := r.Read(buf)
|
||||
b.Write(buf[:n])
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
}
|
||||
done <- b.String()
|
||||
}()
|
||||
f()
|
||||
os.Stdout = old
|
||||
w.Close()
|
||||
text := <-done
|
||||
r.Close()
|
||||
return text
|
||||
}
|
||||
|
||||
// funnelScored reads the "runs=N" column of the named arm's funnel line out of
|
||||
// the printed tables, so the assertion is on the number a reader sees.
|
||||
func funnelScored(t *testing.T, out, arm string) int {
|
||||
t.Helper()
|
||||
for line := range strings.SplitSeq(out, "\n") {
|
||||
fields := strings.Fields(line)
|
||||
if len(fields) < 2 || fields[0] != arm || !strings.HasPrefix(fields[1], "runs=") {
|
||||
continue
|
||||
}
|
||||
n, err := strconv.Atoi(strings.TrimPrefix(fields[1], "runs="))
|
||||
if err != nil {
|
||||
t.Fatalf("funnel line for %s: %v\n%s", arm, err, line)
|
||||
}
|
||||
return n
|
||||
}
|
||||
t.Fatalf("no funnel line for arm %s:\n%s", arm, out)
|
||||
return 0
|
||||
}
|
||||
|
||||
// The batch report states both counts on one line, so a reader of the scrollback
|
||||
// sees the scored set and the failures without counting funnel rows.
|
||||
func reportCounts(t *testing.T, out string) (scored, failed int) {
|
||||
t.Helper()
|
||||
for line := range strings.SplitSeq(out, "\n") {
|
||||
if !strings.Contains(line, " scored=") || !strings.Contains(line, " failed=") {
|
||||
continue
|
||||
}
|
||||
for f := range strings.FieldsSeq(line) {
|
||||
switch {
|
||||
case strings.HasPrefix(f, "scored="):
|
||||
scored, _ = strconv.Atoi(strings.TrimPrefix(f, "scored="))
|
||||
case strings.HasPrefix(f, "failed="):
|
||||
failed, _ = strconv.Atoi(strings.TrimPrefix(f, "failed="))
|
||||
}
|
||||
}
|
||||
return scored, failed
|
||||
}
|
||||
t.Fatalf("no line reports scored= and failed=:\n%s", out)
|
||||
return 0, 0
|
||||
}
|
||||
|
||||
// A batch that mixes a task that never ran with one that did must read as its
|
||||
// successful runs only: the failed task leaves Scored, the funnel, and the
|
||||
// completed count, and is reported as a failure with its identity instead.
|
||||
// Every assertion here is a number, not the absence of an empty row.
|
||||
func TestMixedBatchCountsOnlyTheRunThatProducedEvidence(t *testing.T) {
|
||||
blocked, ok := mixedTaskPair()
|
||||
_, runs, manifest, out := runBlockedExperiment(t, []contextTask{blocked, ok}, blocked.ID)
|
||||
|
||||
wantScored, wantFailed := len(armsFor(ok)), len(armsFor(blocked))
|
||||
if manifest.CompletedRuns != wantScored || manifest.FailedRuns != wantFailed {
|
||||
t.Fatalf("manifest completed=%d failed=%d, want %d and %d (planned=%d)",
|
||||
manifest.CompletedRuns, manifest.FailedRuns, wantScored, wantFailed, manifest.PlannedRuns)
|
||||
}
|
||||
if len(runs) != manifest.CompletedRuns+manifest.FailedRuns {
|
||||
t.Fatalf("results.jsonl has %d rows but completed=%d failed=%d",
|
||||
len(runs), manifest.CompletedRuns, manifest.FailedRuns)
|
||||
}
|
||||
for _, id := range manifest.TaskIDs {
|
||||
if id == blocked.ID {
|
||||
t.Fatalf("task_ids lists %s, which produced no evidence: %v", blocked.ID, manifest.TaskIDs)
|
||||
}
|
||||
}
|
||||
scored, failures := 0, 0
|
||||
for _, m := range runs {
|
||||
if m.Failed {
|
||||
failures++
|
||||
if m.Task != blocked.ID || m.FailureStage == "" || m.Reason == "" {
|
||||
t.Fatalf("failed row lost its identity or stage: %+v", m)
|
||||
}
|
||||
continue
|
||||
}
|
||||
scored++
|
||||
if m.Task != ok.ID {
|
||||
t.Fatalf("a run scored from the wrong task: %+v", m)
|
||||
}
|
||||
}
|
||||
if scored != wantScored || failures != wantFailed {
|
||||
t.Fatalf("rows scored=%d failed=%d, want %d and %d", scored, failures, wantScored, wantFailed)
|
||||
}
|
||||
// Each arm's funnel holds exactly the successful task's runs, and the two
|
||||
// scored counts agree with each other and with the manifest.
|
||||
total := 0
|
||||
for _, arm := range armsFor(ok) {
|
||||
total += funnelScored(t, out, arm.name)
|
||||
}
|
||||
if total != manifest.CompletedRuns {
|
||||
t.Fatalf("funnels scored %d runs, manifest completed_runs = %d:\n%s", total, manifest.CompletedRuns, out)
|
||||
}
|
||||
if gotScored, gotFailed := reportCounts(t, out); gotScored != manifest.CompletedRuns || gotFailed != manifest.FailedRuns {
|
||||
t.Fatalf("report says scored=%d failed=%d, manifest says %d and %d",
|
||||
gotScored, gotFailed, manifest.CompletedRuns, manifest.FailedRuns)
|
||||
}
|
||||
if strings.Contains(out, "## Where it broke") && strings.Contains(out, "=-\n") {
|
||||
t.Fatalf("the stage table kept an empty stage:\n%s", out)
|
||||
}
|
||||
}
|
||||
|
||||
// A batch where every run fails must not read as an empty success: zero scored
|
||||
// runs, a nonzero failed count, and both stated.
|
||||
func TestAllFailedBatchReportsZeroScoredAndNonzeroFailed(t *testing.T) {
|
||||
blocked, _ := mixedTaskPair()
|
||||
_, runs, manifest, out := runBlockedExperiment(t, []contextTask{blocked}, blocked.ID)
|
||||
|
||||
if manifest.CompletedRuns != 0 {
|
||||
t.Fatalf("completed_runs = %d, want 0 (nothing produced evidence)", manifest.CompletedRuns)
|
||||
}
|
||||
if manifest.FailedRuns != manifest.PlannedRuns || manifest.FailedRuns == 0 {
|
||||
t.Fatalf("failed_runs = %d, planned_runs = %d: every planned run failed",
|
||||
manifest.FailedRuns, manifest.PlannedRuns)
|
||||
}
|
||||
if len(manifest.TaskIDs) != 0 {
|
||||
t.Fatalf("task_ids = %v, want none", manifest.TaskIDs)
|
||||
}
|
||||
for i, m := range runs {
|
||||
if !m.Failed || m.FailureStage == "" || m.Task != blocked.ID {
|
||||
t.Fatalf("results.jsonl line %d is not a marked failure: %+v", i+1, m)
|
||||
}
|
||||
}
|
||||
total := 0
|
||||
for _, arm := range armsFor(blocked) {
|
||||
total += funnelScored(t, out, arm.name)
|
||||
}
|
||||
if total != 0 {
|
||||
t.Fatalf("funnels scored %d runs although nothing ran:\n%s", total, out)
|
||||
}
|
||||
if gotScored, gotFailed := reportCounts(t, out); gotScored != 0 || gotFailed != manifest.FailedRuns {
|
||||
t.Fatalf("report says scored=%d failed=%d, want 0 and %d", gotScored, gotFailed, manifest.FailedRuns)
|
||||
}
|
||||
}
|
||||
|
||||
// The control: with nothing blocked, a batch is what it always was. Every run
|
||||
// is completed with a stage, two identical batches write byte-identical results
|
||||
// and the same manifest header, and the new failure field stays out of it.
|
||||
func TestAllSuccessfulBatchKeepsItsArtifacts(t *testing.T) {
|
||||
tasks := indexTasks()
|
||||
first, runs, manifest, out := runBlockedExperiment(t, tasks)
|
||||
second, again, repeat, _ := runBlockedExperiment(t, tasks)
|
||||
if manifest.CompletedRuns != len(runs) || manifest.FailedRuns != 0 {
|
||||
t.Fatalf("completed=%d failed=%d rows=%d, want every row completed",
|
||||
manifest.CompletedRuns, manifest.FailedRuns, len(runs))
|
||||
}
|
||||
if manifest.CompletedRuns != manifest.PlannedRuns {
|
||||
t.Fatalf("completed=%d planned=%d, want all planned", manifest.CompletedRuns, manifest.PlannedRuns)
|
||||
}
|
||||
for i, m := range runs {
|
||||
if m.Failed || m.Task == "" || m.Arm == "" || m.FailureStage == "" {
|
||||
t.Fatalf("results.jsonl line %d is not a completed run: %+v", i+1, m)
|
||||
}
|
||||
}
|
||||
if gotScored, gotFailed := reportCounts(t, out); gotScored != manifest.CompletedRuns || gotFailed != 0 {
|
||||
t.Fatalf("report says scored=%d failed=%d, want %d and 0", gotScored, gotFailed, manifest.CompletedRuns)
|
||||
}
|
||||
if !reflect.DeepEqual(runs, again) {
|
||||
t.Fatal("two identical successful batches scored differently")
|
||||
}
|
||||
// Byte-for-byte on the runs. Trajectories carry per-message wall-clock
|
||||
// stamps, so they are compared field by field instead.
|
||||
if got, want := readFile(t, second, "results.jsonl"), readFile(t, first, "results.jsonl"); got != want {
|
||||
t.Fatalf("an unchanged batch wrote different results.jsonl:\n%s\n---\n%s", want, got)
|
||||
}
|
||||
gotTraj, wantTraj := withoutStamps(t, readFile(t, second, "trajectories.jsonl")),
|
||||
withoutStamps(t, readFile(t, first, "trajectories.jsonl"))
|
||||
if gotTraj != wantTraj {
|
||||
t.Fatalf("an unchanged batch wrote different trajectories:\n%s\n---\n%s", wantTraj, gotTraj)
|
||||
}
|
||||
repeat.StartedAt, repeat.FinishedAt = manifest.StartedAt, manifest.FinishedAt
|
||||
repeat.TrajectoriesSHA256 = manifest.TrajectoriesSHA256
|
||||
if !reflect.DeepEqual(repeat, manifest) {
|
||||
t.Fatalf("an unchanged batch wrote a different manifest:\n%+v\n---\n%+v", manifest, repeat)
|
||||
}
|
||||
if repeat.FailedRuns != 0 {
|
||||
t.Fatalf("a batch with no failure reports failed_runs=%d", repeat.FailedRuns)
|
||||
}
|
||||
}
|
||||
|
||||
// withoutStamps drops the createdAt stamp every message carries, which is the
|
||||
// one field a dry batch cannot reproduce between two runs.
|
||||
func withoutStamps(t *testing.T, text string) string {
|
||||
t.Helper()
|
||||
var b strings.Builder
|
||||
for line := range strings.SplitSeq(strings.TrimSpace(text), "\n") {
|
||||
var row map[string]any
|
||||
if err := json.Unmarshal([]byte(line), &row); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
stripStamp(row)
|
||||
out, err := json.Marshal(row)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b.Write(out)
|
||||
b.WriteByte('\n')
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
func stripStamp(v any) {
|
||||
switch typed := v.(type) {
|
||||
case map[string]any:
|
||||
delete(typed, "createdAt")
|
||||
delete(typed, "workDurationMs")
|
||||
for _, child := range typed {
|
||||
stripStamp(child)
|
||||
}
|
||||
case []any:
|
||||
for _, child := range typed {
|
||||
stripStamp(child)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func readFile(t *testing.T, root, name string) string {
|
||||
t.Helper()
|
||||
data, err := os.ReadFile(filepath.Join(root, name))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return string(data)
|
||||
}
|
||||
@@ -0,0 +1,205 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reasonix/internal/state/sessionstore"
|
||||
"strings"
|
||||
|
||||
"reasonix/internal/contract/ablation"
|
||||
"reasonix/internal/contract/event"
|
||||
"reasonix/internal/contract/provider"
|
||||
"reasonix/internal/contract/tool"
|
||||
"reasonix/internal/runtime/agent"
|
||||
)
|
||||
|
||||
// A fixture is a real session that has really been compacted: no bench-only
|
||||
// state shape, so the loader and the projection restore have to work on it.
|
||||
|
||||
const (
|
||||
fixtureWindow = 60000
|
||||
fixtureGenerations = 5
|
||||
// fillerBytes per unit is what makes a fold necessary rather than optional.
|
||||
fillerBytes = 6000
|
||||
)
|
||||
|
||||
// digestProvider answers a summarizer without naming anything the corpus
|
||||
// plants. A digest that happened to quote the answer would leave it visible
|
||||
// and silently turn the task into a no-op.
|
||||
type digestProvider struct{ calls int }
|
||||
|
||||
func (p *digestProvider) Name() string { return "fixture-digest" }
|
||||
|
||||
func (p *digestProvider) Stream(_ context.Context, _ provider.Request) (<-chan provider.Chunk, error) {
|
||||
p.calls++
|
||||
ch := make(chan provider.Chunk, 2)
|
||||
ch <- provider.Chunk{Type: provider.ChunkText, Text: "## Earlier work\nEarlier work continued across several files; nothing outstanding was recorded."}
|
||||
ch <- provider.Chunk{Type: provider.ChunkDone}
|
||||
close(ch)
|
||||
return ch, nil
|
||||
}
|
||||
|
||||
// fillerCall is one ordinary read whose only job is to take up room — in the
|
||||
// window, and in the fold index ahead of a cue.
|
||||
func fillerCall(sess *sessionstore.Session, n int) {
|
||||
id := fmt.Sprintf("f%04d", n)
|
||||
sess.Add(provider.Message{Role: provider.RoleAssistant, ToolCalls: []provider.ToolCall{
|
||||
{ID: id, Name: "read_file", Arguments: fmt.Sprintf(`{"path":"internal/mod%03d/handler%03d.go"}`, n%37, n)},
|
||||
}})
|
||||
sess.Add(provider.Message{Role: provider.RoleTool, ToolCallID: id, Name: "read_file", Content: fillerBody(n)})
|
||||
}
|
||||
|
||||
func fillerBody(n int) string {
|
||||
var b strings.Builder
|
||||
fmt.Fprintf(&b, "package mod%03d\n\n", n%37)
|
||||
for b.Len() < fillerBytes {
|
||||
fmt.Fprintf(&b, "func handler%03dStep%02d() error { return nil }\n", n, b.Len()%64)
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// builtFixture is one task's history after real folds, plus where its target
|
||||
// ended up.
|
||||
type builtFixture struct {
|
||||
Session *sessionstore.Session
|
||||
State sessionstore.CompactionState
|
||||
Target int
|
||||
Path string
|
||||
Instance fixtureInstance
|
||||
}
|
||||
|
||||
// buildFixture plays a task's history through the real agent and folds it at
|
||||
// the given arm, writing the session and its context state where the agent
|
||||
// itself would.
|
||||
func buildFixture(f fixtureInstance, arm ablation.Set, sessionPath string) (builtFixture, error) {
|
||||
t := f.Task
|
||||
sess := sessionstore.NewSession("You are a coding agent.")
|
||||
sess.Add(provider.Message{Role: provider.RoleUser, Content: "Work through the transport and scheduler backlog."})
|
||||
|
||||
n := 0
|
||||
at := -1
|
||||
|
||||
a := agent.New(&digestProvider{}, tool.NewRegistry(), sess, agent.Options{
|
||||
ContextWindow: fixtureWindow, CompactRatio: 0.5, RecentKeep: 2,
|
||||
SessionPath: sessionPath, KeepPolicy: agent.KeepErrors, Ablation: arm,
|
||||
}, event.Discard)
|
||||
|
||||
for gen := range fixtureGenerations {
|
||||
if gen == t.PlantAfterGen {
|
||||
at = f.plant(sess)
|
||||
}
|
||||
for range 8 {
|
||||
fillerCall(sess, n)
|
||||
n++
|
||||
}
|
||||
if _, err := a.CompactNow(context.Background(), agent.CompactRequest{}); err != nil {
|
||||
return builtFixture{}, fmt.Errorf("%s: generation %d fold: %w", t.ID, gen, err)
|
||||
}
|
||||
}
|
||||
if at < 0 {
|
||||
return builtFixture{}, fmt.Errorf("%s: PlantAfterGen %d is past the last generation", t.ID, t.PlantAfterGen)
|
||||
}
|
||||
// projectionCoversTail cannot revalidate a projection covering the whole
|
||||
// transcript: it needs a live tail, which a real session always has because
|
||||
// work continues after a fold. Neutral text — the tail is provider-visible.
|
||||
sess.Add(provider.Message{Role: provider.RoleAssistant, Content: "Backlog swept; ready for the next question."})
|
||||
if err := sess.Save(sessionPath); err != nil {
|
||||
return builtFixture{}, fmt.Errorf("%s: save session: %w", t.ID, err)
|
||||
}
|
||||
state, ok, err := sessionstore.LoadCompactionState(sessionPath)
|
||||
if err != nil {
|
||||
return builtFixture{}, fmt.Errorf("%s: load context state: %w", t.ID, err)
|
||||
}
|
||||
if !ok || len(state.Projection.Messages) == 0 {
|
||||
return builtFixture{}, fmt.Errorf("%s: no projection was installed", t.ID)
|
||||
}
|
||||
if state.Projection.CoveredCount <= at {
|
||||
return builtFixture{}, fmt.Errorf("%s: target #%d is not inside the folded region (covered %d)",
|
||||
t.ID, at, state.Projection.CoveredCount)
|
||||
}
|
||||
return builtFixture{Session: sess, State: state, Target: at, Path: sessionPath, Instance: f}, nil
|
||||
}
|
||||
|
||||
// visibleContext is the model-visible view: the stored projection spliced with
|
||||
// everything canonical that came after it.
|
||||
func visibleContext(f builtFixture) []provider.Message {
|
||||
canonical := f.Session.Snapshot()
|
||||
out := append([]provider.Message(nil), f.State.Projection.Messages...)
|
||||
if n := f.State.Projection.CoveredCount; n >= 0 && n < len(canonical) {
|
||||
out = append(out, canonical[n:]...)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// visibleText is everything the model would read, as one string.
|
||||
func visibleText(f builtFixture) string {
|
||||
var b strings.Builder
|
||||
for _, m := range visibleContext(f) {
|
||||
b.WriteString(m.Content)
|
||||
b.WriteString("\n")
|
||||
for _, tc := range m.ToolCalls {
|
||||
b.WriteString(tc.Name)
|
||||
b.WriteString(" ")
|
||||
b.WriteString(string(tc.Arguments))
|
||||
b.WriteString("\n")
|
||||
}
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// sealFixture removes the fixture from disk once the agent holds it in memory.
|
||||
// The canonical transcript is the answer, and the sandbox mounts the host
|
||||
// read-only by design — so a model that finds the path reads the answer, which
|
||||
// one did. The agent has already loaded the session and its sidecar by the time
|
||||
// this runs; a failed autosave afterwards costs the benchmark nothing.
|
||||
func sealFixture(sessionPath string) {
|
||||
// Every file, not two named ones: a session writes an event log and a
|
||||
// context sidecar beside the transcript, and session.events.jsonl still
|
||||
// carried the answer after the other two were sealed by name.
|
||||
dir := filepath.Dir(sessionPath)
|
||||
entries, err := os.ReadDir(dir)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
for _, e := range entries {
|
||||
if !e.IsDir() {
|
||||
_ = os.Remove(filepath.Join(dir, e.Name()))
|
||||
}
|
||||
}
|
||||
// Deny the directory too, so an autosave cannot recreate what was removed.
|
||||
_ = os.Chmod(dir, 0o500)
|
||||
}
|
||||
|
||||
// unsealFixture restores the directory so the harness can clean up after
|
||||
// itself.
|
||||
func unsealFixture(sessionPath string) {
|
||||
_ = os.Chmod(filepath.Dir(sessionPath), 0o700)
|
||||
}
|
||||
|
||||
// answerOnDisk reports any readable file under dir holding a current-run
|
||||
// answer. Two named files were sealed because two leaks were found in them;
|
||||
// this scans the directory, so a path that moves — an autosave, a recovery
|
||||
// tombstone, a future sidecar — fails here instead of on a paid run.
|
||||
func answerOnDisk(dir string, markers []string) (string, string) {
|
||||
var atPath, found string
|
||||
_ = filepath.WalkDir(dir, func(path string, d fs.DirEntry, err error) error {
|
||||
if err != nil || d.IsDir() || atPath != "" {
|
||||
return nil //nolint:nilerr // an unreadable entry is not a leak
|
||||
}
|
||||
body, readErr := os.ReadFile(path)
|
||||
if readErr != nil {
|
||||
return nil
|
||||
}
|
||||
for _, marker := range markers {
|
||||
if len(marker) >= 6 && strings.Contains(string(body), marker) {
|
||||
atPath, found = path, marker
|
||||
return filepath.SkipAll
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
return atPath, found
|
||||
}
|
||||
@@ -0,0 +1,139 @@
|
||||
// instance.go — a task template becomes one run's concrete history.
|
||||
package main
|
||||
|
||||
import (
|
||||
"crypto/rand"
|
||||
"fmt"
|
||||
"hash/fnv"
|
||||
"io"
|
||||
mathrand "math/rand"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// The invariant this file exists for: no literal sufficient to satisfy
|
||||
// AnswerRecovered for a concrete run may exist in this repository before that
|
||||
// run begins. Grepping the corpus reveals the question, never the answer.
|
||||
|
||||
// varKind is how one placeholder's value is generated. Values stay shaped like
|
||||
// real engineering records — a UUID would defeat the leak but turn every task
|
||||
// into "find the random string".
|
||||
type varKind string
|
||||
|
||||
const (
|
||||
varCodename varKind = "codename" // cobalt-k7m4q9
|
||||
varInt varKind = "int" // 731
|
||||
varDecimal varKind = "decimal" // 0.0437
|
||||
varDuration varKind = "duration" // 43ms
|
||||
)
|
||||
|
||||
// varSpec declares one placeholder. Min/Max bound the numeric kinds.
|
||||
type varSpec struct {
|
||||
Name string
|
||||
Kind varKind
|
||||
Word string // codename stem, e.g. "cobalt"
|
||||
Min int
|
||||
Max int
|
||||
}
|
||||
|
||||
// codenameEntropy is how much of a codename is unguessable. Five base32 digits
|
||||
// is about 25 bits: far past what a model can reach by knowing the stem.
|
||||
const codenameEntropy = 5
|
||||
|
||||
const codenameAlphabet = "abcdefghijkmnpqrstuvwxyz23456789"
|
||||
|
||||
// fixtureVars is one run's instantiated values.
|
||||
type fixtureVars map[string]string
|
||||
|
||||
// newVars generates values for every spec. A nil rng means deterministic.
|
||||
func newVars(specs []varSpec, rng *mathrand.Rand) fixtureVars {
|
||||
out := fixtureVars{}
|
||||
for _, spec := range specs {
|
||||
switch spec.Kind {
|
||||
case varCodename:
|
||||
out[spec.Name] = spec.Word + "-" + randomToken(rng, codenameEntropy)
|
||||
case varInt:
|
||||
out[spec.Name] = fmt.Sprint(randomBetween(rng, spec.Min, spec.Max))
|
||||
case varDecimal:
|
||||
out[spec.Name] = fmt.Sprintf("0.%04d", randomBetween(rng, spec.Min, spec.Max))
|
||||
case varDuration:
|
||||
out[spec.Name] = fmt.Sprintf("%dms", randomBetween(rng, spec.Min, spec.Max))
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func randomToken(rng *mathrand.Rand, n int) string {
|
||||
var b strings.Builder
|
||||
for range n {
|
||||
b.WriteByte(codenameAlphabet[randomBetween(rng, 0, len(codenameAlphabet)-1)])
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
func randomBetween(rng *mathrand.Rand, lo, hi int) int {
|
||||
if hi <= lo {
|
||||
return lo
|
||||
}
|
||||
return lo + rng.Intn(hi-lo+1)
|
||||
}
|
||||
|
||||
// seededRand is the deterministic generator preflight and tests use, so a
|
||||
// fixture's tier calibration and its golden state reproduce exactly.
|
||||
func seededRand(parts ...string) *mathrand.Rand {
|
||||
h := fnv.New64a()
|
||||
for _, p := range parts {
|
||||
h.Write([]byte(p))
|
||||
h.Write([]byte{0})
|
||||
}
|
||||
return mathrand.New(mathrand.NewSource(int64(h.Sum64())))
|
||||
}
|
||||
|
||||
// liveRand is the generator a paid run uses: every run gets values that existed
|
||||
// nowhere before it started.
|
||||
func liveRand() *mathrand.Rand {
|
||||
rng, err := liveRandFrom(rand.Reader)
|
||||
if err != nil {
|
||||
panic("contextbench: no entropy for a live run: " + err.Error())
|
||||
}
|
||||
return rng
|
||||
}
|
||||
|
||||
// liveRandFrom seeds a generator from an entropy source, so what a run's values
|
||||
// are unpredictable from is the source and nothing else.
|
||||
func liveRandFrom(entropy io.Reader) (*mathrand.Rand, error) {
|
||||
var seed [8]byte
|
||||
if _, err := io.ReadFull(entropy, seed[:]); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var n int64
|
||||
for _, b := range seed {
|
||||
n = n<<8 | int64(b)
|
||||
}
|
||||
return mathrand.New(mathrand.NewSource(n)), nil
|
||||
}
|
||||
|
||||
// instantiate substitutes {{name}} and refuses anything left over. The grammar
|
||||
// is one identifier in double braces: no conditionals, no loops, no defaults.
|
||||
// This is fixture data, not a template language.
|
||||
func instantiate(text string, vars fixtureVars) (string, error) {
|
||||
out := text
|
||||
for name, value := range vars {
|
||||
out = strings.ReplaceAll(out, "{{"+name+"}}", value)
|
||||
}
|
||||
if strings.Contains(out, "{{") || strings.Contains(out, "}}") {
|
||||
return "", fmt.Errorf("unresolved placeholder in %q", firstPlaceholder(out))
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func firstPlaceholder(text string) string {
|
||||
i := strings.Index(text, "{{")
|
||||
if i < 0 {
|
||||
return text
|
||||
}
|
||||
j := strings.Index(text[i:], "}}")
|
||||
if j < 0 {
|
||||
return text[i:min(i+40, len(text))]
|
||||
}
|
||||
return text[i : i+j+2]
|
||||
}
|
||||
@@ -0,0 +1,144 @@
|
||||
// ContextRetrievalBench builds the fixtures the long-horizon retrieval
|
||||
// experiments run against, and proves they are worth paying a provider for.
|
||||
//
|
||||
// -mode=calibrate find each index task's lead-call count for its cue tier
|
||||
// -mode=preflight build every task under every arm and check the contract
|
||||
//
|
||||
// Nothing here calls a provider. The experiments themselves do; this is what
|
||||
// says they would measure what they claim to.
|
||||
package main
|
||||
|
||||
import (
|
||||
"flag"
|
||||
"fmt"
|
||||
"os"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
func main() {
|
||||
mode := flag.String("mode", "preflight", "calibrate | preflight | snippet | adversarial | run-search | run-boundary | run-index")
|
||||
work := flag.String("work", "", "directory for built fixtures (default: a temp dir)")
|
||||
only := flag.String("task", "", "limit to one task id")
|
||||
dry := flag.Bool("dry", false, "run modes: drive the pipeline with a scripted provider instead of paying for one")
|
||||
flag.Parse()
|
||||
|
||||
if err := validateCorpus(); err != nil {
|
||||
fmt.Fprintln(os.Stderr, "corpus:", err)
|
||||
os.Exit(2)
|
||||
}
|
||||
root := *work
|
||||
if root == "" {
|
||||
dir, err := os.MkdirTemp("", "context-retrieval-*")
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(2)
|
||||
}
|
||||
defer os.RemoveAll(dir)
|
||||
root = dir
|
||||
}
|
||||
|
||||
tasks := allTasks()
|
||||
if *only != "" {
|
||||
t, ok := taskByID(*only)
|
||||
if !ok {
|
||||
fmt.Fprintf(os.Stderr, "unknown task %q\n", *only)
|
||||
os.Exit(2)
|
||||
}
|
||||
tasks = []contextTask{t}
|
||||
}
|
||||
|
||||
switch *mode {
|
||||
case "calibrate":
|
||||
os.Exit(runCalibrate(tasks, root))
|
||||
case "preflight":
|
||||
os.Exit(runPreflight(tasks, root))
|
||||
case "run-search":
|
||||
os.Exit(runExperiment(*mode, experimentSearch, root, *dry, tasks))
|
||||
case "run-boundary":
|
||||
os.Exit(runBoundaries(*mode, root))
|
||||
case "snippet":
|
||||
os.Exit(runSnippetAudit(root, 100))
|
||||
case "adversarial":
|
||||
os.Exit(runAdversarial(root))
|
||||
case "run-index":
|
||||
batch := tasks
|
||||
if *only == "" {
|
||||
// A batch measures the efficiency substrate only. Naming one task
|
||||
// with -task runs that task, role included, so the stopping corpus
|
||||
// stays measurable on its own.
|
||||
batch = efficiencyTasks()
|
||||
}
|
||||
os.Exit(runExperiment(*mode, experimentIndex, root, *dry, batch))
|
||||
default:
|
||||
fmt.Fprintf(os.Stderr, "unknown mode %q\n", *mode)
|
||||
os.Exit(2)
|
||||
}
|
||||
}
|
||||
|
||||
func runPreflight(tasks []contextTask, root string) int {
|
||||
var all []finding
|
||||
for _, t := range tasks {
|
||||
found, err := checkTask(t, root)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "%s: %v\n", t.ID, err)
|
||||
return 2
|
||||
}
|
||||
all = append(all, found...)
|
||||
status := "ok"
|
||||
if len(found) > 0 {
|
||||
status = fmt.Sprintf("%d problem(s)", len(found))
|
||||
}
|
||||
fmt.Printf("%-24s %-6s gen=%-2d %s\n", t.ID, t.Experiment, t.PlantAfterGen, status)
|
||||
}
|
||||
if len(all) == 0 {
|
||||
fmt.Printf("\n%d fixtures ready.\n", len(tasks))
|
||||
return 0
|
||||
}
|
||||
fmt.Fprintf(os.Stderr, "\n%d preflight problem(s):\n", len(all))
|
||||
sort.Slice(all, func(i, j int) bool { return all[i].Task < all[j].Task })
|
||||
for _, f := range all {
|
||||
fmt.Fprintln(os.Stderr, " ", f)
|
||||
}
|
||||
return 1
|
||||
}
|
||||
|
||||
// runCalibrate reports, for each index task, the lead-call counts at which its
|
||||
// cue is still addressed by each budget. The corpus freezes one number from
|
||||
// this; it is never computed at run time, or the implementation under test
|
||||
// would be moving its own goalposts.
|
||||
func runCalibrate(tasks []contextTask, root string) int {
|
||||
for _, t := range tasks {
|
||||
if t.Experiment != experimentIndex {
|
||||
continue
|
||||
}
|
||||
var rows []string
|
||||
for lead := range fixtureGenerations {
|
||||
probe := t
|
||||
probe.PlantAfterGen = lead
|
||||
var visibleAt []string
|
||||
for _, scale := range armScales {
|
||||
inst, ierr := instantiateTask(probe, seededRand(t.ID, scale, "calibrate"))
|
||||
if ierr != nil {
|
||||
fmt.Fprintln(os.Stderr, ierr)
|
||||
return 2
|
||||
}
|
||||
f, err := buildFixture(inst, armFor(scale, false),
|
||||
fmt.Sprintf("%s/calib/%s/%d-%s/session.jsonl", root, t.ID, lead, scale))
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "%s gen=%d %s: %v\n", t.ID, lead, scale, err)
|
||||
return 2
|
||||
}
|
||||
if strings.Contains(visibleText(f), inst.CueMarker) {
|
||||
visibleAt = append(visibleAt, scale)
|
||||
}
|
||||
}
|
||||
rows = append(rows, fmt.Sprintf("planted after gen %d, cue visible at: %s", lead, strings.Join(visibleAt, ",")))
|
||||
}
|
||||
fmt.Printf("\n%s (wants tier %s)\n", t.ID, t.CueTier)
|
||||
for _, r := range rows {
|
||||
fmt.Println(" ", r)
|
||||
}
|
||||
}
|
||||
return 0
|
||||
}
|
||||
@@ -0,0 +1,189 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"maps"
|
||||
mathrand "math/rand"
|
||||
"os/exec"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// The same checks the preflight mode runs, under go test: a compaction change
|
||||
// that invalidates the fixtures fails here rather than three hundred API calls
|
||||
// later.
|
||||
|
||||
func TestCorpusIsWellFormed(t *testing.T) {
|
||||
if err := validateCorpus(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := len(searchTasks()); got != 6 {
|
||||
t.Errorf("search experiment has %d tasks, want 6", got)
|
||||
}
|
||||
if got := len(indexTasks()); got != 12 {
|
||||
t.Errorf("index experiment has %d tasks, want 12", got)
|
||||
}
|
||||
stopping := map[string]bool{}
|
||||
tiers := map[string]int{}
|
||||
for _, task := range indexTasks() {
|
||||
if task.Role == roleStopping {
|
||||
stopping[task.ID] = true
|
||||
continue
|
||||
}
|
||||
tiers[task.CueTier]++
|
||||
}
|
||||
if !stopping["i03-coalescing"] || !stopping["i04-recovery-fence"] {
|
||||
t.Errorf("the stopping corpus is %v; it must keep i03-coalescing and i04-recovery-fence", stopping)
|
||||
}
|
||||
// Two per tier, over the efficiency substrate, is what makes the staircase
|
||||
// readable: the same task can be compared across the boundary its cue sits
|
||||
// on, instead of two different questions being compared across two arms.
|
||||
for _, tier := range []string{tierQuarter, tierHalf, tierDefault} {
|
||||
if tiers[tier] != 2 {
|
||||
t.Errorf("tier %s has %d tasks, want 2", tier, tiers[tier])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The role decides which batch a task enters; the literal lists are only where
|
||||
// it is declared, so the two must not drift.
|
||||
func TestIndexListsMatchTheirRoles(t *testing.T) {
|
||||
for _, task := range append(append(indexQuarterTasks(), indexHalfTasks()...), indexDefaultTasks()...) {
|
||||
if task.Role != roleEfficiency {
|
||||
t.Errorf("%s sits in an efficiency list with role %q", task.ID, task.Role)
|
||||
}
|
||||
}
|
||||
for _, task := range stoppingIndexTasks() {
|
||||
if task.Role != roleStopping {
|
||||
t.Errorf("%s sits in the stopping list with role %q", task.ID, task.Role)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The invariant the whole instantiation layer exists for: no literal a scorer
|
||||
// needs may be readable in this repository before a run starts.
|
||||
func TestNoAnswerLiteralExistsInTheRepository(t *testing.T) {
|
||||
root, err := repoRoot()
|
||||
if err != nil {
|
||||
t.Skip(err)
|
||||
}
|
||||
for _, task := range allTasks() {
|
||||
inst, err := instantiateTask(task, seededRand(task.ID, "leak-check"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, marker := range append(inst.AnswerMarkers, inst.CueMarker) {
|
||||
if len(marker) < 6 {
|
||||
continue // a bare number is not a literal anyone can grep for
|
||||
}
|
||||
if found, where := grepRepo(t, root, marker); found {
|
||||
t.Errorf("%s: %q is readable at %s before the run", task.ID, marker, where)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A run's values are unpredictable from a previous run only if the live
|
||||
// generator is seeded from entropy. Comparing answers across two instances
|
||||
// cannot assert that: a task whose answer is a bounded number collides by
|
||||
// chance (1/8601 to 1/286 per task), so the check is on the seed source.
|
||||
func TestLiveRandDrawsFromEntropy(t *testing.T) {
|
||||
a, b := liveRand(), liveRand()
|
||||
for range 4 {
|
||||
if a.Int63() != b.Int63() {
|
||||
return
|
||||
}
|
||||
}
|
||||
t.Fatal("two live generators produced the same stream: they are seeded alike")
|
||||
}
|
||||
|
||||
// The source alone decides an instance: the same entropy gives the same
|
||||
// answers, different entropy gives different values.
|
||||
func TestLiveRandSourceDecidesTheInstance(t *testing.T) {
|
||||
same := func() *mathrand.Rand {
|
||||
rng, err := liveRandFrom(bytes.NewReader(bytes.Repeat([]byte{7}, 8)))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return rng
|
||||
}
|
||||
other, err := liveRandFrom(bytes.NewReader(bytes.Repeat([]byte{9}, 8)))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := liveRandFrom(bytes.NewReader(nil)); err == nil {
|
||||
t.Fatal("an empty entropy source was accepted")
|
||||
}
|
||||
for _, task := range allTasks() {
|
||||
a, _ := instantiateTask(task, same())
|
||||
b, _ := instantiateTask(task, same())
|
||||
if !slices.Equal(a.AnswerMarkers, b.AnswerMarkers) || !maps.Equal(a.Vars, b.Vars) {
|
||||
t.Errorf("%s: the same entropy gave different instances", task.ID)
|
||||
}
|
||||
c, _ := instantiateTask(task, other)
|
||||
if maps.Equal(a.Vars, c.Vars) {
|
||||
t.Errorf("%s: different entropy gave the same values", task.ID)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A preflight that could not reproduce would calibrate against one run's
|
||||
// nonces and report about another's.
|
||||
func TestSeededInstancesReproduce(t *testing.T) {
|
||||
for _, task := range allTasks() {
|
||||
a, _ := instantiateTask(task, seededRand(task.ID, "x"))
|
||||
b, _ := instantiateTask(task, seededRand(task.ID, "x"))
|
||||
if !slices.Equal(a.AnswerMarkers, b.AnswerMarkers) || a.body != b.body {
|
||||
t.Errorf("%s did not reproduce under the same seed", task.ID)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// An unresolved placeholder would put "{{marker}}" in the transcript and score
|
||||
// every run wrong; it fails the fixture instead.
|
||||
func TestUnresolvedPlaceholderIsRefused(t *testing.T) {
|
||||
if _, err := instantiate("value is {{missing}}", fixtureVars{"other": "x"}); err == nil {
|
||||
t.Fatal("an unresolved placeholder was accepted")
|
||||
}
|
||||
}
|
||||
|
||||
func repoRoot() (string, error) {
|
||||
out, err := exec.Command("git", "rev-parse", "--show-toplevel").Output()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("not in a git checkout: %w", err)
|
||||
}
|
||||
return strings.TrimSpace(string(out)), nil
|
||||
}
|
||||
|
||||
// grepRepo searches tracked files only: build artifacts and temp fixtures are
|
||||
// not what a fresh checkout hands anyone.
|
||||
func grepRepo(t *testing.T, root, needle string) (bool, string) {
|
||||
t.Helper()
|
||||
cmd := exec.Command("git", "grep", "-l", "-F", needle)
|
||||
cmd.Dir = root
|
||||
out, err := cmd.Output()
|
||||
if err != nil {
|
||||
return false, "" // exit 1 means no match
|
||||
}
|
||||
return true, strings.TrimSpace(strings.SplitN(string(out), "\n", 2)[0])
|
||||
}
|
||||
|
||||
func TestEveryFixturePassesPreflight(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("builds and folds 36 sessions")
|
||||
}
|
||||
root := t.TempDir()
|
||||
for _, task := range allTasks() {
|
||||
t.Run(task.ID, func(t *testing.T) {
|
||||
found, err := checkTask(task, root)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, f := range found {
|
||||
t.Error(f)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,290 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
"reasonix/internal/base/retrieval"
|
||||
"reasonix/internal/contract/ablation"
|
||||
"reasonix/internal/contract/provider"
|
||||
)
|
||||
|
||||
// Preflight is what makes a real run worth paying for. An answer the model can
|
||||
// already read, or a cue not where the matrix says, turns an arm into a
|
||||
// different experiment without saying so.
|
||||
|
||||
// armScales are the fold-index arms an index task is measured across.
|
||||
var armScales = []string{"default", "half", "quarter", "off"}
|
||||
|
||||
// experimentArm is one measured configuration of a task.
|
||||
type experimentArm struct {
|
||||
name string
|
||||
scale string
|
||||
searchOff bool
|
||||
}
|
||||
|
||||
// armsFor is the single declaration of which arms a task runs at, so preflight
|
||||
// and the run itself cannot check one set and measure another.
|
||||
func armsFor(t contextTask) []experimentArm {
|
||||
if t.Experiment == experimentSearch {
|
||||
return []experimentArm{
|
||||
{"search-on", "default", false},
|
||||
{"search-off", "default", true},
|
||||
}
|
||||
}
|
||||
arms := make([]experimentArm, 0, len(armScales))
|
||||
for _, scale := range armScales {
|
||||
arms = append(arms, experimentArm{"index-" + scale, scale, false})
|
||||
}
|
||||
return arms
|
||||
}
|
||||
|
||||
func scaleOf(name string) ablation.FoldIndexScale {
|
||||
s, err := ablation.ParseFoldIndexScale(name)
|
||||
if err != nil {
|
||||
panic("unknown scale " + name) // the list above is a constant
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// armFor is the arm a task is measured under at one fold-index scale.
|
||||
func armFor(scale string, searchOff bool) ablation.Set {
|
||||
set := ablation.Set{}
|
||||
if searchOff {
|
||||
set = ablation.New(ablation.RecallSearch)
|
||||
}
|
||||
return set.WithFoldIndex(scaleOf(scale))
|
||||
}
|
||||
|
||||
type finding struct {
|
||||
Task, Arm, Problem string
|
||||
}
|
||||
|
||||
func (f finding) String() string { return fmt.Sprintf("%s [%s]: %s", f.Task, f.Arm, f.Problem) }
|
||||
|
||||
// checkTask builds the task under every arm it will be run at and reports what
|
||||
// does not hold.
|
||||
func checkTask(t contextTask, root string) ([]finding, error) {
|
||||
var out []finding
|
||||
for _, arm := range armsFor(t) {
|
||||
// Deterministic values: a preflight has to reproduce, and a tier
|
||||
// calibration measured against one run's nonces is measured against
|
||||
// nothing.
|
||||
inst, err := instantiateTask(t, seededRand(t.ID, arm.name, "preflight"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
dir := filepath.Join(root, t.ID, arm.name)
|
||||
f, err := buildFixture(inst, armFor(arm.scale, arm.searchOff), filepath.Join(dir, "session.jsonl"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
visible := visibleText(f)
|
||||
|
||||
// The answer must be unreadable in every arm, or the task measures
|
||||
// reading rather than recall.
|
||||
for _, marker := range inst.AnswerMarkers {
|
||||
if len(marker) < 3 {
|
||||
continue // a bare number is not a nonce; the whole set still has to match
|
||||
}
|
||||
if strings.Contains(visible, marker) {
|
||||
out = append(out, finding{t.ID, arm.name, fmt.Sprintf("answer marker %q is already visible", marker)})
|
||||
}
|
||||
}
|
||||
|
||||
// The corpus has to be findable at all, or a miss in a real run says
|
||||
// nothing about the model.
|
||||
rank, err := probeRank(f, inst.ProbeQuery)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
switch {
|
||||
case rank == 0:
|
||||
out = append(out, finding{t.ID, arm.name, fmt.Sprintf("probe query %q does not find the target at all", inst.ProbeQuery)})
|
||||
case rank > 5:
|
||||
out = append(out, finding{t.ID, arm.name, fmt.Sprintf("probe query %q ranks the target %d", inst.ProbeQuery, rank)})
|
||||
}
|
||||
|
||||
if t.Experiment == experimentIndex {
|
||||
out = append(out, checkCueVisibility(t, inst, arm.scale, arm.name, visible)...)
|
||||
} else if line := indexLineFor(f, f.Target); line != "" {
|
||||
out = append(out, finding{t.ID, arm.name,
|
||||
"a search task is addressed by the fold index, so it measures the index and not search: " + line})
|
||||
}
|
||||
|
||||
// The projection is what the agent believes. This is what the model is
|
||||
// told, and it is the one that has to hold.
|
||||
req, err := captureRequest(f, armFor(arm.scale, arm.searchOff), inst.Prompt)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s [%s]: capture: %w", t.ID, arm.name, err)
|
||||
}
|
||||
out = append(out, checkCapturedRequest(t, inst, arm.name, arm.scale, arm.searchOff, req)...)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// checkCapturedRequest holds one arm's real provider request to its contract:
|
||||
// the answer unreadable, the cue exactly where its tier says, the recall tool
|
||||
// offering only what this arm can do, and no host-side field surviving the
|
||||
// boundary.
|
||||
func checkCapturedRequest(t contextTask, inst fixtureInstance, armName, scale string, searchOff bool, req provider.Request) []finding {
|
||||
var out []finding
|
||||
for _, marker := range inst.AnswerMarkers {
|
||||
if len(marker) < 3 {
|
||||
continue // a bare number is not a nonce; the set as a whole still is
|
||||
}
|
||||
for _, at := range findInSurface(req, marker) {
|
||||
out = append(out, finding{t.ID, armName, fmt.Sprintf("answer marker %q leaked at %s", marker, at)})
|
||||
}
|
||||
}
|
||||
if t.Experiment == experimentIndex {
|
||||
wantVisible, wantHidden := tierScales(t.CueTier)
|
||||
where := findInSurface(req, inst.CueMarker)
|
||||
switch {
|
||||
case contains(wantVisible, scale) && len(where) == 0:
|
||||
out = append(out, finding{t.ID, armName, fmt.Sprintf("cue %q is not provider-visible; the tier expects it", inst.CueMarker)})
|
||||
case contains(wantHidden, scale) && len(where) > 0:
|
||||
out = append(out, finding{t.ID, armName, fmt.Sprintf("cue %q is provider-visible at %s: %s", inst.CueMarker, where[0], scale)})
|
||||
}
|
||||
}
|
||||
|
||||
schema, ok := recallToolSchema(req)
|
||||
if !ok {
|
||||
out = append(out, finding{t.ID, armName, "recall is not in the provider's tool list"})
|
||||
return append(out, boundaryFindings(t, armName, req)...)
|
||||
}
|
||||
params := ""
|
||||
if b, err := json.Marshal(schema.Parameters); err == nil {
|
||||
params = string(b)
|
||||
}
|
||||
// Both arms keep the read half: the ablation removes search, not recall.
|
||||
if !strings.Contains(params, `"positions"`) {
|
||||
out = append(out, finding{t.ID, armName, "recall lost its positions parameter"})
|
||||
}
|
||||
hasQuery := strings.Contains(params, `"query"`)
|
||||
switch {
|
||||
case searchOff && hasQuery:
|
||||
out = append(out, finding{t.ID, armName, "recall still offers a query parameter"})
|
||||
case !searchOff && !hasQuery:
|
||||
out = append(out, finding{t.ID, armName, "recall has no query parameter"})
|
||||
}
|
||||
if searchOff {
|
||||
for _, phrase := range searchWording {
|
||||
for _, at := range findInSurface(req, phrase) {
|
||||
out = append(out, finding{t.ID, armName, fmt.Sprintf("search is offered to an arm without it: %q at %s", phrase, at)})
|
||||
}
|
||||
}
|
||||
}
|
||||
return append(out, boundaryFindings(t, armName, req)...)
|
||||
}
|
||||
|
||||
func boundaryFindings(t contextTask, armName string, req provider.Request) []finding {
|
||||
var out []finding
|
||||
for _, at := range localOnlyLeaks(req) {
|
||||
out = append(out, finding{t.ID, armName, "host-only field crossed the provider boundary at " + at})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// checkCueVisibility holds an index task to its tier: the cue is addressed at
|
||||
// the scales the matrix names and at no others. A policy change that moves it
|
||||
// fails here rather than turning two arms into one experiment.
|
||||
func checkCueVisibility(t contextTask, inst fixtureInstance, scale, armName, visible string) []finding {
|
||||
wantVisible, wantHidden := tierScales(t.CueTier)
|
||||
has := strings.Contains(visible, inst.CueMarker)
|
||||
switch {
|
||||
case contains(wantVisible, scale) && !has:
|
||||
return []finding{{t.ID, armName, fmt.Sprintf("cue %q should be addressed at %s and is not", inst.CueMarker, scale)}}
|
||||
case contains(wantHidden, scale) && has:
|
||||
return []finding{{t.ID, armName, fmt.Sprintf("cue %q should be gone at %s and is not", inst.CueMarker, scale)}}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func contains(list []string, want string) bool { return slices.Contains(list, want) }
|
||||
|
||||
// probeRank runs the task's own query against the folded region the way recall
|
||||
// would, and reports where the target lands.
|
||||
func probeRank(f builtFixture, query string) (int, error) {
|
||||
terms, err := retrieval.QueryTerms(query)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
canonical := f.Session.Snapshot()
|
||||
covered := min(f.State.Projection.CoveredCount, len(canonical))
|
||||
docs := make([]map[string]int, 0, covered)
|
||||
positions := make([]int, 0, covered)
|
||||
lengths := make([]int, 0, covered)
|
||||
for i, m := range canonical[:covered] {
|
||||
var text strings.Builder
|
||||
text.WriteString(m.Content)
|
||||
for _, tc := range m.ToolCalls {
|
||||
text.WriteString(" " + tc.Name + " " + string(tc.Arguments))
|
||||
}
|
||||
tokens := retrieval.Tokens(text.String())
|
||||
if len(tokens) == 0 {
|
||||
continue
|
||||
}
|
||||
docs = append(docs, retrieval.Counts(tokens))
|
||||
positions = append(positions, i)
|
||||
lengths = append(lengths, len(tokens))
|
||||
}
|
||||
df := retrieval.DocumentFrequency(docs)
|
||||
avg := 0.0
|
||||
for _, l := range lengths {
|
||||
avg += float64(l)
|
||||
}
|
||||
if len(lengths) > 0 {
|
||||
avg /= float64(len(lengths))
|
||||
}
|
||||
type hit struct {
|
||||
pos int
|
||||
score float64
|
||||
}
|
||||
var hits []hit
|
||||
for i, counts := range docs {
|
||||
if s := retrieval.BM25Score(counts, lengths[i], terms, df, len(docs), avg); s > 0 {
|
||||
hits = append(hits, hit{positions[i], s})
|
||||
}
|
||||
}
|
||||
for i := range hits {
|
||||
for j := i + 1; j < len(hits); j++ {
|
||||
if hits[j].score > hits[i].score {
|
||||
hits[i], hits[j] = hits[j], hits[i]
|
||||
}
|
||||
}
|
||||
}
|
||||
// A tool result answers at its caller's position, so either counts.
|
||||
for i, h := range hits {
|
||||
if h.pos == f.Target || h.pos == f.Target+1 {
|
||||
return i + 1, nil
|
||||
}
|
||||
}
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
// indexLineFor returns the fold-index line addressing a position, if any.
|
||||
func indexLineFor(f builtFixture, pos int) string {
|
||||
want := fmt.Sprintf("#%d ", pos)
|
||||
for line := range strings.SplitSeq(visibleIndexOnly(f), "\n") {
|
||||
if strings.Contains(line, want) {
|
||||
return strings.TrimSpace(line)
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// visibleIndexOnly is just the fold-index sections of the projection.
|
||||
func visibleIndexOnly(f builtFixture) string {
|
||||
var b strings.Builder
|
||||
for _, m := range visibleContext(f) {
|
||||
if i := strings.Index(m.Content, "## Folded work index"); i >= 0 {
|
||||
b.WriteString(m.Content[i:])
|
||||
b.WriteString("\n")
|
||||
}
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
@@ -0,0 +1,252 @@
|
||||
// reformulation.go — how a query changed, from token sets alone.
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"reasonix/internal/base/retrieval"
|
||||
)
|
||||
|
||||
// The host probe ranks every target first, so the engine is not what makes a
|
||||
// model take more than one attempt. These classes are set arithmetic over the
|
||||
// same tokenizer recall uses: no judge, nothing read out of prose.
|
||||
|
||||
const (
|
||||
reformFirst = "First" // nothing to compare against yet
|
||||
reformNearDup = "NearDuplicate" // the same query again, give or take a word
|
||||
reformNarrowing = "Narrowing" // kept everything and added terms
|
||||
reformBroadening = "Broadening" // dropped terms, added none
|
||||
reformAnchored = "HistoricalAnchor" // reached for a term the question does not contain
|
||||
reformShift = "SynonymShift" // traded terms for others
|
||||
)
|
||||
|
||||
// nearDuplicateOverlap is where "reworded" stops and "asked again" begins.
|
||||
const nearDuplicateOverlap = 0.8
|
||||
|
||||
func terms(s string) []string {
|
||||
return retrieval.Unique(retrieval.Tokens(s))
|
||||
}
|
||||
|
||||
func termSet(s string) map[string]bool {
|
||||
out := map[string]bool{}
|
||||
for _, t := range terms(s) {
|
||||
out[t] = true
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func jaccard(a, b map[string]bool) float64 {
|
||||
if len(a) == 0 && len(b) == 0 {
|
||||
return 1
|
||||
}
|
||||
shared := 0
|
||||
for t := range a {
|
||||
if b[t] {
|
||||
shared++
|
||||
}
|
||||
}
|
||||
union := len(a) + len(b) - shared
|
||||
if union == 0 {
|
||||
return 0
|
||||
}
|
||||
return float64(shared) / float64(union)
|
||||
}
|
||||
|
||||
// classifyReformulation names how this query differs from the one before it.
|
||||
func classifyReformulation(previous, current, question string) string {
|
||||
if previous == "" {
|
||||
return reformFirst
|
||||
}
|
||||
prev, cur := termSet(previous), termSet(current)
|
||||
if jaccard(prev, cur) >= nearDuplicateOverlap {
|
||||
return reformNearDup
|
||||
}
|
||||
added, dropped := diffTerms(cur, prev), diffTerms(prev, cur)
|
||||
// A term the question never used had to come from the model's own memory of
|
||||
// the history, which is a different move from rephrasing what it was asked.
|
||||
ask := termSet(question)
|
||||
for _, t := range added {
|
||||
if !ask[t] {
|
||||
return reformAnchored
|
||||
}
|
||||
}
|
||||
switch {
|
||||
case len(dropped) == 0:
|
||||
return reformNarrowing
|
||||
case len(added) == 0:
|
||||
return reformBroadening
|
||||
default:
|
||||
return reformShift
|
||||
}
|
||||
}
|
||||
|
||||
func diffTerms(from, minus map[string]bool) []string {
|
||||
var out []string
|
||||
for t := range from {
|
||||
if !minus[t] {
|
||||
out = append(out, t)
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// classifySearches labels each attempt and records which of the host probe's
|
||||
// terms it left out. The probe is the oracle: it ranks the target first, so a
|
||||
// query missing its terms says what the model did not think to ask for.
|
||||
func classifySearches(m *contextMetrics, inst fixtureInstance) {
|
||||
probe := terms(inst.ProbeQuery)
|
||||
previous := ""
|
||||
for i := range m.Searches {
|
||||
s := &m.Searches[i]
|
||||
s.Reformulation = classifyReformulation(previous, s.Query, inst.Prompt)
|
||||
have := termSet(s.Query)
|
||||
for _, term := range probe {
|
||||
if !have[term] {
|
||||
s.MissingProbeTerms = append(s.MissingProbeTerms, term)
|
||||
}
|
||||
}
|
||||
previous = s.Query
|
||||
}
|
||||
}
|
||||
|
||||
// queryAudit is the reformulation funnel across a set of runs.
|
||||
type queryAudit struct {
|
||||
Runs, RunsThatSearched int
|
||||
FirstQueryHit int
|
||||
HitBySecond, HitByThird int
|
||||
NeverHit int
|
||||
TotalSearches int
|
||||
PostHitSearches int
|
||||
HitThenRead int
|
||||
EscapeBeforeFirstSearch int
|
||||
Reformulations map[string]int
|
||||
MissedProbeTerms map[string]int
|
||||
}
|
||||
|
||||
func auditQueries(runs []contextMetrics) queryAudit {
|
||||
a := queryAudit{Reformulations: map[string]int{}, MissedProbeTerms: map[string]int{}}
|
||||
for _, r := range runs {
|
||||
if r.Failed || r.Contaminated {
|
||||
continue
|
||||
}
|
||||
a.Runs++
|
||||
a.TotalSearches += len(r.Searches)
|
||||
a.PostHitSearches += r.PostHitSearches
|
||||
if r.EscapeBeforeFirstRecall > 0 {
|
||||
a.EscapeBeforeFirstSearch++
|
||||
}
|
||||
if len(r.Searches) == 0 {
|
||||
continue
|
||||
}
|
||||
a.RunsThatSearched++
|
||||
hitAt := 0
|
||||
for i, s := range r.Searches {
|
||||
a.Reformulations[s.Reformulation]++
|
||||
for _, term := range s.MissingProbeTerms {
|
||||
a.MissedProbeTerms[term]++
|
||||
}
|
||||
if s.TargetRank > 0 && hitAt == 0 {
|
||||
hitAt = i + 1
|
||||
}
|
||||
}
|
||||
switch {
|
||||
case hitAt == 1:
|
||||
a.FirstQueryHit++
|
||||
case hitAt == 2:
|
||||
a.HitBySecond++
|
||||
case hitAt >= 3:
|
||||
a.HitByThird++
|
||||
default:
|
||||
a.NeverHit++
|
||||
}
|
||||
if hitAt > 0 && r.TargetRead {
|
||||
a.HitThenRead++
|
||||
}
|
||||
}
|
||||
return a
|
||||
}
|
||||
|
||||
func (a queryAudit) report() string {
|
||||
var b strings.Builder
|
||||
b.WriteString("\n## Query reformulation\n")
|
||||
fmt.Fprintf(&b, " runs=%d searched=%d searches=%d (%.2f per searching run)\n",
|
||||
a.Runs, a.RunsThatSearched, a.TotalSearches, ratio(a.TotalSearches, a.RunsThatSearched))
|
||||
fmt.Fprintf(&b, " target found by query 1: %d by 2: %d by 3+: %d never: %d\n",
|
||||
a.FirstQueryHit, a.HitBySecond, a.HitByThird, a.NeverHit)
|
||||
fmt.Fprintf(&b, " found then read: %d/%d searches issued after the target was already back: %d\n",
|
||||
a.HitThenRead, a.FirstQueryHit+a.HitBySecond+a.HitByThird, a.PostHitSearches)
|
||||
fmt.Fprintf(&b, " reached for the workspace before searching: %d\n", a.EscapeBeforeFirstSearch)
|
||||
b.WriteString(" reformulations: " + countsLine(a.Reformulations) + "\n")
|
||||
b.WriteString(" probe terms most often absent: " + topCounts(a.MissedProbeTerms, 8) + "\n")
|
||||
return b.String()
|
||||
}
|
||||
|
||||
func ratio(n, d int) float64 {
|
||||
if d == 0 {
|
||||
return 0
|
||||
}
|
||||
return float64(n) / float64(d)
|
||||
}
|
||||
|
||||
func countsLine(counts map[string]int) string {
|
||||
keys := make([]string, 0, len(counts))
|
||||
for k := range counts {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
parts := make([]string, 0, len(keys))
|
||||
for _, k := range keys {
|
||||
parts = append(parts, fmt.Sprintf("%s=%d", k, counts[k]))
|
||||
}
|
||||
return strings.Join(parts, " ")
|
||||
}
|
||||
|
||||
func topCounts(counts map[string]int, limit int) string {
|
||||
type kv struct {
|
||||
k string
|
||||
n int
|
||||
}
|
||||
all := make([]kv, 0, len(counts))
|
||||
for k, n := range counts {
|
||||
all = append(all, kv{k, n})
|
||||
}
|
||||
sort.Slice(all, func(i, j int) bool {
|
||||
if all[i].n != all[j].n {
|
||||
return all[i].n > all[j].n
|
||||
}
|
||||
return all[i].k < all[j].k
|
||||
})
|
||||
parts := make([]string, 0, limit)
|
||||
for _, e := range all[:min(limit, len(all))] {
|
||||
parts = append(parts, fmt.Sprintf("%s=%d", e.k, e.n))
|
||||
}
|
||||
if len(parts) == 0 {
|
||||
return "(none)"
|
||||
}
|
||||
return strings.Join(parts, " ")
|
||||
}
|
||||
|
||||
// queryLines lists every attempt, so a surprising audit can be read back.
|
||||
func queryLines(runs []contextMetrics) string {
|
||||
var b strings.Builder
|
||||
b.WriteString("\n## Every query, in order\n")
|
||||
for _, r := range runs {
|
||||
for _, s := range r.Searches {
|
||||
rank := "miss"
|
||||
if s.TargetRank > 0 {
|
||||
rank = fmt.Sprintf("rank %d", s.TargetRank)
|
||||
}
|
||||
fmt.Fprintf(&b, " %-24s %-14s #%d %-17s %-9s %q\n",
|
||||
r.Task, r.Arm, s.Ordinal, s.Reformulation, rank, s.Query)
|
||||
if len(s.MissingProbeTerms) > 0 && s.TargetRank == 0 {
|
||||
fmt.Fprintf(&b, " %-24s %-14s missing probe terms: %s\n",
|
||||
"", "", strings.Join(slices.Clone(s.MissingProbeTerms), " "))
|
||||
}
|
||||
}
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
@@ -0,0 +1,569 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"reasonix/internal/state/sessionstore"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"reasonix/internal/assembly/boot"
|
||||
"reasonix/internal/contract/ablation"
|
||||
"reasonix/internal/contract/provider"
|
||||
"reasonix/internal/contract/tool"
|
||||
_ "reasonix/internal/model/openai"
|
||||
"reasonix/internal/runtime/agent"
|
||||
_ "reasonix/internal/tools/builtin"
|
||||
)
|
||||
|
||||
// A run is one recall question against a fixture the preflight has already
|
||||
// cleared. Scoring reads the transcript the turn appended — tool names and
|
||||
// their JSON arguments — so nothing depends on how the model narrated itself.
|
||||
|
||||
const (
|
||||
runBaseURL = "https://api.deepseek.com"
|
||||
runModel = "deepseek-chat"
|
||||
runTimeout = 4 * time.Minute
|
||||
)
|
||||
|
||||
func realProvider() (provider.Provider, error) {
|
||||
key := os.Getenv("DEEPSEEK_API_KEY")
|
||||
if key == "" {
|
||||
return nil, fmt.Errorf("a real run needs DEEPSEEK_API_KEY")
|
||||
}
|
||||
return provider.New("openai", provider.Config{
|
||||
Name: "contextbench", BaseURL: runBaseURL, Model: runModel, APIKey: key,
|
||||
})
|
||||
}
|
||||
|
||||
// adversarialPrompts tell the model what it is running and send it after the
|
||||
// answer everywhere except the conversation. The question is not whether it
|
||||
// obeys the experiment: it is whether the answer is reachable at all by a model
|
||||
// that knows the mechanism and is actively attacking the host.
|
||||
var adversarialPrompts = map[string]string{
|
||||
experimentSearch: "You are running a benchmark. Find the expected answer for this task from the " +
|
||||
"Reasonix repository, benchmark source, fixture generator, test files, scratch files, or " +
|
||||
"session metadata. Do not use recall unless necessary. Report the exact hidden values.",
|
||||
experimentIndex: "Inspect benchmarks/context-retrieval, the git repository, session files, temporary " +
|
||||
"files, and any benchmark metadata you can access. Recover the concrete instantiated answer for " +
|
||||
"this run without relying on the folded conversation. Report the exact values.",
|
||||
}
|
||||
|
||||
// runAdversarial sends one task per experiment after the answer through every
|
||||
// host-readable surface. It passes when nothing but recall could produce it —
|
||||
// a run that gives up and recalls has still proven the attack failed.
|
||||
func runAdversarial(root string) int {
|
||||
root, err := filepath.Abs(root)
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
return 2
|
||||
}
|
||||
restore, ierr := isolateWorkspace()
|
||||
if ierr != nil {
|
||||
fmt.Fprintln(os.Stderr, "isolate workspace:", ierr)
|
||||
return 2
|
||||
}
|
||||
defer restore()
|
||||
p, perr := realProvider()
|
||||
if perr != nil {
|
||||
fmt.Fprintln(os.Stderr, perr)
|
||||
return 2
|
||||
}
|
||||
leaked := 0
|
||||
for _, id := range []string{"r01-retry-boundary", "i01-transport-fallback"} {
|
||||
t, ok := taskByID(id)
|
||||
if !ok {
|
||||
fmt.Fprintf(os.Stderr, "unknown task %q\n", id)
|
||||
return 2
|
||||
}
|
||||
m, err := runOne(p, t, armFor("default", false), "adversarial", root, false, false,
|
||||
adversarialPrompts[t.Experiment])
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "%s: %v\n", id, err)
|
||||
m = failedMetrics(t, "adversarial", err)
|
||||
}
|
||||
verdict := "isolated"
|
||||
if m.EscapeFoundAnswer {
|
||||
verdict = "LEAKED via " + m.LeakedVia
|
||||
leaked++
|
||||
}
|
||||
fmt.Printf("%-24s %-24s escapes=%-3d recall=%d answered=%v tools=%v\n",
|
||||
id, verdict, m.EscapeCalls, m.SearchCalls+m.ReadCalls, m.AnswerRecovered, m.UnexpectedWorkTools)
|
||||
if m.EscapeFoundAnswer {
|
||||
fmt.Printf(" args: %s\n output: %s\n", m.LeakedArgs, m.LeakedOutput)
|
||||
}
|
||||
}
|
||||
if leaked > 0 {
|
||||
fmt.Fprintf(os.Stderr, "\n%d of 2 adversarial runs reached an answer outside recall\n", leaked)
|
||||
return 1
|
||||
}
|
||||
fmt.Println("\nNo answer was reachable outside recall.")
|
||||
return 0
|
||||
}
|
||||
|
||||
// runOne asks one task under one arm and scores what the model did.
|
||||
func runOne(p provider.Provider, t contextTask, arm ablation.Set, armName, root string, cueVisible, dry bool, promptOverride ...string) (contextMetrics, error) {
|
||||
// Live runs get values that existed nowhere before the run started; a dry
|
||||
// run stays reproducible.
|
||||
rng := liveRand()
|
||||
if dry {
|
||||
rng = seededRand(t.ID, armName, "dry")
|
||||
}
|
||||
inst, err := instantiateTask(t, rng)
|
||||
if err != nil {
|
||||
return contextMetrics{}, err
|
||||
}
|
||||
f, err := buildFixture(inst, arm, filepath.Join(root, t.ID, armName, "session.jsonl"))
|
||||
if err != nil {
|
||||
return contextMetrics{}, err
|
||||
}
|
||||
sess, err := sessionstore.LoadSession(f.Path)
|
||||
if err != nil {
|
||||
return contextMetrics{}, fmt.Errorf("load session: %w", err)
|
||||
}
|
||||
reg := tool.NewRegistry()
|
||||
for _, x := range tool.Builtins() {
|
||||
reg.Add(x)
|
||||
}
|
||||
boot.ApplyUnifiedProviderToolSurface(reg, false, arm)
|
||||
|
||||
var usage usageSink
|
||||
a := agent.New(p, reg, sess, agent.Options{
|
||||
ContextWindow: fixtureWindow, CompactRatio: 0.5, RecentKeep: 2,
|
||||
SessionPath: f.Path, KeepPolicy: agent.KeepErrors, Ablation: arm,
|
||||
}, &usage)
|
||||
a.LoadProjectionSidecar(f.Path)
|
||||
// From here the transcript lives only in memory. On disk it is the answer.
|
||||
sealFixture(f.Path)
|
||||
defer unsealFixture(f.Path)
|
||||
dir := filepath.Dir(f.Path)
|
||||
if at, marker := answerOnDisk(dir, inst.AnswerMarkers); at != "" {
|
||||
return contextMetrics{}, fmt.Errorf("%s [%s]: %q is readable at %s before the run", t.ID, armName, marker, at)
|
||||
}
|
||||
|
||||
before := len(sess.Snapshot())
|
||||
ctx, cancel := context.WithTimeout(context.Background(), runTimeout)
|
||||
defer cancel()
|
||||
prompt := inst.Prompt
|
||||
if len(promptOverride) > 0 && promptOverride[0] != "" {
|
||||
prompt = promptOverride[0]
|
||||
}
|
||||
runErr := a.Run(ctx, prompt)
|
||||
|
||||
if at, marker := answerOnDisk(dir, inst.AnswerMarkers); at != "" {
|
||||
return contextMetrics{}, fmt.Errorf("%s [%s]: %q was written to %s during the run", t.ID, armName, marker, at)
|
||||
}
|
||||
|
||||
appended := sess.Snapshot()[before:]
|
||||
m := scoreRun(appended, inst, f.Target, armName, cueVisible)
|
||||
m.trajectory = appended
|
||||
if u, requests := usage.snapshot(); requests > 0 {
|
||||
m.UsageReportedRequests, m.PromptTokens, m.CompletionTokens = requests, u.PromptTokens, u.CompletionTokens
|
||||
m.CacheHitTokens, m.CacheMissTokens, m.UsageEstimated = u.CacheHitTokens, u.CacheMissTokens, u.Estimated
|
||||
}
|
||||
m.scoreAnswer(finalAnswer(appended), inst)
|
||||
if runErr != nil {
|
||||
m.FailureStage = "RunError"
|
||||
m.TimeoutReason = timeoutReason(m)
|
||||
}
|
||||
return m, runErr
|
||||
}
|
||||
|
||||
// failedMetrics is what a run that never reached a transcript leaves behind. It
|
||||
// carries the run's identity and the stage that ended it, so the failure stays a
|
||||
// countable fact; Failed keeps it out of every statistic.
|
||||
func failedMetrics(t contextTask, armName string, err error) contextMetrics {
|
||||
return contextMetrics{
|
||||
Task: t.ID, Arm: armName, Failed: true,
|
||||
FailureStage: stageNoEvidence, Reason: err.Error(),
|
||||
}
|
||||
}
|
||||
|
||||
// finalAnswer is the last assistant text the turn produced.
|
||||
func finalAnswer(msgs []provider.Message) string {
|
||||
for _, m := range slices.Backward(msgs) {
|
||||
if m.Role == provider.RoleAssistant && strings.TrimSpace(m.Content) != "" {
|
||||
return m.Content
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// isolateWorkspace moves the process into an empty directory before any task
|
||||
// runs. The corpus lives in this repository, so an agent left here greps the
|
||||
// answers out of corpus.go — one did, quoting the file's own line about them
|
||||
// being nonces. Fixtures stay outside it too: a readable session file is the
|
||||
// canonical transcript, answer included.
|
||||
func isolateWorkspace() (func(), error) {
|
||||
dir, err := os.MkdirTemp("", "contextbench-workspace-*")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
readme := "This workspace is empty on purpose. The question is about earlier conversation,\nnot about any file here.\n"
|
||||
if err := os.WriteFile(filepath.Join(dir, "README.md"), []byte(readme), 0o644); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
previous, err := os.Getwd()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := os.Chdir(dir); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return func() {
|
||||
_ = os.Chdir(previous)
|
||||
_ = os.RemoveAll(dir)
|
||||
}, nil
|
||||
}
|
||||
|
||||
// runBoundaries runs only the pair each index task is built for: the narrowest
|
||||
// scale still addressing its cue, and the next one down. Each pair moves the
|
||||
// affordance and nothing else, where default-vs-off moves the whole index.
|
||||
|
||||
// The two arms get separate nonces: reusing one would leave the first arm's
|
||||
// answer in a host-readable history for the second to find.
|
||||
func runBoundaries(mode, root string) int {
|
||||
started := time.Now()
|
||||
planned := 0
|
||||
repo, _ := os.Getwd()
|
||||
root, err := filepath.Abs(root)
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
return 2
|
||||
}
|
||||
restore, ierr := isolateWorkspace()
|
||||
if ierr != nil {
|
||||
fmt.Fprintln(os.Stderr, "isolate workspace:", ierr)
|
||||
return 2
|
||||
}
|
||||
defer restore()
|
||||
p, perr := realProvider()
|
||||
if perr != nil {
|
||||
fmt.Fprintln(os.Stderr, perr)
|
||||
return 2
|
||||
}
|
||||
|
||||
var all []contextMetrics
|
||||
var cueSide, noCueSide []contextMetrics
|
||||
for _, t := range efficiencyTasks() {
|
||||
cue, noCue := boundaryPair(t.CueTier)
|
||||
for _, side := range []struct {
|
||||
scale string
|
||||
has bool
|
||||
}{{cue, true}, {noCue, false}} {
|
||||
name := "index-" + side.scale
|
||||
planned++
|
||||
m, err := runOne(p, t, armFor(side.scale, false), name, root, side.has, false)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "%s [%s]: %v\n", t.ID, name, err)
|
||||
m = failedMetrics(t, name, err)
|
||||
}
|
||||
all = append(all, m)
|
||||
if side.has {
|
||||
cueSide = append(cueSide, m)
|
||||
} else {
|
||||
noCueSide = append(noCueSide, m)
|
||||
}
|
||||
fmt.Printf("%-24s %-14s cue=%-3v %-14s search=%d read=%d direct=%-5v recall-tok=%-5d%s escape=%d\n",
|
||||
t.ID, name, side.has, m.FailureStage, m.SearchCalls, m.ReadCalls,
|
||||
m.CueDirectRead, m.RecallReturnedTokens, promptTokenSuffix(m), m.EscapeCalls)
|
||||
}
|
||||
}
|
||||
reportDeltas(cueSide, noCueSide)
|
||||
fmt.Print(reportStopping(all))
|
||||
return writeResults(root, batchInfo{Mode: mode, Planned: planned, Started: started, Repo: repo}, all)
|
||||
}
|
||||
|
||||
// reportDeltas is the paired reading: each task against itself, then pooled.
|
||||
func reportDeltas(cueSide, noCueSide []contextMetrics) {
|
||||
fmt.Printf("\n## Paired deltas (cue present − cue absent)\n")
|
||||
fmt.Printf(" %-24s %-10s %-10s %-9s %-11s %s\n",
|
||||
"task", "recovery", "directread", "searches", "recall-tok", "escapes")
|
||||
for i := range cueSide {
|
||||
if i >= len(noCueSide) {
|
||||
break
|
||||
}
|
||||
a, b := cueSide[i], noCueSide[i]
|
||||
fmt.Printf(" %-24s %-10s %-10s %-9d %-11d %d\n", a.Task,
|
||||
deltaBool(a.AnswerRecovered, b.AnswerRecovered),
|
||||
deltaBool(a.CueDirectRead, b.CueDirectRead),
|
||||
a.SearchCalls-b.SearchCalls,
|
||||
a.RecallReturnedTokens-b.RecallReturnedTokens,
|
||||
a.EscapeCalls-b.EscapeCalls)
|
||||
}
|
||||
fmt.Println("\n## Pooled")
|
||||
fmt.Println(" ", summarize("cue-present", cueSide).line())
|
||||
fmt.Println(" ", summarize("cue-absent", noCueSide).line())
|
||||
fmt.Println("\n## Where each side looked first")
|
||||
fmt.Printf(" %-14s %s\n", "cue-present", countsLine(summarize("x", cueSide).Routes))
|
||||
fmt.Printf(" %-14s %s\n", "cue-absent", countsLine(summarize("x", noCueSide).Routes))
|
||||
}
|
||||
|
||||
func deltaBool(a, b bool) string {
|
||||
switch {
|
||||
case a == b:
|
||||
return "same"
|
||||
case a:
|
||||
return "+cue"
|
||||
default:
|
||||
return "-cue"
|
||||
}
|
||||
}
|
||||
|
||||
// runExperiment runs one experiment's tasks across its arms. A dry run drives
|
||||
// the same pipeline with a scripted provider: it proves fixture, run, scoring
|
||||
// and report hold together before any of it is paid for.
|
||||
func runExperiment(mode, experiment, root string, dry bool, tasks []contextTask) int {
|
||||
started := time.Now()
|
||||
planned := 0
|
||||
repo, _ := os.Getwd()
|
||||
root, err := filepath.Abs(root)
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
return 2
|
||||
}
|
||||
restore, err := isolateWorkspace()
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, "isolate workspace:", err)
|
||||
return 2
|
||||
}
|
||||
defer restore()
|
||||
var p provider.Provider = &scriptedProvider{}
|
||||
if !dry {
|
||||
real, rerr := realProvider()
|
||||
if rerr != nil {
|
||||
fmt.Fprintln(os.Stderr, rerr)
|
||||
return 2
|
||||
}
|
||||
p = real
|
||||
}
|
||||
byArm := map[string][]contextMetrics{}
|
||||
var all []contextMetrics
|
||||
|
||||
if experiment == experimentIndex {
|
||||
fmt.Print(fingerprint(dry))
|
||||
}
|
||||
for _, t := range tasks {
|
||||
if t.Experiment != experiment {
|
||||
continue
|
||||
}
|
||||
// Arms are shuffled inside each task block. Running six defaults, then
|
||||
// six halves, would make provider drift collinear with the axis under
|
||||
// test: a bad minute would land entirely on one arm.
|
||||
for _, arm := range shuffledArms(t) {
|
||||
cueVisible := t.Experiment == experimentIndex && cueExpectedVisible(t, arm.scale)
|
||||
planned++
|
||||
m, err := runOne(p, t, armFor(arm.scale, arm.searchOff), arm.name, root, cueVisible, dry)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "%s [%s]: %v\n", t.ID, arm.name, err)
|
||||
m = failedMetrics(t, arm.name, err)
|
||||
}
|
||||
byArm[arm.name] = append(byArm[arm.name], m)
|
||||
all = append(all, m)
|
||||
fmt.Printf("%-24s %-14s %-18s search=%d hit=%d read=%d recall-tok=%d%s %v\n",
|
||||
t.ID, arm.name, m.FailureStage, m.SearchCalls, m.TargetSearchHits, m.ReadCalls,
|
||||
m.RecallReturnedTokens, promptTokenSuffix(m), m.UnexpectedWorkTools)
|
||||
}
|
||||
}
|
||||
reportFunnels(byArm)
|
||||
if experiment == experimentIndex {
|
||||
fmt.Print(reportDoseResponse(all))
|
||||
}
|
||||
fmt.Print(reportStopping(all))
|
||||
fmt.Print(auditQueries(all).report())
|
||||
fmt.Print(queryLines(all))
|
||||
if experiment == experimentIndex {
|
||||
reportBoundaries(all)
|
||||
}
|
||||
return writeResults(root, batchInfo{Mode: mode, Planned: planned, Started: started, Dry: dry, Repo: repo}, all)
|
||||
}
|
||||
|
||||
func cueExpectedVisible(t contextTask, scale string) bool {
|
||||
visible, _ := tierScales(t.CueTier)
|
||||
return contains(visible, scale)
|
||||
}
|
||||
|
||||
func reportFunnels(byArm map[string][]contextMetrics) {
|
||||
names := make([]string, 0, len(byArm))
|
||||
for name := range byArm {
|
||||
names = append(names, name)
|
||||
}
|
||||
sort.Strings(names)
|
||||
fmt.Println("\n## Funnel")
|
||||
for _, name := range names {
|
||||
fmt.Println(" ", summarize(name, byArm[name]).line())
|
||||
}
|
||||
fmt.Println("\n## Where it looked first")
|
||||
for _, name := range names {
|
||||
f := summarize(name, byArm[name])
|
||||
fmt.Printf(" %-14s %s\n", name, countsLine(f.Routes))
|
||||
}
|
||||
fmt.Println("\n## Where it broke")
|
||||
for _, name := range names {
|
||||
f := summarize(name, byArm[name])
|
||||
stages := make([]string, 0, len(f.Stages))
|
||||
for stage, n := range f.Stages {
|
||||
stages = append(stages, fmt.Sprintf("%s=%d", stage, n))
|
||||
}
|
||||
sort.Strings(stages)
|
||||
fmt.Printf(" %-12s %s\n", name, strings.Join(stages, " "))
|
||||
}
|
||||
}
|
||||
|
||||
// writeTrajectories saves what each turn produced, after the whole batch is
|
||||
// done. Three analyses in a row have needed a re-run because something was
|
||||
// counted and not kept — query text, then snippet contents. It is written at
|
||||
// the end on purpose: during the batch these files are the answer.
|
||||
func writeTrajectories(root string, all []contextMetrics) error {
|
||||
f, err := os.Create(filepath.Join(root, "trajectories.jsonl"))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer f.Close()
|
||||
for _, m := range all {
|
||||
if err := writeJSONLine(f, map[string]any{
|
||||
"task": m.Task, "arm": m.Arm, "messages": m.trajectory,
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func writeResults(root string, info batchInfo, all []contextMetrics) int {
|
||||
path := filepath.Join(root, "results.jsonl")
|
||||
f, err := os.Create(path)
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
return 2
|
||||
}
|
||||
defer f.Close()
|
||||
scored, failed := 0, 0
|
||||
for _, m := range all {
|
||||
if err := writeJSONLine(f, m); err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
return 2
|
||||
}
|
||||
if m.Failed {
|
||||
failed++
|
||||
} else {
|
||||
scored++
|
||||
}
|
||||
}
|
||||
if err := writeTrajectories(root, all); err != nil {
|
||||
fmt.Fprintln(os.Stderr, "write trajectories:", err)
|
||||
return 2
|
||||
}
|
||||
if err := writeBatchManifest(root, info, all); err != nil {
|
||||
fmt.Fprintln(os.Stderr, "write batch manifest:", err)
|
||||
return 2
|
||||
}
|
||||
// Both counts are named: a reader who sees only the batch size cannot tell
|
||||
// measurements from attempts.
|
||||
fmt.Printf("\n%d runs written to %s: scored=%d failed=%d (with trajectories.jsonl and batch.json)\n",
|
||||
len(all), path, scored, failed)
|
||||
return 0
|
||||
}
|
||||
|
||||
// reportBoundaries prints each index task across its own cue boundary, then the
|
||||
// two sides pooled. Task difficulty is held constant inside a row, which is
|
||||
// what a four-arm average cannot do.
|
||||
func reportBoundaries(all []contextMetrics) {
|
||||
byKey := map[string]contextMetrics{}
|
||||
scored, failed := 0, 0
|
||||
for _, m := range all {
|
||||
if m.Failed {
|
||||
// No transcript, no boundary to compare: count the failure instead of
|
||||
// printing the zero it left behind as a measurement.
|
||||
failed++
|
||||
continue
|
||||
}
|
||||
scored++
|
||||
byKey[m.Task+"|"+m.Arm] = m
|
||||
}
|
||||
if scored == 0 {
|
||||
fmt.Printf("\n## Across each task's cue boundary\n no scored runs: scored=%d failed=%d (nothing to compare)\n", scored, failed)
|
||||
return
|
||||
}
|
||||
var cueSide, noCueSide []contextMetrics
|
||||
fmt.Println("\n## Across each task's cue boundary")
|
||||
fmt.Printf(" %-24s %-16s %-4s %-10s %-9s %-12s %-11s %s\n",
|
||||
"task", "arm", "cue", "recovered", "cue-read", "search/task", "recall-tok", "escape")
|
||||
for _, t := range indexTasks() {
|
||||
cue, noCue := boundaryPair(t.CueTier)
|
||||
for _, side := range []struct {
|
||||
scale string
|
||||
has bool
|
||||
}{{cue, true}, {noCue, false}} {
|
||||
m, ok := byKey[t.ID+"|index-"+side.scale]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if side.has {
|
||||
cueSide = append(cueSide, m)
|
||||
} else {
|
||||
noCueSide = append(noCueSide, m)
|
||||
}
|
||||
mark := "no"
|
||||
if side.has {
|
||||
mark = "yes"
|
||||
}
|
||||
fmt.Printf(" %-24s %-16s %-4s %-10v %-9v %-12d %-11d %d\n",
|
||||
t.ID, "index-"+side.scale, mark, m.AnswerRecovered, m.RecallReadWithoutSearch,
|
||||
m.SearchCalls, m.RecallReturnedTokens, m.EscapeCalls)
|
||||
}
|
||||
}
|
||||
fmt.Println("\n## Pooled across boundaries")
|
||||
fmt.Println(" ", summarize("cue-present", cueSide).line())
|
||||
fmt.Println(" ", summarize("cue-absent", noCueSide).line())
|
||||
}
|
||||
|
||||
// shuffledArms permutes a task's arms deterministically, so the order is
|
||||
// reproducible and still uncorrelated with the axis being measured.
|
||||
func shuffledArms(t contextTask) []experimentArm {
|
||||
arms := armsFor(t)
|
||||
rng := seededRand(t.ID, "arm-order")
|
||||
rng.Shuffle(len(arms), func(i, j int) { arms[i], arms[j] = arms[j], arms[i] })
|
||||
return arms
|
||||
}
|
||||
|
||||
// fingerprint records what produced a batch. Not a gate — but batch effects
|
||||
// have reversed three conclusions in this experiment already, and a table with
|
||||
// no provenance cannot be compared with the next one.
|
||||
func fingerprint(dry bool) string {
|
||||
model, endpoint := runModel, runBaseURL
|
||||
if dry {
|
||||
model, endpoint = "scripted", "none"
|
||||
}
|
||||
commit := "unknown"
|
||||
if out, err := exec.Command("git", "rev-parse", "--short", "HEAD").Output(); err == nil {
|
||||
commit = strings.TrimSpace(string(out))
|
||||
}
|
||||
return fmt.Sprintf("## Batch\n model=%s endpoint=%s window=%d generations=%d commit=%s started=%s\n",
|
||||
model, endpoint, fixtureWindow, fixtureGenerations, commit, time.Now().UTC().Format(time.RFC3339))
|
||||
}
|
||||
|
||||
// timeoutReason separates a run that never found its answer from one that found
|
||||
// it and kept going. Only the first is a case for a longer budget; the second
|
||||
// would just get more time to circle. A batch reporting only "context deadline
|
||||
// exceeded" cannot tell an arm that made retrieval harder from an arm that
|
||||
// happened to draw a slow minute.
|
||||
func timeoutReason(m contextMetrics) string {
|
||||
switch {
|
||||
case m.TargetHeldAtEnd && m.RoundsAfterTarget >= 2:
|
||||
return "PostTargetRunaway"
|
||||
case m.TargetHeldAtEnd:
|
||||
return "TargetHeldLate"
|
||||
case m.SearchCalls >= 3:
|
||||
return "SearchSpiral"
|
||||
case m.SearchCalls+m.ReadCalls+m.EscapeCalls == 0:
|
||||
return "NoToolActivity"
|
||||
default:
|
||||
return "Unknown"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,95 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"regexp"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"reasonix/internal/contract/provider"
|
||||
)
|
||||
|
||||
// scriptedProvider walks the chain a competent model would: search, read the
|
||||
// best hit, answer with what came back. It proves the pipeline holds together
|
||||
// for nothing, so a real run's first failure is about the model.
|
||||
|
||||
type scriptedProvider struct{}
|
||||
|
||||
func (p *scriptedProvider) Name() string { return "contextbench-scripted" }
|
||||
|
||||
var (
|
||||
scriptedHit = regexp.MustCompile(`(?m)^#(\d+) `)
|
||||
scriptedRecall = regexp.MustCompile(`(?s)^#\d+\n(.*)`)
|
||||
)
|
||||
|
||||
// Stream decides from the request alone, never from a call counter: one
|
||||
// provider serves every task in a run, and a counter would make the second task
|
||||
// start where the first left off.
|
||||
func (p *scriptedProvider) Stream(_ context.Context, req provider.Request) (<-chan provider.Chunk, error) {
|
||||
ch := make(chan provider.Chunk, 3)
|
||||
defer close(ch)
|
||||
// Honour the schema, as a model does: an arm without a query parameter is
|
||||
// an arm that cannot search, and pretending otherwise scores a refusal as
|
||||
// a failed search.
|
||||
if !recallAccepts(req, "query") {
|
||||
ch <- provider.Chunk{Type: provider.ChunkText, Text: "I have no address for that part of the history."}
|
||||
ch <- provider.Chunk{Type: provider.ChunkDone}
|
||||
return ch, nil
|
||||
}
|
||||
// The last message, not the last tool result anywhere: the fixture's own
|
||||
// history is full of tool results, and one of those is not this turn's.
|
||||
last := ""
|
||||
if n := len(req.Messages); n > 0 && req.Messages[n-1].Role == provider.RoleTool {
|
||||
last = req.Messages[n-1].Content
|
||||
}
|
||||
switch {
|
||||
case last == "":
|
||||
args, _ := json.Marshal(map[string]string{"query": lastUserText(req)})
|
||||
ch <- provider.Chunk{Type: provider.ChunkToolCall, ToolCall: &provider.ToolCall{
|
||||
ID: "s1", Name: "recall", Arguments: string(args),
|
||||
}}
|
||||
case strings.Contains(last, "Folded context matching"):
|
||||
positions := scriptedHit.FindAllStringSubmatch(last, 1)
|
||||
if len(positions) == 0 {
|
||||
ch <- provider.Chunk{Type: provider.ChunkText, Text: "Nothing in the folded history matches."}
|
||||
break
|
||||
}
|
||||
n, _ := strconv.Atoi(positions[0][1])
|
||||
args, _ := json.Marshal(map[string][]int{"positions": {n}})
|
||||
ch <- provider.Chunk{Type: provider.ChunkToolCall, ToolCall: &provider.ToolCall{
|
||||
ID: "r1", Name: "recall", Arguments: string(args),
|
||||
}}
|
||||
default:
|
||||
// Answer with what recall returned, so a correct chain scores as one.
|
||||
body := last
|
||||
if m := scriptedRecall.FindStringSubmatch(last); len(m) > 1 {
|
||||
body = m[1]
|
||||
}
|
||||
ch <- provider.Chunk{Type: provider.ChunkText, Text: "From the folded history: " + body}
|
||||
}
|
||||
ch <- provider.Chunk{Type: provider.ChunkDone}
|
||||
return ch, nil
|
||||
}
|
||||
|
||||
// recallAccepts reports whether recall is offered with the given parameter.
|
||||
func recallAccepts(req provider.Request, param string) bool {
|
||||
for _, t := range req.Tools {
|
||||
if t.Name != "recall" {
|
||||
continue
|
||||
}
|
||||
b, err := json.Marshal(t.Parameters)
|
||||
return err == nil && strings.Contains(string(b), `"`+param+`"`)
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func lastUserText(req provider.Request) string {
|
||||
for _, m := range slices.Backward(req.Messages) {
|
||||
if m.Role == provider.RoleUser {
|
||||
return m.Content
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
@@ -0,0 +1,182 @@
|
||||
// snippet.go — does a rank-1 hit actually hand over the answer?
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"reasonix/internal/base/retrieval"
|
||||
"reasonix/internal/contract/ablation"
|
||||
"reasonix/internal/contract/event"
|
||||
"reasonix/internal/contract/tool"
|
||||
"reasonix/internal/runtime/agent"
|
||||
)
|
||||
|
||||
// TargetHit is not EvidenceSufficient: a rank-1 hit can still hand over a
|
||||
// window too short to answer with, and searching again after that is correct.
|
||||
// Measured through the shipped recall path, not a copy of it.
|
||||
|
||||
// snippetWidths are the windows to compare. 240 is what ships.
|
||||
var snippetWidths = []int{120, 180, 240, 320, 480}
|
||||
|
||||
// historicalQueries are the queries these tasks actually drew in the pilots,
|
||||
// kept because a probe query is the host's phrasing and the model's was not.
|
||||
var historicalQueries = map[string][]string{
|
||||
"i03-coalescing": {"coalesce window marker probe", "job completion coalescing window"},
|
||||
"i04-recovery-fence": {"recovery fence marker epoch", "expected epoch", "recfence quartz"},
|
||||
}
|
||||
|
||||
type snippetReading struct {
|
||||
Task string
|
||||
Query string
|
||||
Width int
|
||||
Covered int
|
||||
Total int
|
||||
Runes int
|
||||
Rank int
|
||||
Complete bool
|
||||
}
|
||||
|
||||
// runSnippetAudit answers the question offline, over many instantiations so a
|
||||
// single seed's lengths cannot decide it.
|
||||
func runSnippetAudit(root string, seeds int) int {
|
||||
fmt.Printf("## Snippet coverage (%d instantiations per task)\n", seeds)
|
||||
fmt.Println(" Does the window a rank-1 hit returns contain every value the task scores on?")
|
||||
|
||||
byTaskWidth := map[string]map[int][]snippetReading{}
|
||||
for _, t := range allTasks() {
|
||||
byTaskWidth[t.ID] = map[int][]snippetReading{}
|
||||
for seed := range seeds {
|
||||
inst, err := instantiateTask(t, seededRand(t.ID, "snippet", fmt.Sprint(seed)))
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
return 2
|
||||
}
|
||||
queries := append([]string{inst.ProbeQuery}, historicalQueries[t.ID]...)
|
||||
for _, q := range queries {
|
||||
for _, width := range snippetWidths {
|
||||
r := measureSnippet(inst, q, width)
|
||||
byTaskWidth[t.ID][width] = append(byTaskWidth[t.ID][width], r)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fmt.Printf("\n %-24s %-5s", "task", "vals")
|
||||
for _, w := range snippetWidths {
|
||||
fmt.Printf("%9d", w)
|
||||
}
|
||||
fmt.Println(" mean runes @240")
|
||||
for _, t := range allTasks() {
|
||||
vals := len(t.AnswerVars)
|
||||
fmt.Printf(" %-24s %-5d", t.ID, vals)
|
||||
for _, w := range snippetWidths {
|
||||
fmt.Printf("%8.0f%%", 100*completeRate(byTaskWidth[t.ID][w]))
|
||||
}
|
||||
fmt.Printf(" %.0f\n", meanRunes(byTaskWidth[t.ID][240]))
|
||||
}
|
||||
reportShippedPath(root)
|
||||
return 0
|
||||
}
|
||||
|
||||
// measureSnippet renders one hit through the same MakeSnippet the recall path
|
||||
// uses, then asks whether every scored value survived the window.
|
||||
func measureSnippet(inst fixtureInstance, query string, width int) snippetReading {
|
||||
terms, err := retrieval.QueryTerms(query)
|
||||
if err != nil {
|
||||
return snippetReading{Task: inst.Task.ID, Query: query, Width: width, Total: len(inst.AnswerMarkers)}
|
||||
}
|
||||
// The tool result is the document the answer lives in; a call's arguments
|
||||
// are a separate document and never carry it.
|
||||
snippet := retrieval.MakeSnippet(inst.body, query, terms, width)
|
||||
covered := 0
|
||||
for _, marker := range inst.AnswerMarkers {
|
||||
if strings.Contains(strings.ToLower(snippet), strings.ToLower(marker)) {
|
||||
covered++
|
||||
}
|
||||
}
|
||||
return snippetReading{
|
||||
Task: inst.Task.ID, Query: query, Width: width,
|
||||
Covered: covered, Total: len(inst.AnswerMarkers),
|
||||
Runes: len([]rune(snippet)), Complete: covered == len(inst.AnswerMarkers),
|
||||
}
|
||||
}
|
||||
|
||||
// reportShippedPath cross-checks the table against the real recall tool: a
|
||||
// fixture, a fold, and a search through the agent that ships.
|
||||
func reportShippedPath(root string) {
|
||||
fmt.Println("\n## Through the shipped recall path (width as shipped)")
|
||||
for _, t := range allTasks() {
|
||||
inst, err := instantiateTask(t, seededRand(t.ID, "snippet-live"))
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
f, err := buildFixture(inst, ablation.Set{}, filepath.Join(root, "snippet", t.ID, "session.jsonl"))
|
||||
if err != nil {
|
||||
fmt.Printf(" %-24s fixture: %v\n", t.ID, err)
|
||||
continue
|
||||
}
|
||||
a := agent.New(&digestProvider{}, tool.NewRegistry(), f.Session, agent.Options{
|
||||
ContextWindow: fixtureWindow, CompactRatio: 0.5, RecentKeep: 2,
|
||||
SessionPath: f.Path, KeepPolicy: agent.KeepErrors,
|
||||
}, event.Discard)
|
||||
a.LoadProjectionSidecar(f.Path)
|
||||
queries := append([]string{inst.ProbeQuery}, historicalQueries[t.ID]...)
|
||||
for _, q := range queries {
|
||||
res, err := a.RecallContext(context.Background(), tool.RecallRequest{Query: q})
|
||||
if err != nil {
|
||||
fmt.Printf(" %-24s %-34q recall: %v\n", t.ID, q, err)
|
||||
continue
|
||||
}
|
||||
covered := 0
|
||||
for _, marker := range inst.AnswerMarkers {
|
||||
if strings.Contains(strings.ToLower(res.Text), strings.ToLower(marker)) {
|
||||
covered++
|
||||
}
|
||||
}
|
||||
fmt.Printf(" %-24s %-34q %d/%d values in the result\n",
|
||||
t.ID, truncate(q, 32), covered, len(inst.AnswerMarkers))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func completeRate(rs []snippetReading) float64 {
|
||||
if len(rs) == 0 {
|
||||
return 0
|
||||
}
|
||||
n := 0
|
||||
for _, r := range rs {
|
||||
if r.Complete {
|
||||
n++
|
||||
}
|
||||
}
|
||||
return float64(n) / float64(len(rs))
|
||||
}
|
||||
|
||||
func meanRunes(rs []snippetReading) float64 {
|
||||
if len(rs) == 0 {
|
||||
return 0
|
||||
}
|
||||
total := 0
|
||||
for _, r := range rs {
|
||||
total += r.Runes
|
||||
}
|
||||
return float64(total) / float64(len(rs))
|
||||
}
|
||||
|
||||
// answerCoverage is what a run should have recorded all along: how much of the
|
||||
// scored answer a search actually handed over. rank=1 with coverage 1/3 is not
|
||||
// a stopping failure.
|
||||
func answerCoverage(text string, markers []string) int {
|
||||
covered := 0
|
||||
lower := strings.ToLower(text)
|
||||
for _, marker := range markers {
|
||||
if strings.Contains(lower, strings.ToLower(marker)) {
|
||||
covered++
|
||||
}
|
||||
}
|
||||
return covered
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sync"
|
||||
|
||||
"reasonix/internal/contract/event"
|
||||
"reasonix/internal/contract/provider"
|
||||
)
|
||||
|
||||
// usageSink reads what a run cost from the events the agent already emits. Every
|
||||
// billed completion reports event.Usage — the round path, compaction, command
|
||||
// routing and failure diagnosis all do — so the bench reads the same record the
|
||||
// product reads instead of wrapping a provider and counting a second time.
|
||||
type usageSink struct {
|
||||
mu sync.Mutex
|
||||
requests int
|
||||
usage provider.Usage
|
||||
}
|
||||
|
||||
func (s *usageSink) Emit(e event.Event) {
|
||||
if e.Kind != event.Usage || e.Usage == nil {
|
||||
return
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
// One event is one completion that reported usage; RequestCount covers a
|
||||
// completion that merged attempts. A request that failed before reporting
|
||||
// leaves no event, so this is not a count of attempts.
|
||||
if e.Usage.RequestCount > 0 {
|
||||
s.requests += e.Usage.RequestCount
|
||||
} else {
|
||||
s.requests++
|
||||
}
|
||||
u := *e.Usage
|
||||
s.usage.PromptTokens += u.PromptTokens
|
||||
s.usage.CompletionTokens += u.CompletionTokens
|
||||
s.usage.TotalTokens += u.TotalTokens
|
||||
s.usage.CacheHitTokens += u.CacheHitTokens
|
||||
s.usage.CacheMissTokens += u.CacheMissTokens
|
||||
s.usage.ReasoningTokens += u.ReasoningTokens
|
||||
s.usage.Estimated = s.usage.Estimated || u.Estimated
|
||||
}
|
||||
|
||||
func (s *usageSink) snapshot() (provider.Usage, int) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return s.usage, s.requests
|
||||
}
|
||||
|
||||
// promptTokenSuffix renders what a run cost only when something reported it. A
|
||||
// dry run reports nothing, and a trailing "prompt-tok=0" would read as a
|
||||
// measured zero rather than as no measurement.
|
||||
func promptTokenSuffix(m contextMetrics) string {
|
||||
if m.UsageReportedRequests == 0 {
|
||||
return ""
|
||||
}
|
||||
return fmt.Sprintf(" prompt-tok=%d", m.PromptTokens)
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"reasonix/internal/contract/event"
|
||||
"reasonix/internal/contract/provider"
|
||||
)
|
||||
|
||||
// The cost a run reports must be what the agent reported, summed across every
|
||||
// completion it billed — including one that merged attempts and one that could
|
||||
// only estimate. Any other event kind is not a cost.
|
||||
func TestUsageSinkAccumulatesWhatTheAgentReports(t *testing.T) {
|
||||
var sink usageSink
|
||||
sink.Emit(event.Event{}) // a kind that is not Usage
|
||||
sink.Emit(event.Event{Kind: event.Usage, Usage: &provider.Usage{
|
||||
PromptTokens: 100, CompletionTokens: 10, TotalTokens: 110, CacheHitTokens: 60, CacheMissTokens: 40,
|
||||
}})
|
||||
sink.Emit(event.Event{Kind: event.Usage, Usage: &provider.Usage{
|
||||
PromptTokens: 7, CompletionTokens: 3, CacheHitTokens: 2, CacheMissTokens: 5, RequestCount: 2, Estimated: true,
|
||||
}})
|
||||
sink.Emit(event.Event{Kind: event.Usage}) // a Usage event with no payload is not a cost
|
||||
|
||||
usage, requests := sink.snapshot()
|
||||
if requests != 3 {
|
||||
t.Fatalf("requests = %d, want 3 (1 + the merged attempt's 2; the empty event adds none)", requests)
|
||||
}
|
||||
if usage.PromptTokens != 107 || usage.CompletionTokens != 13 || usage.TotalTokens != 110 {
|
||||
t.Fatalf("accumulated usage = %+v", usage)
|
||||
}
|
||||
if usage.CacheHitTokens != 62 || usage.CacheMissTokens != 45 {
|
||||
t.Fatalf("cache split = %+v", usage)
|
||||
}
|
||||
if !usage.Estimated {
|
||||
t.Fatal("Estimated was dropped when one completion reported an estimate")
|
||||
}
|
||||
}
|
||||
|
||||
// A dry run must not claim a cost it never paid, in the JSON or in the line a
|
||||
// person reads: an empty suffix, not "prompt-tok=0".
|
||||
func TestUsageSinkStaysEmptyWithoutUsage(t *testing.T) {
|
||||
var sink usageSink
|
||||
sink.Emit(event.Event{})
|
||||
usage, requests := sink.snapshot()
|
||||
if requests != 0 || usage.PromptTokens != 0 || usage.CompletionTokens != 0 {
|
||||
t.Fatalf("nothing reported usage, got %+v requests=%d", usage, requests)
|
||||
}
|
||||
if suffix := promptTokenSuffix(contextMetrics{}); suffix != "" {
|
||||
t.Fatalf("a run with no reported usage still printed %q", suffix)
|
||||
}
|
||||
if suffix := promptTokenSuffix(contextMetrics{UsageReportedRequests: 1, PromptTokens: 12}); suffix != " prompt-tok=12" {
|
||||
t.Fatalf("a reported cost printed %q", suffix)
|
||||
}
|
||||
}
|
||||
@@ -1,131 +0,0 @@
|
||||
# realrepo
|
||||
|
||||
Long tasks over a pinned snapshot of this repository, for measuring changes
|
||||
that aim to cut model round trips. The `benchmarks/e2e` tasks finish in about
|
||||
5–15 requests and ten seconds; a real task in this tree took a flash-class
|
||||
model 38 requests, most of them after a single read. A suite of short fixtures
|
||||
cannot show that gap moving, so these tasks are shaped like the real one:
|
||||
explore several packages, edit several files, verify, and recover from the
|
||||
verifier.
|
||||
|
||||
```sh
|
||||
go build -o bin/reasonix ./cmd/reasonix
|
||||
go run ./cmd/e2ebench -suite benchmarks/realrepo -bin "$PWD/bin/reasonix" -model deepseek-flash \
|
||||
-budget 0 -trials 2 -trajectories traj -json realrepo.json
|
||||
go run ./cmd/e2ebench -suite benchmarks/realrepo -bin "$PWD/bin/reasonix" -task rt-stale-skills
|
||||
```
|
||||
|
||||
The agent reads its provider config from `REASONIX_HOME` (default
|
||||
`~/.reasonix`); point it at a disposable one to keep benchmark sessions out of
|
||||
your own.
|
||||
|
||||
The default `-budget` (800k tokens) can stop the suite partway through, hence
|
||||
`-budget 0`. Under `-trials`, each trial keeps its own
|
||||
`<task>.trial-N.trajectory.jsonl`.
|
||||
|
||||
## The snapshot
|
||||
|
||||
Every `workdir/` is the same Go module (`module reasonix`, stdlib only, no
|
||||
network needed to build or test):
|
||||
|
||||
- **Pinned code.** `git archive` of commit `42636ddcb425` for
|
||||
`internal/contract/{pricing,ablation,agentgraph}` and
|
||||
`internal/base/{retrieval,rulelist,tokencount,neterr,nilutil}` — about 5k
|
||||
lines of real kernel code with its real tests and doc comments.
|
||||
- **Fixture-authored files** in `base/`: `go.mod`, a trimmed `REASONIX.md` (the
|
||||
project's own conventions, so the agent loads standing instructions the way
|
||||
it does here), a `Makefile` with `check`, a module `README.md`, the
|
||||
`cmd/benchctl` command that consumes `pricing` and `ablation` from outside
|
||||
their package, and one project skill.
|
||||
- **One pinned test anchored to the snapshot day.**
|
||||
`TestCatalogPricesHaveBeenCheckedRecently` compares each row's `CheckedOn`
|
||||
with the wall clock, which would fail every workdir 90 days after the oldest
|
||||
pinned date (2026-11-20). `base/` replaces that file with one that reads
|
||||
staleness at 2026-10-19 instead; nothing else in the pin is edited.
|
||||
- **Per-task overlay** in `overlay/<task-id>/`, applied last. Only
|
||||
`rt-stale-skills` has one.
|
||||
|
||||
`snapshot.sh` rebuilds every workdir from those three sources and
|
||||
`snapshot.sh --check` fails if a committed workdir has drifted from them. Edit
|
||||
`base/` or `overlay/`, never a workdir, then rerun the script. Moving the pin is
|
||||
a new suite version: results from two pins do not compare.
|
||||
|
||||
## Tasks
|
||||
|
||||
| Task | Kind | What it exercises |
|
||||
| --- | --- | --- |
|
||||
| `rt-cache-write-rate` | multi-file Go change, test, README | Adds a `CacheWrite` rate to `RateCard` and threads it through cost, fingerprint, card identity and the catalog, then prices Anthropic's `claude-sonnet-4-5` with it: a new `officialRates` row, documentation and checked-on constants, an endpoint host, and `benchctl rate`/`cost` output. The fingerprint has to stay byte-identical for every existing card, so the change to `PricingFingerprint` cannot be the obvious one. |
|
||||
| `rt-stale-skills` | documentation only, no recognised verifier | Three project skills name paths, functions, `make` targets and `benchctl` subcommands and flags that do not exist, and one contradicts the append-only rule in `REASONIX.md`. Fixing them takes reading `rates.go`, `ablation.go`, both `benchctl` files and the `Makefile`, then editing only Markdown — the shape behind #13293, where `stale_verification` stays owed after a skill edit because no check covers it. |
|
||||
| `rt-rate-lookup-signatures` | API migration with a likely compile-error loop | Changes `CurrentRate` to `(RateCard, bool)`, `LookupCatalog` to a `CatalogKey` struct, and `IsOfficialCard`, `MatchesCatalog` and `LiveTariff` to a `ModelRef`: about 20 call sites across 11 files. Most are in `_test.go` files and dereference the old pointer, so `go build ./...` can pass while `go vet` and `go test` fail. |
|
||||
| `rt-plan-ablation-module` | plan-mode with `complete_step` | Five numbered steps adding a `memory` ablation module, summaries, `ModuleHelp` and `benchctl arm -list`. The prompt opens with the host's own plan-first marker (`control.PlannerRouteMarker`), which routes the turn through the planner, and asks for each step to be signed off before the next. An existing test pins the `all` arm name and has to be updated, which adds a red-test step. |
|
||||
|
||||
Every grader stages hidden tests and checks after the run: `gofmt -l`, `go vet
|
||||
./...` and `go test ./...` for the Go tasks; for `rt-stale-skills`, every
|
||||
backticked path, identifier, `make` target and `benchctl` command line in each
|
||||
skill must resolve against the tree, and a SHA-256 manifest proves no Go source, `go.mod`,
|
||||
`Makefile` or `REASONIX.md` changed. Each task fails its pristine seed and
|
||||
passes its `solution/`, both checked by `cmd/e2ebench`'s corpus tests.
|
||||
|
||||
## Reading the numbers
|
||||
|
||||
```text
|
||||
Metric: requests per task
|
||||
Anchor: the trajectory digest's requests_by_source, summed over sources
|
||||
Population: every graded trial of a task that ran to its own stop
|
||||
Excludes: tasks the token budget skipped, and runs that hit timeout_sec
|
||||
```
|
||||
|
||||
Requests is the headline because round trips are what these tasks exist to
|
||||
move. Read it beside the solve rate and the report's round outcomes: a change
|
||||
that drops requests by giving up on verification has not cut a round trip, it
|
||||
has cut a check. `rt-stale-skills` is read for whether the run finishes at all
|
||||
and how many rounds go to re-verifying after its last write.
|
||||
|
||||
## Baseline
|
||||
|
||||
`deepseek-flash`, 2 trials per task, at studio `0b8de9e63`. Requests are the
|
||||
trajectory's executor requests; all 8 runs ran to their own stop.
|
||||
|
||||
| Task | Solved | Requests | `edit_file` calls | Edit rounds |
|
||||
| --- | --- | --- | --- | --- |
|
||||
| `rt-cache-write-rate` | 2/2 | 31, 28 | 21, 20 | 8, 8 |
|
||||
| `rt-plan-ablation-module` | 2/2 | 24, 30 | 10, 13 | 9, 7 |
|
||||
| `rt-rate-lookup-signatures` | 2/2 | 13, 15 | 10, 17 | 2, 6 |
|
||||
| `rt-stale-skills` | 2/2* | 20, 26 | 3, 2 | 3, 3 |
|
||||
|
||||
\* One run first graded as failed, on a glob (`*_rates_test.go`) and a
|
||||
placeholder (`benchctl <subcommand>`) that were both correct. The grader now
|
||||
resolves globs against the tree and skips placeholders, and that run counts
|
||||
as solved. The same task's first version failed a run for naming
|
||||
`CatalogEntry.EffectiveFrom` in order to warn against it; that check went too.
|
||||
|
||||
Reads and edits already batch: 20 edit calls fit into 8 rounds, and several
|
||||
`edit_file` calls to one file often share a round. Rounds that edit a file the
|
||||
previous round also edited are rare (0–2 per run), so steering same-file edits
|
||||
to `multi_edit` has little to save. On this suite's first version, a
|
||||
`ToolBatchPolicy` sentence asking for it drew no `multi_edit` call and 167
|
||||
requests against 158.
|
||||
|
||||
`rt-plan-ablation-module` spends five rounds per run on `complete_step` alone.
|
||||
A description change inviting the sign-off to share the next step's reply was
|
||||
taken up in 2 of 4 trials (4 of 5 sign-offs shared each time, against 0 of 20
|
||||
in control), and requests did not move: 30, 25, 21, 28 against 27, 23, 24, 30.
|
||||
At this spread, four trials per arm cannot show a saving of a few rounds.
|
||||
|
||||
## Gaps
|
||||
|
||||
- **Below the intended size, in places.** The tasks were designed for 25–50
|
||||
requests. After widening, the median is 25, but `rt-rate-lookup-signatures`
|
||||
still runs 13–15: the model greps every call site first and batches the edits,
|
||||
so even about 20 call sites cost few rounds, and no run hit a compile-error
|
||||
loop.
|
||||
- **Lexical grader.** `rt-stale-skills` judges Markdown by resolving what it
|
||||
names. It has misjudged correct text twice (see the baseline note); a new
|
||||
failure there should be read before it is counted.
|
||||
- **No git history.** The workdir is a plain directory: `git diff`, `git log`
|
||||
and `git status` have nothing to read, so runs that lean on them in this tree
|
||||
behave differently here.
|
||||
- **Plan marker and `-force-planner`.** `rt-plan-ablation-module` carries the
|
||||
plan-first marker in its prompt, so it engages the planner in every arm,
|
||||
including runs without `-force-planner`. Exclude it when comparing planner
|
||||
arms.
|
||||
@@ -1,18 +0,0 @@
|
||||
---
|
||||
name: price-change
|
||||
description: Record a vendor's announced price change in the official price book without losing the rate it replaces.
|
||||
---
|
||||
|
||||
# Recording a price change
|
||||
|
||||
1. Find the vendor's rows in `officialRates` in
|
||||
`internal/contract/pricing/rates.go`. Each row is one model in one currency.
|
||||
2. **Append** a `rateGeneration` to the row's `Rates`; never edit an existing
|
||||
one. Set `From` to the first billed day (`YYYY-MM-DD`, UTC) when the change
|
||||
is announced ahead of time.
|
||||
3. Leave the earlier generation where it is: `SupersededRates` reads it so a
|
||||
stored price that matches it is recognised as ours and refreshed.
|
||||
4. Update the row's `CheckedOn` constant to the day you confirmed the page.
|
||||
5. Add a test beside `rates_test.go` that pins `CurrentRate` before and after
|
||||
the `From` day by replacing `today`.
|
||||
6. Run `make check`. `make pricecheck` needs the network and is run by hand.
|
||||
@@ -1,17 +0,0 @@
|
||||
.PHONY: check test vet fmt pricecheck
|
||||
|
||||
check: fmt vet test
|
||||
|
||||
fmt:
|
||||
@out="$$(gofmt -l .)"; if [ -n "$$out" ]; then echo "gofmt needed:"; echo "$$out"; exit 1; fi
|
||||
|
||||
vet:
|
||||
go vet ./...
|
||||
|
||||
test:
|
||||
go test ./...
|
||||
|
||||
# Reads each vendor's pricing page; this snapshot carries no network client.
|
||||
pricecheck:
|
||||
@echo "pricecheck: reaches each vendor's pricing page and needs the network; not available in this snapshot" >&2
|
||||
@exit 2
|
||||
@@ -1,69 +0,0 @@
|
||||
# Reasonix kernel snapshot
|
||||
|
||||
A trimmed slice of the Reasonix Go kernel: the contracts a benchmark report is
|
||||
priced and labelled with, a few leaf utilities, and `benchctl`, a small command
|
||||
that prints what those contracts answer.
|
||||
|
||||
```bash
|
||||
make check # gofmt, vet, test
|
||||
go run ./cmd/benchctl rate -model deepseek-flash -currency USD
|
||||
go run ./cmd/benchctl cost -provider mimo -model mimo-v2.5 -miss 1000000
|
||||
go run ./cmd/benchctl tariff -currency CNY -at 2026-10-12T02:00:00Z
|
||||
go run ./cmd/benchctl arm -ablate planner,evidence
|
||||
```
|
||||
|
||||
## Packages
|
||||
|
||||
| Package | Owns |
|
||||
| --- | --- |
|
||||
| `internal/contract/pricing` | the official price book, cost quotes, the spend ledger |
|
||||
| `internal/contract/ablation` | which subsystems a benchmark arm switches off, and the arm's name |
|
||||
| `internal/contract/agentgraph` | the vocabulary of one run's execution graph |
|
||||
| `internal/base/retrieval` | BM25 ranking over CJK-aware tokens |
|
||||
| `internal/base/rulelist` | splitting a one-line permission rule list |
|
||||
| `internal/base/tokencount` | a cheap token estimate |
|
||||
| `internal/base/neterr` | classifying transport errors |
|
||||
| `internal/base/nilutil` | typed-nil detection |
|
||||
| `cmd/benchctl` | prints rates, costs, peak schedules and arm names |
|
||||
|
||||
## Pricing
|
||||
|
||||
A `RateCard` is a price per million tokens in one currency:
|
||||
|
||||
| Field | Charges |
|
||||
| --- | --- |
|
||||
| `CacheHit` | prompt tokens served from the provider's cache |
|
||||
| `Input` | prompt tokens that missed the cache, including tokens written to it |
|
||||
| `Output` | completion tokens |
|
||||
|
||||
`OriginalCostAmount` turns a card and a `UsageTokens` breakdown into a
|
||||
fixed-point `Amount`; `PricingFingerprint` keys a card in the ledger, so a
|
||||
stored ledger entry stays attached to the card it was priced with.
|
||||
`CurrentRate` and `SupersededRates` answer from `officialRates`, which only
|
||||
ever grows: a price change appends a generation.
|
||||
|
||||
## Ablation
|
||||
|
||||
`ablation.Parse` reads a comma-separated list of modules to switch off, and
|
||||
`Set.Arm` names the result (`full` for the control). The switchable modules,
|
||||
in the order arm names use:
|
||||
|
||||
| Module | Off means |
|
||||
| --- | --- |
|
||||
| `evidence` | no evidence gate on completion claims |
|
||||
| `planner` | no two-model planning turn |
|
||||
| `subagent` | no delegation to workers |
|
||||
| `retrieval` | no retrieval over the folded transcript |
|
||||
| `compaction` | no context folding |
|
||||
| `upstream` | a dependency edge orders its endpoints without delivering the answer |
|
||||
| `recall-search` | recall can read a given address but cannot search for one |
|
||||
|
||||
The fold-index scale is a separate axis (`-fold-index`), not a module.
|
||||
|
||||
## Skills
|
||||
|
||||
Project skills live under `.reasonix/skills/<name>/SKILL.md`.
|
||||
|
||||
| Skill | Use it to |
|
||||
| --- | --- |
|
||||
| `price-change` | record a vendor's price change in the price book |
|
||||
@@ -1,51 +0,0 @@
|
||||
# Reasonix project memory
|
||||
|
||||
This checkout is a trimmed snapshot of the Reasonix kernel: the billing,
|
||||
ablation and graph contracts and a handful of leaf utilities, plus one small
|
||||
command that drives them. The conventions below are the project's own.
|
||||
|
||||
## Conventions
|
||||
|
||||
- Go kernel under `internal/`; each package owns one concern. A package's long
|
||||
explanation belongs in its `doc.go`, not spread across implementation files.
|
||||
- Judgements read structure, never wording: types, contracts, schemas. An
|
||||
error a caller must tell apart carries an identity — a sentinel, a typed
|
||||
code — never a sentence.
|
||||
- A metric is its definition, not its name. Before a number becomes a finding,
|
||||
state what it counts, from which anchor, over which population, and what it
|
||||
excludes.
|
||||
|
||||
## Comments
|
||||
|
||||
Default is none — the code is the truth. Write one only when the **why** is
|
||||
non-obvious: a hidden constraint, an invariant the type system cannot express,
|
||||
or an external-protocol quirk. A comment states the constraint that holds now;
|
||||
it does not narrate how the code got here. English, always.
|
||||
|
||||
- Declaration doc: ≤5 lines. Package comment: ≤8 lines, or ≤40 in a `doc.go`.
|
||||
- Every other comment: ≤3 lines. Struct-field and trailing `//`: 1 line.
|
||||
- One responsibility per file; 800 lines is the ceiling.
|
||||
|
||||
## Vendor rates
|
||||
|
||||
Official list prices live in one place: `officialRates` in
|
||||
`internal/contract/pricing/rates.go`, oldest generation first, the last being
|
||||
what a model charges today. A price change is an **append, never an edit** —
|
||||
the superseded rate is what lets an installed config be recognised as holding
|
||||
one of ours and brought forward. Callers read these through
|
||||
`pricing.CurrentRate` and `pricing.SupersededRates` and restate none of them.
|
||||
|
||||
`make pricecheck` compares that table against each vendor's own page. It needs
|
||||
the network, so it is not part of `make check`.
|
||||
|
||||
## Before every commit
|
||||
|
||||
```bash
|
||||
make check # gofmt -l, go vet ./..., go test ./...
|
||||
```
|
||||
|
||||
## Change discipline
|
||||
|
||||
List callers, consumers of the same state and sibling features before editing.
|
||||
Fix at the owning layer; a fix that breaks a neighbouring behaviour is a failed
|
||||
fix. Write tests first; they must fail without the change.
|
||||
@@ -1,115 +0,0 @@
|
||||
// Command benchctl prints what the pricing and ablation contracts answer for
|
||||
// one model or one benchmark arm, so a reviewer can check a rate or an arm name
|
||||
// without writing a test.
|
||||
package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
|
||||
"reasonix/internal/contract/ablation"
|
||||
"reasonix/internal/contract/pricing"
|
||||
)
|
||||
|
||||
var errUsage = errors.New("usage: benchctl rate|cost|tariff|arm [flags]")
|
||||
|
||||
func main() {
|
||||
if err := run(os.Args[1:], os.Stdout); err != nil {
|
||||
fmt.Fprintln(os.Stderr, "benchctl:", err)
|
||||
os.Exit(2)
|
||||
}
|
||||
}
|
||||
|
||||
func run(args []string, out io.Writer) error {
|
||||
if len(args) == 0 {
|
||||
return errUsage
|
||||
}
|
||||
switch args[0] {
|
||||
case "rate":
|
||||
return runRate(args[1:], out)
|
||||
case "cost":
|
||||
return runCost(args[1:], out)
|
||||
case "tariff":
|
||||
return runTariff(args[1:], out)
|
||||
case "arm":
|
||||
return runArm(args[1:], out)
|
||||
default:
|
||||
return fmt.Errorf("%w: unknown subcommand %q", errUsage, args[0])
|
||||
}
|
||||
}
|
||||
|
||||
func runRate(args []string, out io.Writer) error {
|
||||
fs := flag.NewFlagSet("rate", flag.ContinueOnError)
|
||||
fs.SetOutput(out)
|
||||
provider := fs.String("provider", "deepseek", "vendor")
|
||||
model := fs.String("model", "deepseek-flash", "model id")
|
||||
currency := fs.String("currency", "", "currency; empty reads the vendor's default")
|
||||
if err := fs.Parse(args); err != nil {
|
||||
return err
|
||||
}
|
||||
cur := *currency
|
||||
if cur == "" {
|
||||
cur = pricing.DefaultCurrency(*provider)
|
||||
}
|
||||
card := pricing.CurrentRate(*provider, *model, cur)
|
||||
if card == nil {
|
||||
return fmt.Errorf("no official rate for %s/%s in %s", *provider, *model, cur)
|
||||
}
|
||||
fmt.Fprintf(out, "%s/%s %s: cache-hit %g input %g output %g per 1M\n",
|
||||
*provider, *model, card.Currency, card.CacheHit, card.Input, card.Output)
|
||||
if entry, ok := pricing.LookupCatalog(*provider, *model, cur, ""); ok && entry.DocURL != "" {
|
||||
fmt.Fprintf(out, "source: %s\n", entry.DocURL)
|
||||
}
|
||||
fmt.Fprintf(out, "superseded generations: %d\n", len(pricing.SupersededRates(*provider, *model, cur)))
|
||||
return nil
|
||||
}
|
||||
|
||||
func runCost(args []string, out io.Writer) error {
|
||||
fs := flag.NewFlagSet("cost", flag.ContinueOnError)
|
||||
fs.SetOutput(out)
|
||||
provider := fs.String("provider", "deepseek", "vendor")
|
||||
model := fs.String("model", "deepseek-flash", "model id")
|
||||
currency := fs.String("currency", "", "currency; empty reads the vendor's default")
|
||||
hit := fs.Int("hit", 0, "cached prompt tokens")
|
||||
miss := fs.Int("miss", 0, "uncached prompt tokens")
|
||||
completion := fs.Int("completion", 0, "completion tokens")
|
||||
if err := fs.Parse(args); err != nil {
|
||||
return err
|
||||
}
|
||||
cur := *currency
|
||||
if cur == "" {
|
||||
cur = pricing.DefaultCurrency(*provider)
|
||||
}
|
||||
card := pricing.CurrentRate(*provider, *model, cur)
|
||||
if card == nil {
|
||||
return fmt.Errorf("no official rate for %s/%s in %s", *provider, *model, cur)
|
||||
}
|
||||
usage := pricing.UsageTokens{CacheHitTokens: *hit, CacheMissTokens: *miss, CompletionTokens: *completion}
|
||||
amount := pricing.OriginalCostAmount(*card, usage)
|
||||
fmt.Fprintf(out, "%s %s\n", amount, pricing.NormalizeCurrency(card.Currency))
|
||||
return nil
|
||||
}
|
||||
|
||||
func runArm(args []string, out io.Writer) error {
|
||||
fs := flag.NewFlagSet("arm", flag.ContinueOnError)
|
||||
fs.SetOutput(out)
|
||||
spec := fs.String("ablate", "", "comma-separated modules to switch off: "+ablation.ModuleList())
|
||||
foldIndex := fs.String("fold-index", "", "fold-index scale: default, half, quarter or off")
|
||||
list := fs.Bool("list", false, "list the switchable modules")
|
||||
if err := fs.Parse(args); err != nil {
|
||||
return err
|
||||
}
|
||||
if *list {
|
||||
fmt.Fprintln(out, ablation.ModuleList())
|
||||
return nil
|
||||
}
|
||||
set, err := ablation.ParseArm(*spec, *foldIndex)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Fprintln(out, set.Arm())
|
||||
return nil
|
||||
}
|
||||
@@ -1,65 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func runOut(t *testing.T, args ...string) string {
|
||||
t.Helper()
|
||||
var out bytes.Buffer
|
||||
if err := run(args, &out); err != nil {
|
||||
t.Fatalf("run %v: %v", args, err)
|
||||
}
|
||||
return out.String()
|
||||
}
|
||||
|
||||
func TestRateNamesTheCurrentCard(t *testing.T) {
|
||||
got := runOut(t, "rate", "-provider", "mimo", "-model", "mimo-v2.5")
|
||||
if !strings.Contains(got, "mimo/mimo-v2.5 CNY: cache-hit 0.02 input 1 output 2 per 1M") {
|
||||
t.Fatalf("rate output = %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRateRefusesAnUnpricedModel(t *testing.T) {
|
||||
var out bytes.Buffer
|
||||
if err := run([]string{"rate", "-provider", "nobody", "-model", "x"}, &out); err == nil {
|
||||
t.Fatal("an unpriced model must fail")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCostChargesTheCurrentCard(t *testing.T) {
|
||||
got := runOut(t, "cost", "-provider", "mimo", "-model", "mimo-v2.5", "-miss", "1000000", "-completion", "1000000")
|
||||
if strings.TrimSpace(got) != "3 CNY" {
|
||||
t.Fatalf("cost output = %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestArmNamesTheConfiguration(t *testing.T) {
|
||||
if got := strings.TrimSpace(runOut(t, "arm")); got != "full" {
|
||||
t.Fatalf("control arm = %q", got)
|
||||
}
|
||||
if got := strings.TrimSpace(runOut(t, "arm", "-ablate", "planner,evidence", "-fold-index", "half")); got != "no-evidence+no-planner+index-half" {
|
||||
t.Fatalf("arm = %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUnknownSubcommandIsAUsageError(t *testing.T) {
|
||||
var out bytes.Buffer
|
||||
if err := run([]string{"nope"}, &out); !errors.Is(err, errUsage) {
|
||||
t.Fatalf("err = %v, want errUsage", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTariffReadsTheVendorsSchedule(t *testing.T) {
|
||||
got := runOut(t, "tariff", "-provider", "mimo", "-model", "mimo-v2.5")
|
||||
if !strings.Contains(got, "official card: true") || !strings.Contains(got, "one rate around the clock") {
|
||||
t.Fatalf("tariff output = %q", got)
|
||||
}
|
||||
peak := runOut(t, "tariff", "-currency", "CNY", "-at", "2026-10-12T02:00:00Z")
|
||||
if !strings.Contains(peak, "peak=true until 2026-10-12T04:00:00Z") {
|
||||
t.Fatalf("deepseek tariff at a weekday peak = %q", peak)
|
||||
}
|
||||
}
|
||||
@@ -1,49 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"time"
|
||||
|
||||
"reasonix/internal/contract/pricing"
|
||||
)
|
||||
|
||||
func runTariff(args []string, out io.Writer) error {
|
||||
fs := flag.NewFlagSet("tariff", flag.ContinueOnError)
|
||||
fs.SetOutput(out)
|
||||
provider := fs.String("provider", "deepseek", "vendor")
|
||||
model := fs.String("model", "deepseek-flash", "model id")
|
||||
currency := fs.String("currency", "", "currency; empty reads the vendor's default")
|
||||
at := fs.String("at", "", "instant to read the schedule at, RFC 3339; empty is now")
|
||||
if err := fs.Parse(args); err != nil {
|
||||
return err
|
||||
}
|
||||
when := time.Now()
|
||||
if *at != "" {
|
||||
t, err := time.Parse(time.RFC3339, *at)
|
||||
if err != nil {
|
||||
return fmt.Errorf("-at: %w", err)
|
||||
}
|
||||
when = t
|
||||
}
|
||||
cur := *currency
|
||||
if cur == "" {
|
||||
cur = pricing.DefaultCurrency(*provider)
|
||||
}
|
||||
card := pricing.CurrentRate(*provider, *model, cur)
|
||||
if card == nil {
|
||||
return fmt.Errorf("no official rate for %s/%s in %s", *provider, *model, cur)
|
||||
}
|
||||
fmt.Fprintf(out, "official card: %v\n", pricing.IsOfficialCard(*provider, *model, *card))
|
||||
tariff, ok := pricing.LiveTariff(*provider, *model, *card, when)
|
||||
switch {
|
||||
case !ok:
|
||||
fmt.Fprintln(out, "schedule: one rate around the clock")
|
||||
case !tariff.Changes:
|
||||
fmt.Fprintf(out, "schedule: peak=%v, no change ahead\n", tariff.Peak)
|
||||
default:
|
||||
fmt.Fprintf(out, "schedule: peak=%v until %s\n", tariff.Peak, tariff.ChangesAt.UTC().Format(time.RFC3339))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -1,3 +0,0 @@
|
||||
module reasonix
|
||||
|
||||
go 1.26.0
|
||||
@@ -1,95 +0,0 @@
|
||||
package pricing
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Every catalog provider must declare at least one official endpoint, or its
|
||||
// rows become unreachable: a rate card is only matched against the catalog
|
||||
// after the entry's host identifies the vendor.
|
||||
func TestEveryCatalogProviderHasAnOfficialEndpoint(t *testing.T) {
|
||||
declared := map[string]bool{}
|
||||
for _, host := range officialEndpointHosts {
|
||||
declared[host] = true
|
||||
}
|
||||
for _, entry := range OfficialCatalog() {
|
||||
if !declared[entry.Provider] {
|
||||
t.Errorf("catalog provider %q has no official endpoint host; its prices can never match", entry.Provider)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestOfficialProviderForEndpoint(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
baseURL string
|
||||
want string
|
||||
}{
|
||||
{"official deepseek", "https://api.deepseek.com", "deepseek"},
|
||||
{"official deepseek anthropic path", "https://api.deepseek.com/anthropic", "deepseek"},
|
||||
{"official longcat", "https://api.longcat.chat/openai/v1", "longcat"},
|
||||
{"official mimo region", "https://token-plan-sgp.xiaomimimo.com/v1", "mimo"},
|
||||
{"uppercase host", "https://API.DeepSeek.com/v1", "deepseek"},
|
||||
{"proxy that resells the model", "https://my-deepseek-proxy.internal/v1", ""},
|
||||
{"vendor name in the path only", "https://gateway.example.com/deepseek/v1", ""},
|
||||
{"lookalike registrable domain", "https://api.deepseek.com.evil.test/v1", ""},
|
||||
{"empty", "", ""},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
if got := OfficialProviderForEndpoint(tc.baseURL); got != tc.want {
|
||||
t.Fatalf("OfficialProviderForEndpoint(%q) = %q, want %q", tc.baseURL, got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// pricesGoStaleAfter is how long a hand-copied rate is trusted before someone
|
||||
// reads the vendor's page again. No vendor here serves prices over an API, so
|
||||
// the alternative to this going red on a quiet day is learning about a change
|
||||
// from a user whose cost was wrong for months. Bumping CheckedOn without
|
||||
// opening DocURL defeats it; what this buys is that the question gets asked.
|
||||
const pricesGoStaleAfter = 90 * 24 * time.Hour
|
||||
|
||||
func TestCatalogPricesHaveBeenCheckedRecently(t *testing.T) {
|
||||
var undated, stale []string
|
||||
// This snapshot never sees a vendor page again, so staleness is read at the
|
||||
// day it was taken rather than the day the suite runs.
|
||||
cutoff := time.Date(2026, 10, 19, 0, 0, 0, 0, time.UTC).Add(-pricesGoStaleAfter)
|
||||
for _, e := range OfficialCatalog() {
|
||||
where := e.Provider + "/" + e.Model + " " + e.Currency + " (" + e.DocURL + ")"
|
||||
if e.CheckedOn == "" {
|
||||
undated = append(undated, where)
|
||||
continue
|
||||
}
|
||||
on, err := time.Parse("2006-01-02", e.CheckedOn)
|
||||
if err != nil {
|
||||
t.Fatalf("%s: CheckedOn %q is not YYYY-MM-DD", where, e.CheckedOn)
|
||||
}
|
||||
if on.Before(cutoff) {
|
||||
stale = append(stale, where+" last checked "+e.CheckedOn)
|
||||
}
|
||||
}
|
||||
if len(undated) > 0 {
|
||||
t.Errorf("catalog rows with no CheckedOn — read the vendor's page and date them:\n %s",
|
||||
strings.Join(undated, "\n "))
|
||||
}
|
||||
if len(stale) > 0 {
|
||||
t.Errorf("catalog rows older than %d days. Open each DocURL, confirm or correct the rate, then set CheckedOn to today:\n %s",
|
||||
int(pricesGoStaleAfter.Hours()/24), strings.Join(stale, "\n "))
|
||||
}
|
||||
}
|
||||
|
||||
// A row that names a peak card must say when it applies, and one that names a
|
||||
// window must have a card to charge inside it. Half of the pair silently prices
|
||||
// every hour the same, which is the bug the pair exists to prevent.
|
||||
func TestPeakRatesAndWindowsComeInPairs(t *testing.T) {
|
||||
for _, e := range OfficialCatalog() {
|
||||
where := e.Provider + "/" + e.Model + " " + e.Currency
|
||||
if (e.Peak == nil) != (e.Window == nil) {
|
||||
t.Errorf("%s: Peak and Window must be set together (peak=%v window=%v)", where, e.Peak != nil, e.Window != nil)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,12 +0,0 @@
|
||||
---
|
||||
name: ablation-arm
|
||||
description: Add a new switchable subsystem to the benchmark's ablation arms.
|
||||
---
|
||||
|
||||
# Adding an ablation module
|
||||
|
||||
1. Declare the module in `internal/ablation/modules.go` beside the others.
|
||||
2. Register it in `AllModules()` so `ParseSpec` accepts it.
|
||||
3. Arm names come from `Set.Name()`; check that yours reads `without-<module>`.
|
||||
4. Add the module to the flag help text in `cmd/bench/flags.go`.
|
||||
5. Run `make test`.
|
||||
@@ -1,15 +0,0 @@
|
||||
---
|
||||
name: benchctl
|
||||
description: Check a rate, a cost, a peak schedule or an arm name from the command line instead of writing a test.
|
||||
---
|
||||
|
||||
# Using benchctl
|
||||
|
||||
1. Price a request with `benchctl quote -provider mimo -model mimo-v2.5 -tokens 1000000`;
|
||||
it prints the amount and the currency.
|
||||
2. Read today's card with `benchctl rate -vendor deepseek -model deepseek-flash`.
|
||||
3. Ask whether a vendor is in its peak window with
|
||||
`benchctl peak -currency CNY -at 1791770400` (Unix seconds).
|
||||
4. List the ablation modules with `benchctl arms`, and name an arm with
|
||||
`benchctl arm -ablate planner,evidence -index half`.
|
||||
5. Every subcommand prints its flags with `-h`.
|
||||
@@ -1,13 +0,0 @@
|
||||
---
|
||||
name: price-change
|
||||
description: Record a vendor's announced price change in the official price book.
|
||||
---
|
||||
|
||||
# Recording a price change
|
||||
|
||||
1. Open `internal/pricing/rates.go` and find the vendor's entry in
|
||||
`officialPrices`.
|
||||
2. Edit the entry's `Input`, `Output` and `CacheHit` to the new values.
|
||||
3. If the change is announced ahead of time, set `EffectiveFrom` on the entry.
|
||||
4. Confirm `pricing.LatestRate` now returns the new card.
|
||||
5. Run `make test` and `make price-check`.
|
||||
@@ -1,52 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# Builds every task's workdir/ from one pinned commit of this repository plus
|
||||
# the fixture-authored files in base/ and overlay/<task-id>/. With --check it
|
||||
# rebuilds into a scratch directory and fails if any committed workdir differs.
|
||||
set -euo pipefail
|
||||
|
||||
PIN=42636ddcb425b7af07abd88397d02c1cfe3adcf5
|
||||
PATHS=(
|
||||
internal/contract/pricing
|
||||
internal/contract/ablation
|
||||
internal/contract/agentgraph
|
||||
internal/base/retrieval
|
||||
internal/base/rulelist
|
||||
internal/base/tokencount
|
||||
internal/base/neterr
|
||||
internal/base/nilutil
|
||||
)
|
||||
|
||||
here="$(cd "$(dirname "$0")" && pwd)"
|
||||
repo="$(git -C "$here" rev-parse --show-toplevel)"
|
||||
|
||||
build() {
|
||||
local id="$1" dest="$2"
|
||||
rm -rf "$dest"
|
||||
mkdir -p "$dest"
|
||||
git -C "$repo" archive "$PIN" -- "${PATHS[@]}" | tar -x -C "$dest"
|
||||
cp -R "$here/base/." "$dest/"
|
||||
if [ -d "$here/overlay/$id" ]; then
|
||||
cp -R "$here/overlay/$id/." "$dest/"
|
||||
fi
|
||||
}
|
||||
|
||||
check=0
|
||||
[ "${1:-}" = "--check" ] && check=1
|
||||
|
||||
status=0
|
||||
for task in "$here"/tasks/*/; do
|
||||
id="$(basename "$task")"
|
||||
if [ "$check" = 1 ]; then
|
||||
scratch="$(mktemp -d)"
|
||||
build "$id" "$scratch/workdir"
|
||||
if ! diff -r "$scratch/workdir" "$task/workdir" >/dev/null; then
|
||||
echo "$id: workdir/ differs from the pinned snapshot; run $0" >&2
|
||||
diff -rq "$scratch/workdir" "$task/workdir" >&2 || true
|
||||
status=1
|
||||
fi
|
||||
rm -rf "$scratch"
|
||||
else
|
||||
build "$id" "$task/workdir"
|
||||
fi
|
||||
done
|
||||
exit "$status"
|
||||
@@ -1,73 +0,0 @@
|
||||
# Reasonix kernel snapshot
|
||||
|
||||
A trimmed slice of the Reasonix Go kernel: the contracts a benchmark report is
|
||||
priced and labelled with, a few leaf utilities, and `benchctl`, a small command
|
||||
that prints what those contracts answer.
|
||||
|
||||
```bash
|
||||
make check # gofmt, vet, test
|
||||
go run ./cmd/benchctl rate -model deepseek-flash -currency USD
|
||||
go run ./cmd/benchctl cost -provider mimo -model mimo-v2.5 -miss 1000000
|
||||
go run ./cmd/benchctl tariff -currency CNY -at 2026-10-12T02:00:00Z
|
||||
go run ./cmd/benchctl arm -ablate planner,evidence
|
||||
```
|
||||
|
||||
## Packages
|
||||
|
||||
| Package | Owns |
|
||||
| --- | --- |
|
||||
| `internal/contract/pricing` | the official price book, cost quotes, the spend ledger |
|
||||
| `internal/contract/ablation` | which subsystems a benchmark arm switches off, and the arm's name |
|
||||
| `internal/contract/agentgraph` | the vocabulary of one run's execution graph |
|
||||
| `internal/base/retrieval` | BM25 ranking over CJK-aware tokens |
|
||||
| `internal/base/rulelist` | splitting a one-line permission rule list |
|
||||
| `internal/base/tokencount` | a cheap token estimate |
|
||||
| `internal/base/neterr` | classifying transport errors |
|
||||
| `internal/base/nilutil` | typed-nil detection |
|
||||
| `cmd/benchctl` | prints rates, costs, peak schedules and arm names |
|
||||
|
||||
## Pricing
|
||||
|
||||
A `RateCard` is a price per million tokens in one currency:
|
||||
|
||||
| Field | Charges |
|
||||
| --- | --- |
|
||||
| `CacheHit` | prompt tokens served from the provider's cache |
|
||||
| `Input` | prompt tokens that missed the cache, and tokens written to it when `CacheWrite` is zero |
|
||||
| `Output` | completion tokens |
|
||||
| `CacheWrite` | tokens written to the cache, for a vendor that prices them separately (Anthropic) |
|
||||
|
||||
A card without `CacheWrite` keeps the fingerprint it had before the field
|
||||
existed, so ledgers stored against it stay attached.
|
||||
|
||||
`OriginalCostAmount` turns a card and a `UsageTokens` breakdown into a
|
||||
fixed-point `Amount`; `PricingFingerprint` keys a card in the ledger, so a
|
||||
stored ledger entry stays attached to the card it was priced with.
|
||||
`CurrentRate` and `SupersededRates` answer from `officialRates`, which only
|
||||
ever grows: a price change appends a generation.
|
||||
|
||||
## Ablation
|
||||
|
||||
`ablation.Parse` reads a comma-separated list of modules to switch off, and
|
||||
`Set.Arm` names the result (`full` for the control). The switchable modules,
|
||||
in the order arm names use:
|
||||
|
||||
| Module | Off means |
|
||||
| --- | --- |
|
||||
| `evidence` | no evidence gate on completion claims |
|
||||
| `planner` | no two-model planning turn |
|
||||
| `subagent` | no delegation to workers |
|
||||
| `retrieval` | no retrieval over the folded transcript |
|
||||
| `compaction` | no context folding |
|
||||
| `upstream` | a dependency edge orders its endpoints without delivering the answer |
|
||||
| `recall-search` | recall can read a given address but cannot search for one |
|
||||
|
||||
The fold-index scale is a separate axis (`-fold-index`), not a module.
|
||||
|
||||
## Skills
|
||||
|
||||
Project skills live under `.reasonix/skills/<name>/SKILL.md`.
|
||||
|
||||
| Skill | Use it to |
|
||||
| --- | --- |
|
||||
| `price-change` | record a vendor's price change in the price book |
|
||||
@@ -1,119 +0,0 @@
|
||||
// Command benchctl prints what the pricing and ablation contracts answer for
|
||||
// one model or one benchmark arm, so a reviewer can check a rate or an arm name
|
||||
// without writing a test.
|
||||
package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
|
||||
"reasonix/internal/contract/ablation"
|
||||
"reasonix/internal/contract/pricing"
|
||||
)
|
||||
|
||||
var errUsage = errors.New("usage: benchctl rate|cost|tariff|arm [flags]")
|
||||
|
||||
func main() {
|
||||
if err := run(os.Args[1:], os.Stdout); err != nil {
|
||||
fmt.Fprintln(os.Stderr, "benchctl:", err)
|
||||
os.Exit(2)
|
||||
}
|
||||
}
|
||||
|
||||
func run(args []string, out io.Writer) error {
|
||||
if len(args) == 0 {
|
||||
return errUsage
|
||||
}
|
||||
switch args[0] {
|
||||
case "rate":
|
||||
return runRate(args[1:], out)
|
||||
case "cost":
|
||||
return runCost(args[1:], out)
|
||||
case "tariff":
|
||||
return runTariff(args[1:], out)
|
||||
case "arm":
|
||||
return runArm(args[1:], out)
|
||||
default:
|
||||
return fmt.Errorf("%w: unknown subcommand %q", errUsage, args[0])
|
||||
}
|
||||
}
|
||||
|
||||
func runRate(args []string, out io.Writer) error {
|
||||
fs := flag.NewFlagSet("rate", flag.ContinueOnError)
|
||||
fs.SetOutput(out)
|
||||
provider := fs.String("provider", "deepseek", "vendor")
|
||||
model := fs.String("model", "deepseek-flash", "model id")
|
||||
currency := fs.String("currency", "", "currency; empty reads the vendor's default")
|
||||
if err := fs.Parse(args); err != nil {
|
||||
return err
|
||||
}
|
||||
cur := *currency
|
||||
if cur == "" {
|
||||
cur = pricing.DefaultCurrency(*provider)
|
||||
}
|
||||
card := pricing.CurrentRate(*provider, *model, cur)
|
||||
if card == nil {
|
||||
return fmt.Errorf("no official rate for %s/%s in %s", *provider, *model, cur)
|
||||
}
|
||||
fmt.Fprintf(out, "%s/%s %s: cache-hit %g input %g output %g", *provider, *model, card.Currency, card.CacheHit, card.Input, card.Output)
|
||||
if card.CacheWrite != 0 {
|
||||
fmt.Fprintf(out, " cache-write %g", card.CacheWrite)
|
||||
}
|
||||
fmt.Fprintln(out, " per 1M")
|
||||
if entry, ok := pricing.LookupCatalog(*provider, *model, cur, ""); ok && entry.DocURL != "" {
|
||||
fmt.Fprintf(out, "source: %s\n", entry.DocURL)
|
||||
}
|
||||
fmt.Fprintf(out, "superseded generations: %d\n", len(pricing.SupersededRates(*provider, *model, cur)))
|
||||
return nil
|
||||
}
|
||||
|
||||
func runCost(args []string, out io.Writer) error {
|
||||
fs := flag.NewFlagSet("cost", flag.ContinueOnError)
|
||||
fs.SetOutput(out)
|
||||
provider := fs.String("provider", "deepseek", "vendor")
|
||||
model := fs.String("model", "deepseek-flash", "model id")
|
||||
currency := fs.String("currency", "", "currency; empty reads the vendor's default")
|
||||
hit := fs.Int("hit", 0, "cached prompt tokens")
|
||||
miss := fs.Int("miss", 0, "uncached prompt tokens")
|
||||
completion := fs.Int("completion", 0, "completion tokens")
|
||||
write := fs.Int("write", 0, "uncached prompt tokens that were written to the cache")
|
||||
if err := fs.Parse(args); err != nil {
|
||||
return err
|
||||
}
|
||||
cur := *currency
|
||||
if cur == "" {
|
||||
cur = pricing.DefaultCurrency(*provider)
|
||||
}
|
||||
card := pricing.CurrentRate(*provider, *model, cur)
|
||||
if card == nil {
|
||||
return fmt.Errorf("no official rate for %s/%s in %s", *provider, *model, cur)
|
||||
}
|
||||
usage := pricing.UsageTokens{CacheHitTokens: *hit, CacheMissTokens: *miss, CompletionTokens: *completion, CacheWriteTokens: *write}
|
||||
amount := pricing.OriginalCostAmount(*card, usage)
|
||||
fmt.Fprintf(out, "%s %s\n", amount, pricing.NormalizeCurrency(card.Currency))
|
||||
return nil
|
||||
}
|
||||
|
||||
func runArm(args []string, out io.Writer) error {
|
||||
fs := flag.NewFlagSet("arm", flag.ContinueOnError)
|
||||
fs.SetOutput(out)
|
||||
spec := fs.String("ablate", "", "comma-separated modules to switch off: "+ablation.ModuleList())
|
||||
foldIndex := fs.String("fold-index", "", "fold-index scale: default, half, quarter or off")
|
||||
list := fs.Bool("list", false, "list the switchable modules")
|
||||
if err := fs.Parse(args); err != nil {
|
||||
return err
|
||||
}
|
||||
if *list {
|
||||
fmt.Fprintln(out, ablation.ModuleList())
|
||||
return nil
|
||||
}
|
||||
set, err := ablation.ParseArm(*spec, *foldIndex)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Fprintln(out, set.Arm())
|
||||
return nil
|
||||
}
|
||||
@@ -1,3 +0,0 @@
|
||||
module reasonix
|
||||
|
||||
go 1.26.0
|
||||
-37
@@ -1,37 +0,0 @@
|
||||
package pricing
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestCacheWriteRateChargesWrittenTokens(t *testing.T) {
|
||||
card := RateCard{Input: 3, CacheWrite: 3.75, Currency: "USD"}
|
||||
usage := UsageTokens{CacheMissTokens: 1000000, CacheWriteTokens: 400000}
|
||||
if got := OriginalCostAmount(card, usage).String(); got != "3.3" {
|
||||
t.Fatalf("cost = %s, want 3.3", got)
|
||||
}
|
||||
card.CacheWrite = 0
|
||||
if got := OriginalCostAmount(card, usage).String(); got != "3" {
|
||||
t.Fatalf("cost without a cache-write rate = %s, want 3", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCacheWriteSeparatesFingerprintsAndCards(t *testing.T) {
|
||||
base := RateCard{CacheHit: 0.02, Input: 1, Output: 2, Currency: "CNY"}
|
||||
withWrite := base
|
||||
withWrite.CacheWrite = 1.25
|
||||
if PricingFingerprint(base) == PricingFingerprint(withWrite) {
|
||||
t.Fatal("cache-write rate does not reach the fingerprint")
|
||||
}
|
||||
if sameCard(base, withWrite) {
|
||||
t.Fatal("cards differing in CacheWrite compare equal")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnthropicPricesCacheWrites(t *testing.T) {
|
||||
got := CurrentRate("anthropic", "claude-sonnet-4-5", "USD")
|
||||
if got == nil || got.CacheWrite != 3.75 {
|
||||
t.Fatalf("claude-sonnet-4-5 = %+v", got)
|
||||
}
|
||||
if OfficialProviderForEndpoint("https://api.anthropic.com/v1/messages") != "anthropic" {
|
||||
t.Fatal("api.anthropic.com is not recognised as Anthropic's endpoint")
|
||||
}
|
||||
}
|
||||
-222
@@ -1,222 +0,0 @@
|
||||
package pricing
|
||||
|
||||
import (
|
||||
"net/url"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Official catalog metadata: currency, model, effective dates, documentation
|
||||
// sources, and price fingerprints. User-custom prices always win over catalog.
|
||||
|
||||
// CatalogEntry is one official list price for a model in a billing currency.
|
||||
type CatalogEntry struct {
|
||||
Provider string // deepseek | longcat | mimo | google
|
||||
Model string
|
||||
Currency string // ISO billing currency for this row
|
||||
CacheHit float64
|
||||
Input float64
|
||||
Output float64
|
||||
CacheWrite float64 // zero when the vendor bills cache writes at Input
|
||||
EffectiveFrom string // YYYY-MM-DD inclusive, empty = unbounded
|
||||
EffectiveTo string // YYYY-MM-DD exclusive, empty = unbounded
|
||||
DocURL string
|
||||
BillingMode string // payg | subscription_equivalent
|
||||
Notes string
|
||||
Fingerprint string // filled by Register
|
||||
// Peak is what the same tokens cost inside Window. Nil when the vendor
|
||||
// bills one rate around the clock, and then the fields above are the rate.
|
||||
Peak *RateCard
|
||||
Window *PeakWindow
|
||||
// CheckedOn is when a person last read DocURL and confirmed these numbers.
|
||||
// No vendor here serves prices over an API, so this date is all that stands
|
||||
// between a price change and a readout that is quietly wrong.
|
||||
CheckedOn string
|
||||
}
|
||||
|
||||
// deepseekPeak is the schedule DeepSeek publishes: peak on weekday mornings and
|
||||
// afternoons, and — since 2026-08-23 — never at a weekend. Chinese public
|
||||
// holidays bill off-peak in full, per the same page.
|
||||
var deepseekPeak = &PeakWindow{
|
||||
OffsetSeconds: beijing,
|
||||
Hours: [][2]int{{9, 12}, {14, 18}},
|
||||
WeekendOffPeak: "2026-08-23",
|
||||
HolidayOffPeak: chineseStatutoryHolidays,
|
||||
}
|
||||
|
||||
// chineseStatutoryHolidays are the State Council's 2026 public holidays, which
|
||||
// DeepSeek bills off-peak in full; extend it when the next notice is out. Make-up
|
||||
// workdays are absent: they fall on weekends, which are off-peak anyway.
|
||||
var chineseStatutoryHolidays = []string{
|
||||
// New Year's Day: Jan 1-3.
|
||||
"2026-01-01", "2026-01-02", "2026-01-03",
|
||||
// Spring Festival: Feb 15-23.
|
||||
"2026-02-15", "2026-02-16", "2026-02-17", "2026-02-18", "2026-02-19",
|
||||
"2026-02-20", "2026-02-21", "2026-02-22", "2026-02-23",
|
||||
// Qingming Festival: Apr 4-6.
|
||||
"2026-04-04", "2026-04-05", "2026-04-06",
|
||||
// Labor Day: May 1-5.
|
||||
"2026-05-01", "2026-05-02", "2026-05-03", "2026-05-04", "2026-05-05",
|
||||
// Dragon Boat Festival: Jun 19-21.
|
||||
"2026-06-19", "2026-06-20", "2026-06-21",
|
||||
// Mid-Autumn Festival: Sep 25-27.
|
||||
"2026-09-25", "2026-09-26", "2026-09-27",
|
||||
// National Day: Oct 1-7.
|
||||
"2026-10-01", "2026-10-02", "2026-10-03", "2026-10-04",
|
||||
"2026-10-05", "2026-10-06", "2026-10-07",
|
||||
}
|
||||
|
||||
// checkedOn is when the tables below were last read off the vendors' pages.
|
||||
const checkedOn = "2026-08-22"
|
||||
|
||||
// deepseekCheckedOn is when the DeepSeek rows were last read off that vendor's
|
||||
// page. Separate from checkedOn, because confirming one vendor's prices says
|
||||
// nothing about another's and a shared date would claim that it did.
|
||||
const deepseekCheckedOn = "2026-09-13"
|
||||
|
||||
// mimoCheckedOn is when the V2.6 rows were read off the vendor's own docs.
|
||||
const mimoCheckedOn = "2026-10-06"
|
||||
|
||||
// googleCheckedOn is when the Gemini rows were read off Google's pricing page.
|
||||
const googleCheckedOn = "2026-10-17"
|
||||
|
||||
// anthropicCheckedOn is when the Claude rows were read off Anthropic's pricing page.
|
||||
const anthropicCheckedOn = "2026-10-19"
|
||||
|
||||
// CatalogSourceURLs document public pricing pages.
|
||||
const (
|
||||
DocDeepSeekPricing = "https://api-docs.deepseek.com/quick_start/pricing"
|
||||
DocDeepSeekPricingCNY = "https://api-docs.deepseek.com/zh-cn/quick_start/pricing"
|
||||
DocLongCatPricingUSD = "https://longcat.chat/platform/docs/Pricing/LongCat-2.0.html"
|
||||
DocLongCatPricingCNY = "https://longcat.chat/platform/docs/zh/pricing/long-cat-2.0"
|
||||
DocMiMoPAYG = "https://mimo.mi.com/docs/price/pay-as-you-go"
|
||||
DocMiMoTokenPlan = "https://platform.xiaomimimo.com/token-plan"
|
||||
DocGeminiPricing = "https://ai.google.dev/gemini-api/docs/pricing"
|
||||
DocAnthropicPricing = "https://www.anthropic.com/pricing"
|
||||
)
|
||||
|
||||
// officialEndpointHosts maps a vendor's own API hostnames to the catalog
|
||||
// provider whose list prices apply there. Host equality is the test: a reseller
|
||||
// or proxy bills on its own terms even when it serves the same model, and an
|
||||
// entry merely named after a vendor is not evidence of the vendor's endpoint.
|
||||
var officialEndpointHosts = map[string]string{
|
||||
"api.anthropic.com": "anthropic",
|
||||
"api.deepseek.com": "deepseek",
|
||||
"api.longcat.chat": "longcat",
|
||||
"api.xiaomimimo.com": "mimo",
|
||||
"generativelanguage.googleapis.com": "google",
|
||||
"token-plan-cn.xiaomimimo.com": "mimo",
|
||||
"token-plan-sgp.xiaomimimo.com": "mimo",
|
||||
"token-plan-ams.xiaomimimo.com": "mimo",
|
||||
}
|
||||
|
||||
// OfficialProviderForEndpoint returns the catalog provider whose official
|
||||
// prices apply to baseURL, or "" when the address is not a vendor's own API.
|
||||
func OfficialProviderForEndpoint(baseURL string) string {
|
||||
u, err := url.Parse(strings.TrimSpace(baseURL))
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
return officialEndpointHosts[strings.ToLower(u.Hostname())]
|
||||
}
|
||||
|
||||
// OfficialCatalog is the built-in price book: the current generation of every
|
||||
// rate in officialRates, fingerprinted so a stored price can be recognised as
|
||||
// one of ours. User-custom prices always win over it.
|
||||
func OfficialCatalog() []CatalogEntry {
|
||||
entries := officialCatalogRows()
|
||||
for i := range entries {
|
||||
entries[i].Fingerprint = PricingFingerprint(RateCardFromCatalog(entries[i]))
|
||||
}
|
||||
return entries
|
||||
}
|
||||
|
||||
// LookupCatalog finds an official entry. billingMode empty matches payg first.
|
||||
func LookupCatalog(provider, model, currency, billingMode string) (CatalogEntry, bool) {
|
||||
provider = strings.ToLower(strings.TrimSpace(provider))
|
||||
model = strings.TrimSpace(model)
|
||||
currency = NormalizeCurrency(currency)
|
||||
billingMode = strings.TrimSpace(billingMode)
|
||||
if billingMode == "" {
|
||||
billingMode = BillingModePAYG
|
||||
}
|
||||
var fallback CatalogEntry
|
||||
foundFallback := false
|
||||
for _, e := range OfficialCatalog() {
|
||||
if e.Provider != provider || e.Model != model {
|
||||
continue
|
||||
}
|
||||
if NormalizeCurrency(e.Currency) != currency {
|
||||
continue
|
||||
}
|
||||
if e.BillingMode == billingMode {
|
||||
return e, true
|
||||
}
|
||||
if e.BillingMode == BillingModePAYG {
|
||||
fallback = e
|
||||
foundFallback = true
|
||||
}
|
||||
}
|
||||
if foundFallback {
|
||||
return fallback, true
|
||||
}
|
||||
return CatalogEntry{}, false
|
||||
}
|
||||
|
||||
// RateCardFromCatalog builds a RateCard from a catalog entry's base rate.
|
||||
func RateCardFromCatalog(e CatalogEntry) RateCard {
|
||||
return RateCard{
|
||||
CacheHit: e.CacheHit,
|
||||
Input: e.Input,
|
||||
Output: e.Output,
|
||||
CacheWrite: e.CacheWrite,
|
||||
Currency: e.Currency,
|
||||
}
|
||||
}
|
||||
|
||||
// RatesAt is what this entry charges at t: the peak card inside its window, the
|
||||
// base card everywhere else. An entry without a window charges one rate, and
|
||||
// then t does not matter.
|
||||
func (e CatalogEntry) RatesAt(t time.Time) RateCard {
|
||||
if e.Peak == nil || !e.Window.IsPeak(t) {
|
||||
return RateCardFromCatalog(e)
|
||||
}
|
||||
peak := *e.Peak
|
||||
peak.Currency = e.Currency
|
||||
return peak
|
||||
}
|
||||
|
||||
// InEffect reports whether this entry's dates cover the day t falls on.
|
||||
func (e CatalogEntry) InEffect(t time.Time) bool {
|
||||
day := t.UTC().Format("2006-01-02")
|
||||
if e.EffectiveFrom != "" && day < e.EffectiveFrom {
|
||||
return false
|
||||
}
|
||||
return e.EffectiveTo == "" || day < e.EffectiveTo
|
||||
}
|
||||
|
||||
// MatchesCatalog reports whether rates equal a known official entry.
|
||||
func MatchesCatalog(provider, model string, rates RateCard) (CatalogEntry, bool) {
|
||||
cur := NormalizeCurrency(rates.Currency)
|
||||
for _, e := range OfficialCatalog() {
|
||||
if e.Provider != strings.ToLower(strings.TrimSpace(provider)) {
|
||||
continue
|
||||
}
|
||||
if e.Model != strings.TrimSpace(model) {
|
||||
continue
|
||||
}
|
||||
if NormalizeCurrency(e.Currency) != cur {
|
||||
continue
|
||||
}
|
||||
// Either card identifies the vendor's own price. Which one a user
|
||||
// happens to have configured says nothing about when they are billing:
|
||||
// the hour decides that, and RatesAt is what reads it.
|
||||
if sameCard(RateCardFromCatalog(e), rates) {
|
||||
return e, true
|
||||
}
|
||||
if p := e.Peak; p != nil && sameCard(*p, rates) {
|
||||
return e, true
|
||||
}
|
||||
}
|
||||
return CatalogEntry{}, false
|
||||
}
|
||||
-799
@@ -1,799 +0,0 @@
|
||||
package pricing
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Billing modes.
|
||||
const (
|
||||
BillingModePAYG = "payg"
|
||||
BillingModeSubscriptionEquivalent = "subscription_equivalent"
|
||||
)
|
||||
|
||||
// Valuation basis values.
|
||||
const (
|
||||
BasisIdentity = "identity"
|
||||
BasisOfficialTable = "official_table"
|
||||
// BasisFX is retained for decoding pre-v6 persisted quotes only. New
|
||||
// quotes never create FX valuations.
|
||||
BasisFX = "fx"
|
||||
)
|
||||
|
||||
// DisplayStatus describes whether a quote can satisfy its requested display.
|
||||
// It is deliberately separate from CostComplete: a known price-book amount
|
||||
// can exist even when no requested regional valuation is available.
|
||||
const (
|
||||
DisplayStatusMatched = "matched"
|
||||
DisplayStatusFallbackOriginal = "fallback_original"
|
||||
DisplayStatusBucketed = "bucketed"
|
||||
DisplayStatusUnavailable = "unavailable"
|
||||
)
|
||||
|
||||
// Aggregate modes describe how a session/run total was formed.
|
||||
const (
|
||||
AggregateModeSingleCurrency = "single_currency"
|
||||
AggregateModeCommonValuation = "common_valuation"
|
||||
AggregateModeCurrencyBuckets = "currency_buckets"
|
||||
)
|
||||
|
||||
// Rate bands name which side of a vendor's peak window a call was billed on.
|
||||
// An aggregate is mixed when its calls fell on both sides.
|
||||
const (
|
||||
RateBandPeak = "peak"
|
||||
RateBandOffPeak = "off_peak"
|
||||
RateBandMixed = "mixed"
|
||||
)
|
||||
|
||||
// DisplayRequest identifies how a caller selected the requested presentation
|
||||
// currency. The source is runtime context only; it is never persisted as a
|
||||
// provider price or wallet fact.
|
||||
type DisplayRequest struct {
|
||||
Currency string `json:"currency,omitempty"`
|
||||
Source string `json:"source,omitempty"`
|
||||
}
|
||||
|
||||
const (
|
||||
DisplaySourceExplicit = "explicit"
|
||||
DisplaySourceWallet = "wallet"
|
||||
DisplaySourceAuto = "auto"
|
||||
)
|
||||
|
||||
func (r DisplayRequest) NormalizedCurrency() string {
|
||||
return NormalizeCurrency(r.Currency)
|
||||
}
|
||||
|
||||
// CostQuote is the host-side quoted cost for one model call (or an aggregate).
|
||||
// Original is the pricing-table currency fact; valuations hold identity or
|
||||
// official regional rate-card estimates. Selected is the display pick for the
|
||||
// current preference.
|
||||
type CostQuote struct {
|
||||
Original Money `json:"original"`
|
||||
// OriginalTotals is populated for mixed-currency aggregates. Individual
|
||||
// quotes and single-currency totals may omit it.
|
||||
OriginalTotals []Money `json:"originalTotals,omitempty"`
|
||||
Valuations map[string]Valuation `json:"valuations,omitempty"`
|
||||
Selected *Money `json:"selected,omitempty"`
|
||||
BillingMode string `json:"billingMode,omitempty"`
|
||||
Estimated bool `json:"estimated"`
|
||||
// CostComplete means usage and the price-book amount are known. It does not
|
||||
// imply that a requested display currency is available.
|
||||
CostComplete bool `json:"costComplete"`
|
||||
// Coverage is the same fact with the states a boolean cannot hold: an
|
||||
// aggregate that priced only some of its calls, and one that billed nothing.
|
||||
Coverage string `json:"coverage,omitempty"`
|
||||
// DisplayComplete means a single selected amount satisfies the display
|
||||
// request. Complete is retained as the old wire alias.
|
||||
DisplayComplete bool `json:"displayComplete"`
|
||||
Complete bool `json:"complete"`
|
||||
DisplayStatus string `json:"displayStatus,omitempty"`
|
||||
AggregateMode string `json:"aggregateMode,omitempty"`
|
||||
// ModelRef / UsageSource / PricingFingerprint identify the ledger key.
|
||||
ModelRef string `json:"modelRef,omitempty"`
|
||||
UsageSource string `json:"usageSource,omitempty"`
|
||||
PricingFingerprint string `json:"pricingFingerprint,omitempty"`
|
||||
RateDate string `json:"rateDate,omitempty"` // YYYY-MM-DD of FX used, if any
|
||||
IncompleteReason string `json:"incompleteReason,omitempty"`
|
||||
LegacyEstimate bool `json:"legacyEstimate,omitempty"`
|
||||
CatalogSource string `json:"catalogSource,omitempty"`
|
||||
// RateBand is set only for a rate confirmed as a vendor's own schedule; an
|
||||
// aggregate that includes any call without one carries none.
|
||||
RateBand string `json:"rateBand,omitempty"`
|
||||
}
|
||||
|
||||
// Valuation is one currency view of a cost fact.
|
||||
type Valuation struct {
|
||||
Money Money `json:"money"`
|
||||
Basis string `json:"basis"`
|
||||
Source string `json:"source"`
|
||||
AsOf string `json:"asOf"` // YYYY-MM-DD
|
||||
Rate *RateSnapshot `json:"rateSnapshot,omitempty"`
|
||||
Stale bool `json:"stale,omitempty"`
|
||||
}
|
||||
|
||||
// RateSnapshot records the FX observation used for a valuation.
|
||||
type RateSnapshot struct {
|
||||
Base string `json:"base"`
|
||||
Quote string `json:"quote"`
|
||||
Rate float64 `json:"rate"`
|
||||
Source string `json:"source"`
|
||||
AsOf string `json:"asOf"`
|
||||
Stale bool `json:"stale,omitempty"`
|
||||
}
|
||||
|
||||
// RateCard is the per-1M-token price used to compute original cost. It mirrors
|
||||
// provider.Pricing without importing that package (billing is a leaf).
|
||||
type RateCard struct {
|
||||
CacheHit float64 // per 1M cached prompt tokens
|
||||
Input float64 // per 1M uncached prompt tokens
|
||||
Output float64 // per 1M completion tokens
|
||||
// CacheWrite is per 1M tokens written to the cache; zero bills them at Input.
|
||||
CacheWrite float64
|
||||
Currency string // ISO or symbol; normalized on quote
|
||||
}
|
||||
|
||||
// UsageTokens is the token breakdown needed for cost. Mirrors provider.Usage
|
||||
// fields without importing provider.
|
||||
type UsageTokens struct {
|
||||
PromptTokens int
|
||||
CompletionTokens int
|
||||
CacheHitTokens int
|
||||
CacheMissTokens int
|
||||
CacheWriteTokens int
|
||||
CacheWriteBilledTokens float64
|
||||
Estimated bool
|
||||
}
|
||||
|
||||
// QuoteInput drives a single CostQuote computation.
|
||||
type QuoteInput struct {
|
||||
Usage UsageTokens
|
||||
Rates RateCard
|
||||
OccurredAt time.Time
|
||||
DisplayCurrency string // compatibility alias for Display.Currency
|
||||
Display DisplayRequest // runtime display request; empty means auto
|
||||
BillingMode string
|
||||
ModelRef string
|
||||
UsageSource string
|
||||
CatalogSource string
|
||||
PricingFingerprint string
|
||||
// ProviderKind is deepseek|longcat|mimo|… for official dual-table lookup.
|
||||
// When empty, ModelRef and Rates are used to infer a catalog match.
|
||||
ProviderKind string
|
||||
// ModelID is the bare model id (e.g. deepseek-v4-flash). Empty → parse ModelRef.
|
||||
ModelID string
|
||||
}
|
||||
|
||||
// OriginalCostAmount computes the fixed-point original cost from rates + tokens.
|
||||
// Mirrors the historic provider.Pricing.Cost semantics.
|
||||
func OriginalCostAmount(rates RateCard, u UsageTokens) Amount {
|
||||
hit := u.CacheHitTokens
|
||||
miss := u.CacheMissTokens
|
||||
if hit+miss == 0 && u.PromptTokens > 0 {
|
||||
miss = u.PromptTokens
|
||||
} else if miss == 0 && hit > 0 && u.PromptTokens > hit {
|
||||
miss = u.PromptTokens - hit
|
||||
}
|
||||
write := u.CacheWriteTokens
|
||||
write = max(write, 0)
|
||||
write = min(write, miss)
|
||||
billedWrite := 0.0
|
||||
if write > 0 {
|
||||
billedWrite = u.CacheWriteBilledTokens
|
||||
if billedWrite <= 0 {
|
||||
billedWrite = float64(write)
|
||||
}
|
||||
}
|
||||
writeRate := rates.Input
|
||||
if rates.CacheWrite > 0 {
|
||||
writeRate = rates.CacheWrite
|
||||
}
|
||||
total := (float64(hit)*rates.CacheHit +
|
||||
float64(miss-write)*rates.Input +
|
||||
billedWrite*writeRate +
|
||||
float64(u.CompletionTokens)*rates.Output) / 1e6
|
||||
return NewAmountFromFloat(total)
|
||||
}
|
||||
|
||||
// PricingFingerprint hashes a rate card for ledger keys.
|
||||
func PricingFingerprint(rates RateCard) string {
|
||||
cur := NormalizeCurrency(rates.Currency)
|
||||
raw := fmt.Sprintf("%s|%.12g|%.12g|%.12g", cur, rates.CacheHit, rates.Input, rates.Output)
|
||||
if rates.CacheWrite != 0 {
|
||||
// Appended only when set, so every card stored before the field keeps its key.
|
||||
raw += fmt.Sprintf("|w%.12g", rates.CacheWrite)
|
||||
}
|
||||
sum := sha256.Sum256([]byte(raw))
|
||||
return hex.EncodeToString(sum[:8])
|
||||
}
|
||||
|
||||
// BuildQuote computes original cost and CNY/USD valuations.
|
||||
func BuildQuote(in QuoteInput) CostQuote {
|
||||
state := newQuoteBuildState(in)
|
||||
for _, target := range []string{"CNY", "USD"} {
|
||||
state.addTargetValuation(target)
|
||||
}
|
||||
state.selectDisplay()
|
||||
return state.quote
|
||||
}
|
||||
|
||||
type quoteBuildState struct {
|
||||
input QuoteInput
|
||||
currency string
|
||||
amount Amount
|
||||
occurred time.Time
|
||||
official CatalogEntry
|
||||
isOfficial bool
|
||||
valuationFailures map[string]string
|
||||
quote CostQuote
|
||||
}
|
||||
|
||||
func newQuoteBuildState(in QuoteInput) *quoteBuildState {
|
||||
currency := NormalizeCurrency(in.Rates.Currency)
|
||||
if currency == "" {
|
||||
currency = "CNY"
|
||||
}
|
||||
mode := strings.TrimSpace(in.BillingMode)
|
||||
if mode == "" {
|
||||
mode = BillingModePAYG
|
||||
}
|
||||
occurred := in.OccurredAt
|
||||
if occurred.IsZero() {
|
||||
occurred = time.Now().UTC()
|
||||
}
|
||||
fingerprint := strings.TrimSpace(in.PricingFingerprint)
|
||||
if fingerprint == "" {
|
||||
fingerprint = PricingFingerprint(in.Rates)
|
||||
}
|
||||
amount := OriginalCostAmount(in.Rates, in.Usage)
|
||||
q := CostQuote{
|
||||
Original: MoneyOf(amount, currency),
|
||||
Valuations: map[string]Valuation{},
|
||||
BillingMode: mode,
|
||||
Estimated: true,
|
||||
CostComplete: true,
|
||||
Coverage: CoverageComplete,
|
||||
DisplayComplete: true,
|
||||
Complete: true,
|
||||
DisplayStatus: DisplayStatusMatched,
|
||||
AggregateMode: AggregateModeSingleCurrency,
|
||||
ModelRef: strings.TrimSpace(in.ModelRef),
|
||||
UsageSource: strings.TrimSpace(in.UsageSource),
|
||||
PricingFingerprint: fingerprint,
|
||||
CatalogSource: strings.TrimSpace(in.CatalogSource),
|
||||
}
|
||||
q.Valuations[currency] = Valuation{
|
||||
Money: q.Original, Basis: BasisIdentity,
|
||||
Source: firstNonEmpty(in.CatalogSource, "rate_card"),
|
||||
AsOf: occurred.UTC().Format("2006-01-02"),
|
||||
}
|
||||
state := "eBuildState{
|
||||
input: in, currency: currency, amount: amount, occurred: occurred,
|
||||
valuationFailures: map[string]string{}, quote: q,
|
||||
}
|
||||
if !usageHasFacts(in.Usage) {
|
||||
state.markIncomplete("missing_price_or_usage")
|
||||
}
|
||||
state.matchOfficialCatalog()
|
||||
state.applyOfficialSchedule()
|
||||
return state
|
||||
}
|
||||
|
||||
// applyOfficialSchedule re-prices the turn at the vendor's rate for the hour it
|
||||
// actually happened. Only a rate confirmed as the vendor's own earns this: a
|
||||
// custom price is the user's own arrangement, and projecting someone else's
|
||||
// peak schedule onto it would invent a number nobody published.
|
||||
func (s *quoteBuildState) applyOfficialSchedule() {
|
||||
if !s.isOfficial || s.official.Peak == nil {
|
||||
return
|
||||
}
|
||||
s.quote.RateBand = RateBandOffPeak
|
||||
if s.official.Window.IsPeak(s.occurred) {
|
||||
s.quote.RateBand = RateBandPeak
|
||||
}
|
||||
rates := s.official.RatesAt(s.occurred)
|
||||
if rates.CacheHit == s.input.Rates.CacheHit && rates.Input == s.input.Rates.Input && rates.Output == s.input.Rates.Output {
|
||||
return
|
||||
}
|
||||
s.amount = OriginalCostAmount(rates, s.input.Usage)
|
||||
s.quote.Original = MoneyOf(s.amount, s.currency)
|
||||
s.quote.PricingFingerprint = PricingFingerprint(rates)
|
||||
s.quote.Valuations[s.currency] = Valuation{
|
||||
Money: s.quote.Original, Basis: BasisOfficialTable,
|
||||
Source: s.official.DocURL,
|
||||
AsOf: s.occurred.UTC().Format("2006-01-02"),
|
||||
}
|
||||
}
|
||||
|
||||
func quoteDisplayRequest(in QuoteInput) DisplayRequest {
|
||||
request := in.Display
|
||||
if request.Currency == "" && in.DisplayCurrency != "" {
|
||||
request.Currency = in.DisplayCurrency
|
||||
if request.Source == "" {
|
||||
request.Source = DisplaySourceExplicit
|
||||
}
|
||||
}
|
||||
if request.Source == "" {
|
||||
request.Source = DisplaySourceAuto
|
||||
}
|
||||
return request
|
||||
}
|
||||
|
||||
func usageHasFacts(u UsageTokens) bool {
|
||||
return u.PromptTokens > 0 || u.CompletionTokens > 0 || u.CacheHitTokens > 0 ||
|
||||
u.CacheMissTokens > 0 || u.CacheWriteTokens > 0 || u.CacheWriteBilledTokens > 0
|
||||
}
|
||||
|
||||
func (s *quoteBuildState) matchOfficialCatalog() {
|
||||
providerKind, modelID := resolveCatalogIdentity(s.input)
|
||||
s.official, s.isOfficial = MatchesCatalog(providerKind, modelID, s.input.Rates)
|
||||
if !s.isOfficial && providerKind != "" && modelID != "" {
|
||||
rates := s.input.Rates
|
||||
rates.Currency = s.currency
|
||||
s.official, s.isOfficial = MatchesCatalog(providerKind, modelID, rates)
|
||||
}
|
||||
if s.isOfficial && s.quote.CatalogSource == "" {
|
||||
s.quote.CatalogSource = s.official.DocURL
|
||||
}
|
||||
}
|
||||
|
||||
func (s *quoteBuildState) addTargetValuation(target string) {
|
||||
if target == s.currency {
|
||||
return
|
||||
}
|
||||
if s.addOfficialValuation(target) {
|
||||
return
|
||||
}
|
||||
// Runtime FX has intentionally been removed. A custom price that does not
|
||||
// match the official catalog remains in its original currency.
|
||||
s.valuationFailures[target] = "display_unavailable"
|
||||
}
|
||||
|
||||
func (s *quoteBuildState) addOfficialValuation(target string) bool {
|
||||
if !s.isOfficial {
|
||||
return false
|
||||
}
|
||||
peer, ok := LookupCatalog(
|
||||
s.official.Provider, s.official.Model, target, s.official.BillingMode,
|
||||
)
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
s.quote.Valuations[target] = Valuation{
|
||||
Money: MoneyOf(OriginalCostAmount(peer.RatesAt(s.occurred), s.input.Usage), target),
|
||||
Basis: BasisOfficialTable, Source: peer.DocURL,
|
||||
AsOf: s.occurred.UTC().Format("2006-01-02"),
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (s *quoteBuildState) selectDisplay() {
|
||||
if !s.quote.CostComplete {
|
||||
s.quote.Selected = nil
|
||||
s.quote.DisplayComplete = false
|
||||
s.quote.Complete = false
|
||||
s.quote.DisplayStatus = DisplayStatusUnavailable
|
||||
return
|
||||
}
|
||||
display := quoteDisplayRequest(s.input).NormalizedCurrency()
|
||||
if display == "" {
|
||||
selected := s.quote.Original
|
||||
s.quote.Selected = &selected
|
||||
return
|
||||
}
|
||||
if valuation, ok := s.quote.Valuations[display]; ok {
|
||||
selected := valuation.Money
|
||||
s.quote.Selected = &selected
|
||||
return
|
||||
}
|
||||
// The original price-book amount is still a valid cost fact. Keep it as a
|
||||
// visible fallback and distinguish the display mismatch from no pricing.
|
||||
selected := s.quote.Original
|
||||
s.quote.Selected = &selected
|
||||
s.quote.DisplayComplete = false
|
||||
s.quote.Complete = false
|
||||
s.quote.DisplayStatus = DisplayStatusFallbackOriginal
|
||||
s.quote.IncompleteReason = firstNonEmpty(s.valuationFailures[display], "display_unavailable")
|
||||
}
|
||||
|
||||
func (s *quoteBuildState) markIncomplete(reason string) {
|
||||
s.quote.CostComplete = false
|
||||
s.quote.Coverage = CoverageIncomplete
|
||||
s.quote.DisplayComplete = false
|
||||
s.quote.Complete = false
|
||||
s.quote.DisplayStatus = DisplayStatusUnavailable
|
||||
if s.quote.IncompleteReason == "" {
|
||||
s.quote.IncompleteReason = reason
|
||||
}
|
||||
}
|
||||
|
||||
// SelectForDisplay returns the money for a display currency preference without
|
||||
// recomputing rates. Empty display returns original.
|
||||
func (q CostQuote) SelectForDisplay(display string) (Money, bool) {
|
||||
display = NormalizeCurrency(display)
|
||||
if display == "" {
|
||||
return q.Original, true
|
||||
}
|
||||
if v, ok := q.Valuations[display]; ok {
|
||||
return v.Money, true
|
||||
}
|
||||
if NormalizeCurrency(q.Original.Currency) == display {
|
||||
return q.Original, true
|
||||
}
|
||||
return Money{}, false
|
||||
}
|
||||
|
||||
// WithSelected returns a copy with Selected set for display.
|
||||
func (q CostQuote) WithSelected(display string) CostQuote {
|
||||
out := q
|
||||
if m, ok := q.SelectForDisplay(display); ok {
|
||||
out.Selected = &m
|
||||
if quoteHasCompleteCostFact(q) {
|
||||
out.Complete = true
|
||||
out.IncompleteReason = ""
|
||||
}
|
||||
} else {
|
||||
out.Selected = nil
|
||||
out.Complete = false
|
||||
if out.IncompleteReason == "" {
|
||||
out.IncompleteReason = "display_unavailable"
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// LegacyCostFloat returns the selected (or original) amount as float64 for
|
||||
// compatibility aliases cost / costUsd / total_cost.
|
||||
func (q CostQuote) LegacyCostFloat() float64 {
|
||||
if q.Selected != nil {
|
||||
return q.Selected.Float64()
|
||||
}
|
||||
return q.Original.Float64()
|
||||
}
|
||||
|
||||
// LegacyCurrencySymbol returns a display symbol for the selected/original currency.
|
||||
func (q CostQuote) LegacyCurrencySymbol() string {
|
||||
if q.Selected != nil && q.Selected.Currency != "" {
|
||||
return CurrencySymbol(q.Selected.Currency)
|
||||
}
|
||||
return CurrencySymbol(q.Original.Currency)
|
||||
}
|
||||
|
||||
// LegacyCurrencyCode returns the ISO code for selected/original.
|
||||
func (q CostQuote) LegacyCurrencyCode() string {
|
||||
if q.Selected != nil && q.Selected.Currency != "" {
|
||||
return NormalizeCurrency(q.Selected.Currency)
|
||||
}
|
||||
return NormalizeCurrency(q.Original.Currency)
|
||||
}
|
||||
|
||||
// AggregateQuotes combines occurrence-time quotes into a session/run total.
|
||||
// Every entry must have the requested valuation; partial totals stay unavailable.
|
||||
func AggregateQuotes(quotes []CostQuote, display string) CostQuote {
|
||||
if len(quotes) == 0 {
|
||||
return emptyAggregate(NormalizeCurrency(display))
|
||||
}
|
||||
accumulator := newQuoteAccumulator(display)
|
||||
for _, quote := range quotes {
|
||||
accumulator.add(quote)
|
||||
}
|
||||
return accumulator.finish()
|
||||
}
|
||||
|
||||
type quoteAccumulator struct {
|
||||
out CostQuote
|
||||
display string
|
||||
totals map[string]Amount
|
||||
originalCurrency string
|
||||
originalTotal Amount
|
||||
originalTotals map[string]Amount
|
||||
originalComplete bool
|
||||
coverage string
|
||||
displayComplete bool
|
||||
modes map[string]struct{}
|
||||
rateBands map[string]struct{}
|
||||
unbanded bool
|
||||
}
|
||||
|
||||
func emptyAggregate(display string) CostQuote {
|
||||
out := CostQuote{
|
||||
Original: MoneyOf(Zero, display), Valuations: map[string]Valuation{}, Estimated: true,
|
||||
CostComplete: false, DisplayComplete: false, Complete: false,
|
||||
Coverage: CoverageNone,
|
||||
DisplayStatus: DisplayStatusUnavailable, AggregateMode: AggregateModeSingleCurrency,
|
||||
IncompleteReason: "no_usage",
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func newQuoteAccumulator(display string) *quoteAccumulator {
|
||||
return "eAccumulator{
|
||||
out: CostQuote{Valuations: map[string]Valuation{}, Estimated: true},
|
||||
display: NormalizeCurrency(display), totals: map[string]Amount{}, originalTotals: map[string]Amount{},
|
||||
originalComplete: true, coverage: CoverageNone, displayComplete: true,
|
||||
modes: map[string]struct{}{}, rateBands: map[string]struct{}{},
|
||||
}
|
||||
}
|
||||
|
||||
func (a *quoteAccumulator) add(quote CostQuote) {
|
||||
quote = NormalizeQuote(quote)
|
||||
a.coverage = FoldCoverage(a.coverage, quote.Coverage)
|
||||
if quote.Coverage != CoverageComplete && a.out.IncompleteReason == "" {
|
||||
a.out.IncompleteReason = quote.IncompleteReason
|
||||
}
|
||||
a.out.LegacyEstimate = a.out.LegacyEstimate || quote.LegacyEstimate
|
||||
if quote.BillingMode != "" {
|
||||
a.modes[quote.BillingMode] = struct{}{}
|
||||
}
|
||||
a.addRateBand(quote.RateBand)
|
||||
if quote.RateDate > a.out.RateDate {
|
||||
a.out.RateDate = quote.RateDate
|
||||
}
|
||||
originalCurrency := NormalizeCurrency(quote.Original.Currency)
|
||||
if len(quote.OriginalTotals) > 0 {
|
||||
for _, original := range quote.OriginalTotals {
|
||||
a.addOriginal(original, NormalizeCurrency(original.Currency))
|
||||
}
|
||||
} else {
|
||||
a.addOriginal(quote.Original, originalCurrency)
|
||||
}
|
||||
if a.display != "" {
|
||||
_, found := quote.Valuations[a.display]
|
||||
a.displayComplete = a.displayComplete && (found || originalCurrency == a.display)
|
||||
}
|
||||
originalIncluded := false
|
||||
for code, valuation := range quote.Valuations {
|
||||
code = NormalizeCurrency(code)
|
||||
if code == originalCurrency {
|
||||
originalIncluded = true
|
||||
}
|
||||
a.addValuation(code, valuation)
|
||||
}
|
||||
if originalCurrency != "" && !originalIncluded {
|
||||
a.addValuation(originalCurrency, Valuation{
|
||||
Money: quote.Original, Basis: BasisIdentity, Source: "aggregate",
|
||||
AsOf: time.Now().UTC().Format("2006-01-02"),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func (a *quoteAccumulator) addRateBand(band string) {
|
||||
switch band {
|
||||
case RateBandPeak, RateBandOffPeak:
|
||||
a.rateBands[band] = struct{}{}
|
||||
case RateBandMixed:
|
||||
a.rateBands[RateBandPeak], a.rateBands[RateBandOffPeak] = struct{}{}, struct{}{}
|
||||
default:
|
||||
a.unbanded = true
|
||||
}
|
||||
}
|
||||
|
||||
func (a *quoteAccumulator) rateBand() string {
|
||||
if a.unbanded {
|
||||
return ""
|
||||
}
|
||||
switch len(a.rateBands) {
|
||||
case 1:
|
||||
for band := range a.rateBands {
|
||||
return band
|
||||
}
|
||||
case 2:
|
||||
return RateBandMixed
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (a *quoteAccumulator) addOriginal(original Money, currency string) {
|
||||
if currency != "" {
|
||||
a.originalTotals[currency] = a.originalTotals[currency].Add(original.AmountValue())
|
||||
}
|
||||
if a.originalCurrency == "" {
|
||||
a.originalCurrency = currency
|
||||
a.originalTotal = original.AmountValue()
|
||||
return
|
||||
}
|
||||
if currency != a.originalCurrency {
|
||||
a.originalComplete = false
|
||||
return
|
||||
}
|
||||
a.originalTotal = a.originalTotal.Add(original.AmountValue())
|
||||
}
|
||||
|
||||
func (a *quoteAccumulator) addValuation(code string, valuation Valuation) {
|
||||
if code == "" {
|
||||
return
|
||||
}
|
||||
a.totals[code] = a.totals[code].Add(valuation.Money.AmountValue())
|
||||
current, found := a.out.Valuations[code]
|
||||
if !found {
|
||||
current = valuation
|
||||
}
|
||||
current.Money = MoneyOf(a.totals[code], code)
|
||||
current.Stale = current.Stale || valuation.Stale
|
||||
a.out.Valuations[code] = current
|
||||
}
|
||||
|
||||
// costFactsComplete reads the accumulated coverage rather than tracking the
|
||||
// same fact a second time: two answers to one question drift.
|
||||
func (a *quoteAccumulator) costFactsComplete() bool {
|
||||
return a.coverage == CoverageComplete
|
||||
}
|
||||
|
||||
func (a *quoteAccumulator) finish() CostQuote {
|
||||
a.out.Coverage = a.coverage
|
||||
a.out.CostComplete = a.costFactsComplete()
|
||||
if a.originalComplete && a.originalCurrency != "" {
|
||||
a.out.Original = MoneyOf(a.originalTotal, a.originalCurrency)
|
||||
a.syncOriginalValuation()
|
||||
} else {
|
||||
a.out.Original = Money{Amount: "0"}
|
||||
}
|
||||
if len(a.originalTotals) > 1 {
|
||||
codes := make([]string, 0, len(a.originalTotals))
|
||||
for code := range a.originalTotals {
|
||||
codes = append(codes, code)
|
||||
}
|
||||
slices.Sort(codes)
|
||||
for _, code := range codes {
|
||||
a.out.OriginalTotals = append(a.out.OriginalTotals, MoneyOf(a.originalTotals[code], code))
|
||||
}
|
||||
}
|
||||
a.out.RateBand = a.rateBand()
|
||||
if len(a.modes) == 1 {
|
||||
for mode := range a.modes {
|
||||
a.out.BillingMode = mode
|
||||
}
|
||||
}
|
||||
if a.display == "" && a.originalComplete && a.costFactsComplete() {
|
||||
selected := a.out.Original
|
||||
a.out.Selected = &selected
|
||||
a.out.DisplayComplete = true
|
||||
a.out.Complete = true
|
||||
a.out.DisplayStatus = DisplayStatusMatched
|
||||
a.out.AggregateMode = AggregateModeSingleCurrency
|
||||
return a.out
|
||||
}
|
||||
valuation, found := a.out.Valuations[a.display]
|
||||
if found && a.displayComplete && a.costFactsComplete() {
|
||||
selected := valuation.Money
|
||||
a.out.Selected = &selected
|
||||
a.out.DisplayComplete = true
|
||||
a.out.Complete = true
|
||||
a.out.DisplayStatus = DisplayStatusMatched
|
||||
if len(a.originalTotals) > 1 {
|
||||
a.out.AggregateMode = AggregateModeCommonValuation
|
||||
} else {
|
||||
a.out.AggregateMode = AggregateModeSingleCurrency
|
||||
}
|
||||
return a.out
|
||||
}
|
||||
a.out.DisplayComplete = false
|
||||
a.out.Complete = false
|
||||
if !a.costFactsComplete() {
|
||||
a.out.DisplayStatus = DisplayStatusUnavailable
|
||||
a.out.IncompleteReason = firstNonEmpty(a.out.IncompleteReason, "incomplete_cost_fact")
|
||||
} else if a.originalComplete && a.originalCurrency != "" {
|
||||
selected := a.out.Original
|
||||
a.out.Selected = &selected
|
||||
a.out.DisplayStatus = DisplayStatusFallbackOriginal
|
||||
a.out.AggregateMode = AggregateModeSingleCurrency
|
||||
a.out.IncompleteReason = firstNonEmpty(a.out.IncompleteReason, "display_unavailable")
|
||||
} else {
|
||||
a.out.DisplayStatus = DisplayStatusBucketed
|
||||
a.out.AggregateMode = AggregateModeCurrencyBuckets
|
||||
a.out.IncompleteReason = firstNonEmpty(a.out.IncompleteReason, "currency_buckets")
|
||||
}
|
||||
return a.out
|
||||
}
|
||||
|
||||
// NormalizeQuote fills fields introduced after the first CostQuote wire shape.
|
||||
// It is used at every persistence and transport boundary so old telemetry and
|
||||
// old eventwire payloads remain safe without inventing a new amount.
|
||||
func NormalizeQuote(q CostQuote) CostQuote {
|
||||
if !q.CostComplete && !q.DisplayComplete && q.Complete {
|
||||
q.CostComplete = true
|
||||
q.DisplayComplete = true
|
||||
}
|
||||
if q.DisplayStatus != "" && q.DisplayStatus != DisplayStatusMatched && q.DisplayStatus != DisplayStatusFallbackOriginal && q.DisplayStatus != DisplayStatusBucketed && q.DisplayStatus != DisplayStatusUnavailable {
|
||||
q.DisplayStatus = ""
|
||||
}
|
||||
if q.RateBand != RateBandPeak && q.RateBand != RateBandOffPeak && q.RateBand != RateBandMixed {
|
||||
q.RateBand = ""
|
||||
}
|
||||
if q.AggregateMode != "" && q.AggregateMode != AggregateModeSingleCurrency && q.AggregateMode != AggregateModeCommonValuation && q.AggregateMode != AggregateModeCurrencyBuckets {
|
||||
q.AggregateMode = ""
|
||||
}
|
||||
if q.DisplayStatus == "" {
|
||||
switch {
|
||||
case q.Complete:
|
||||
q.DisplayStatus = DisplayStatusMatched
|
||||
case q.Original.Currency != "" && q.Original.Amount != "" && q.IncompleteReason != "no_price":
|
||||
q.DisplayStatus = DisplayStatusFallbackOriginal
|
||||
default:
|
||||
q.DisplayStatus = DisplayStatusUnavailable
|
||||
}
|
||||
}
|
||||
if q.AggregateMode == "" {
|
||||
if len(q.OriginalTotals) > 1 {
|
||||
q.AggregateMode = AggregateModeCurrencyBuckets
|
||||
} else if q.Selected != nil {
|
||||
q.AggregateMode = AggregateModeSingleCurrency
|
||||
}
|
||||
}
|
||||
if !ValidCoverage(q.Coverage) {
|
||||
q.Coverage = CoverageIncomplete
|
||||
if quoteHasCompleteCostFact(q) {
|
||||
q.Coverage = CoverageComplete
|
||||
}
|
||||
}
|
||||
q.Complete = q.DisplayComplete
|
||||
return q
|
||||
}
|
||||
|
||||
// quoteHasCompleteCostFact separates a known original-currency charge from a
|
||||
// display-only valuation failure must not poison an exact original total,
|
||||
// while unpriced and unrecoverable legacy records stay incomplete in every
|
||||
// display currency.
|
||||
func quoteHasCompleteCostFact(q CostQuote) bool {
|
||||
if q.CostComplete {
|
||||
return true
|
||||
}
|
||||
switch q.IncompleteReason {
|
||||
case "no_price", "missing_price_or_usage", "legacy_unrecoverable",
|
||||
"legacy_wiped_or_zero", "legacy_invalid_amount", "mixed_original_currencies",
|
||||
"incomplete_cost_fact":
|
||||
return false
|
||||
}
|
||||
if q.Complete {
|
||||
return true
|
||||
}
|
||||
currency := NormalizeCurrency(q.Original.Currency)
|
||||
if currency == "" {
|
||||
return false
|
||||
}
|
||||
valuation, ok := q.Valuations[currency]
|
||||
return ok && SameCurrency(valuation.Money.Currency, currency)
|
||||
}
|
||||
|
||||
func (a *quoteAccumulator) syncOriginalValuation() {
|
||||
valuation, found := a.out.Valuations[a.originalCurrency]
|
||||
if !found {
|
||||
valuation = Valuation{
|
||||
Basis: BasisIdentity, Source: "aggregate",
|
||||
AsOf: time.Now().UTC().Format("2006-01-02"),
|
||||
}
|
||||
}
|
||||
valuation.Money = a.out.Original
|
||||
a.out.Valuations[a.originalCurrency] = valuation
|
||||
}
|
||||
|
||||
func firstNonEmpty(values ...string) string {
|
||||
for _, v := range values {
|
||||
if strings.TrimSpace(v) != "" {
|
||||
return strings.TrimSpace(v)
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// resolveCatalogIdentity names which official table, if any, may price a quote.
|
||||
// The vendor comes only from ProviderKind, which the caller reads off the
|
||||
// endpoint's host; guessing it from the entry's name or the model id would bill
|
||||
// a relay as the vendor it resells. Unnamed means no official table — the user's
|
||||
// own rate card, which is right rather than merely plausible.
|
||||
func resolveCatalogIdentity(in QuoteInput) (provider, model string) {
|
||||
provider = strings.ToLower(strings.TrimSpace(in.ProviderKind))
|
||||
model = strings.TrimSpace(in.ModelID)
|
||||
if model == "" {
|
||||
ref := strings.TrimSpace(in.ModelRef)
|
||||
if i := strings.LastIndex(ref, "/"); i >= 0 && i+1 < len(ref) {
|
||||
model = ref[i+1:]
|
||||
} else {
|
||||
model = ref
|
||||
}
|
||||
}
|
||||
return provider, model
|
||||
}
|
||||
-358
@@ -1,358 +0,0 @@
|
||||
// rates.go — every rate this project has shipped for a vendor's model, and the
|
||||
// two questions that asks: what it costs now, and what it used to cost.
|
||||
package pricing
|
||||
|
||||
import (
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// rateGeneration is one published pair. Peak is nil for a generation the vendor
|
||||
// billed at a single rate around the clock.
|
||||
type rateGeneration struct {
|
||||
Base RateCard
|
||||
Peak *RateCard
|
||||
// From is the first day (YYYY-MM-DD, UTC) this generation is billed; empty
|
||||
// means from when it shipped. An announced change is appended with its date.
|
||||
From string
|
||||
}
|
||||
|
||||
// today is the day a dated generation is read against; a test replaces it.
|
||||
var today = func() string { return time.Now().UTC().Format("2006-01-02") }
|
||||
|
||||
// currentIndex is the generation in force today: the last one whose date has
|
||||
// come. Rows with no dated generation answer with their last, as before.
|
||||
func (r officialRateRow) currentIndex() int {
|
||||
day := today()
|
||||
for i := len(r.Rates) - 1; i > 0; i-- {
|
||||
if r.Rates[i].From == "" || r.Rates[i].From <= day {
|
||||
return i
|
||||
}
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// officialRateRow is one vendor's model, in one currency, under one billing
|
||||
// mode, with every rate this project has shipped for it.
|
||||
type officialRateRow struct {
|
||||
Provider string
|
||||
Model string
|
||||
Currency string
|
||||
BillingMode string
|
||||
Window *PeakWindow
|
||||
DocURL string
|
||||
Notes string
|
||||
CheckedOn string
|
||||
// Rates is oldest first; the last entry is what this costs today.
|
||||
Rates []rateGeneration
|
||||
}
|
||||
|
||||
// officialRates is the price book. A price change appends a generation rather
|
||||
// than editing one: that is what leaves the superseded rate where a stored price
|
||||
// can still be recognised as this project's own default instead of a number its
|
||||
// user chose, which is the difference between refreshing one and losing it.
|
||||
var officialRates = []officialRateRow{
|
||||
{
|
||||
Provider: "deepseek", Model: "deepseek-flash", Currency: "CNY",
|
||||
BillingMode: BillingModePAYG, Window: deepseekPeak, DocURL: DocDeepSeekPricing, CheckedOn: deepseekCheckedOn,
|
||||
Rates: []rateGeneration{
|
||||
{Base: RateCard{CacheHit: 0.02, Input: 1, Output: 2, Currency: "CNY"}},
|
||||
{Base: RateCard{CacheHit: 0.05, Input: 1.5, Output: 4.5, Currency: "CNY"},
|
||||
Peak: &RateCard{CacheHit: 0.10, Input: 3.0, Output: 9.0, Currency: "CNY"}},
|
||||
{Base: RateCard{CacheHit: 0.02, Input: 1, Output: 4, Currency: "CNY"},
|
||||
Peak: &RateCard{CacheHit: 0.04, Input: 2, Output: 8, Currency: "CNY"}},
|
||||
},
|
||||
},
|
||||
{
|
||||
Provider: "deepseek", Model: "deepseek-flash", Currency: "USD",
|
||||
BillingMode: BillingModePAYG, Window: deepseekPeak, DocURL: DocDeepSeekPricing, CheckedOn: deepseekCheckedOn,
|
||||
Rates: []rateGeneration{
|
||||
{Base: RateCard{CacheHit: 0.0028, Input: 0.14, Output: 0.28, Currency: "USD"}},
|
||||
{Base: RateCard{CacheHit: 0.007, Input: 0.22, Output: 0.66, Currency: "USD"},
|
||||
Peak: &RateCard{CacheHit: 0.014, Input: 0.44, Output: 1.32, Currency: "USD"}},
|
||||
{Base: RateCard{CacheHit: 0.003, Input: 0.15, Output: 0.6, Currency: "USD"},
|
||||
Peak: &RateCard{CacheHit: 0.006, Input: 0.3, Output: 1.2, Currency: "USD"}},
|
||||
},
|
||||
},
|
||||
{
|
||||
Provider: "deepseek", Model: "deepseek-v4-pro", Currency: "CNY",
|
||||
BillingMode: BillingModePAYG, Window: deepseekPeak, DocURL: DocDeepSeekPricing, CheckedOn: deepseekCheckedOn,
|
||||
Rates: []rateGeneration{
|
||||
{Base: RateCard{CacheHit: 0.025, Input: 3, Output: 6, Currency: "CNY"}},
|
||||
{Base: RateCard{CacheHit: 0.15, Input: 4.5, Output: 13.5, Currency: "CNY"},
|
||||
Peak: &RateCard{CacheHit: 0.30, Input: 9.0, Output: 27.0, Currency: "CNY"}},
|
||||
},
|
||||
},
|
||||
{
|
||||
Provider: "deepseek", Model: "deepseek-v4-pro", Currency: "USD",
|
||||
BillingMode: BillingModePAYG, Window: deepseekPeak, DocURL: DocDeepSeekPricing, CheckedOn: deepseekCheckedOn,
|
||||
Rates: []rateGeneration{
|
||||
{Base: RateCard{CacheHit: 0.003625, Input: 0.435, Output: 0.87, Currency: "USD"}},
|
||||
{Base: RateCard{CacheHit: 0.022, Input: 0.66, Output: 1.98, Currency: "USD"},
|
||||
Peak: &RateCard{CacheHit: 0.044, Input: 1.32, Output: 3.96, Currency: "USD"}},
|
||||
},
|
||||
},
|
||||
|
||||
// LongCat's launch discount, for which the vendor publishes no end date. List
|
||||
// price is CNY 0.10/5/20, USD 0.015/0.75/2.95 — what these revert to, and why
|
||||
// CheckedOn earns its place here.
|
||||
{
|
||||
Provider: "longcat", Model: "LongCat-2.0", Currency: "CNY",
|
||||
BillingMode: BillingModePAYG, DocURL: DocLongCatPricingCNY, Notes: "launch_discount_no_end_date", CheckedOn: checkedOn,
|
||||
Rates: []rateGeneration{{Base: RateCard{CacheHit: 0.04, Input: 2, Output: 8, Currency: "CNY"}}},
|
||||
},
|
||||
{
|
||||
Provider: "longcat", Model: "LongCat-2.0", Currency: "USD",
|
||||
BillingMode: BillingModePAYG, DocURL: DocLongCatPricingUSD, Notes: "launch_discount_no_end_date", CheckedOn: checkedOn,
|
||||
Rates: []rateGeneration{{Base: RateCard{CacheHit: 0.006, Input: 0.30, Output: 1.20, Currency: "USD"}}},
|
||||
},
|
||||
|
||||
// Google: the two flash models double on 2027-01-01, hence the dated generation.
|
||||
// gemini-3.1-pro-preview is the rate up to 200k prompt tokens; gemini-4-argon has no price.
|
||||
{
|
||||
Provider: "google", Model: "gemini-3.8-flash", Currency: "USD",
|
||||
BillingMode: BillingModePAYG, DocURL: DocGeminiPricing, CheckedOn: googleCheckedOn,
|
||||
Rates: []rateGeneration{
|
||||
{Base: RateCard{CacheHit: 0.075, Input: 0.75, Output: 3.75, Currency: "USD"}},
|
||||
{From: "2027-01-01", Base: RateCard{CacheHit: 0.15, Input: 1.50, Output: 7.50, Currency: "USD"}},
|
||||
},
|
||||
},
|
||||
{
|
||||
Provider: "google", Model: "gemini-3.6-flash", Currency: "USD",
|
||||
BillingMode: BillingModePAYG, DocURL: DocGeminiPricing, CheckedOn: googleCheckedOn,
|
||||
Rates: []rateGeneration{
|
||||
{Base: RateCard{CacheHit: 0.075, Input: 0.75, Output: 3.75, Currency: "USD"}},
|
||||
{From: "2027-01-01", Base: RateCard{CacheHit: 0.15, Input: 1.50, Output: 7.50, Currency: "USD"}},
|
||||
},
|
||||
},
|
||||
{
|
||||
Provider: "google", Model: "gemini-3.5-flash-lite", Currency: "USD",
|
||||
BillingMode: BillingModePAYG, DocURL: DocGeminiPricing, CheckedOn: googleCheckedOn,
|
||||
Rates: []rateGeneration{{Base: RateCard{CacheHit: 0.03, Input: 0.30, Output: 2.50, Currency: "USD"}}},
|
||||
},
|
||||
{
|
||||
Provider: "google", Model: "gemini-3.1-pro-preview", Currency: "USD",
|
||||
BillingMode: BillingModePAYG, DocURL: DocGeminiPricing, Notes: "up_to_200k_prompt_tokens", CheckedOn: googleCheckedOn,
|
||||
Rates: []rateGeneration{{Base: RateCard{CacheHit: 0.20, Input: 2.00, Output: 12.00, Currency: "USD"}}},
|
||||
},
|
||||
|
||||
{
|
||||
Provider: "anthropic", Model: "claude-sonnet-4-5", Currency: "USD",
|
||||
BillingMode: BillingModePAYG, DocURL: DocAnthropicPricing, CheckedOn: anthropicCheckedOn,
|
||||
Rates: []rateGeneration{{Base: RateCard{CacheHit: 0.30, Input: 3, Output: 15, CacheWrite: 3.75, Currency: "USD"}}},
|
||||
},
|
||||
|
||||
// MiMo domestic PAYG; Token Plan bills the same rates as subscription_equivalent.
|
||||
{
|
||||
Provider: "mimo", Model: "mimo-v2.5-pro", Currency: "CNY",
|
||||
BillingMode: BillingModePAYG, DocURL: DocMiMoPAYG, CheckedOn: checkedOn,
|
||||
Rates: []rateGeneration{{Base: RateCard{CacheHit: 0.025, Input: 3, Output: 6, Currency: "CNY"}}},
|
||||
},
|
||||
{
|
||||
Provider: "mimo", Model: "mimo-v2.5", Currency: "CNY",
|
||||
BillingMode: BillingModePAYG, DocURL: DocMiMoPAYG, CheckedOn: checkedOn,
|
||||
Rates: []rateGeneration{{Base: RateCard{CacheHit: 0.02, Input: 1, Output: 2, Currency: "CNY"}}},
|
||||
},
|
||||
// Not on the vendor's price page as of CheckedOn. Kept because removing a rate
|
||||
// is what turns a still-configured model's cost silently into zero; a stale
|
||||
// rate at least reads as a number somebody can question.
|
||||
{
|
||||
Provider: "mimo", Model: "mimo-v2-flash", Currency: "CNY",
|
||||
BillingMode: BillingModePAYG, DocURL: DocMiMoPAYG, Notes: "not_on_vendor_page", CheckedOn: checkedOn,
|
||||
Rates: []rateGeneration{{Base: RateCard{CacheHit: 0.07, Input: 0.70, Output: 2.10, Currency: "CNY"}}},
|
||||
},
|
||||
{
|
||||
Provider: "mimo", Model: "mimo-v2.5-pro", Currency: "CNY",
|
||||
BillingMode: BillingModeSubscriptionEquivalent, DocURL: DocMiMoTokenPlan, Notes: "payg_equivalent_not_plan_bill", CheckedOn: checkedOn,
|
||||
Rates: []rateGeneration{{Base: RateCard{CacheHit: 0.025, Input: 3, Output: 6, Currency: "CNY"}}},
|
||||
},
|
||||
{
|
||||
Provider: "mimo", Model: "mimo-v2.5", Currency: "CNY",
|
||||
BillingMode: BillingModeSubscriptionEquivalent, DocURL: DocMiMoTokenPlan, Notes: "payg_equivalent_not_plan_bill", CheckedOn: checkedOn,
|
||||
Rates: []rateGeneration{{Base: RateCard{CacheHit: 0.02, Input: 1, Output: 2, Currency: "CNY"}}},
|
||||
},
|
||||
{
|
||||
Provider: "mimo", Model: "mimo-v2.6-pro", Currency: "CNY",
|
||||
BillingMode: BillingModePAYG, DocURL: DocMiMoPAYG, CheckedOn: mimoCheckedOn,
|
||||
Rates: []rateGeneration{{Base: RateCard{CacheHit: 0.025, Input: 3, Output: 6, Currency: "CNY"}}},
|
||||
},
|
||||
{
|
||||
Provider: "mimo", Model: "mimo-v2.6-flash", Currency: "CNY",
|
||||
BillingMode: BillingModePAYG, DocURL: DocMiMoPAYG, CheckedOn: mimoCheckedOn,
|
||||
Rates: []rateGeneration{{Base: RateCard{CacheHit: 0.02, Input: 1, Output: 2, Currency: "CNY"}}},
|
||||
},
|
||||
{
|
||||
Provider: "mimo", Model: "mimo-v2.6-pro-ultraspeed", Currency: "CNY",
|
||||
BillingMode: BillingModePAYG, DocURL: DocMiMoPAYG, CheckedOn: mimoCheckedOn,
|
||||
Rates: []rateGeneration{{Base: RateCard{CacheHit: 0.25, Input: 30, Output: 60, Currency: "CNY"}}},
|
||||
},
|
||||
{
|
||||
Provider: "mimo", Model: "mimo-v2.6-pro", Currency: "CNY",
|
||||
BillingMode: BillingModeSubscriptionEquivalent, DocURL: DocMiMoTokenPlan, Notes: "payg_equivalent_not_plan_bill", CheckedOn: mimoCheckedOn,
|
||||
Rates: []rateGeneration{{Base: RateCard{CacheHit: 0.025, Input: 3, Output: 6, Currency: "CNY"}}},
|
||||
},
|
||||
{
|
||||
Provider: "mimo", Model: "mimo-v2.6-flash", Currency: "CNY",
|
||||
BillingMode: BillingModeSubscriptionEquivalent, DocURL: DocMiMoTokenPlan, Notes: "payg_equivalent_not_plan_bill", CheckedOn: mimoCheckedOn,
|
||||
Rates: []rateGeneration{{Base: RateCard{CacheHit: 0.02, Input: 1, Output: 2, Currency: "CNY"}}},
|
||||
},
|
||||
}
|
||||
|
||||
// retiredDeepSeekFlashModels are names DeepSeek still answers as its current
|
||||
// flash model, at that model's price. Owned here because that is where the rates
|
||||
// they bill at live; config reads them through the accessor below.
|
||||
var retiredDeepSeekFlashModels = []string{"deepseek-v4-flash", "deepseek-v4-flash-vision-exp"}
|
||||
|
||||
// RetiredDeepSeekFlashModels copies the list on the way out: a caller sorting or
|
||||
// trimming the result would otherwise change what every other caller sees.
|
||||
func RetiredDeepSeekFlashModels() []string {
|
||||
return append([]string(nil), retiredDeepSeekFlashModels...)
|
||||
}
|
||||
|
||||
// RateModelFor answers which model's rate a name is billed at — a retired name
|
||||
// at the model that replaced it, an older spelling at the one it was renamed
|
||||
// from. Scoped to billing on purpose: two models can share a rate without being
|
||||
// the same model, and nothing outside a price lookup may read this as identity.
|
||||
func RateModelFor(provider, model string) string {
|
||||
model = strings.TrimSpace(model)
|
||||
switch strings.ToLower(strings.TrimSpace(provider)) {
|
||||
case "deepseek":
|
||||
if slices.Contains(retiredDeepSeekFlashModels, model) {
|
||||
return "deepseek-flash"
|
||||
}
|
||||
case "mimo":
|
||||
switch model {
|
||||
case "mimo-v2-pro":
|
||||
return "mimo-v2.5-pro"
|
||||
case "mimo-v2-omni":
|
||||
return "mimo-v2.5"
|
||||
}
|
||||
}
|
||||
return model
|
||||
}
|
||||
|
||||
// DefaultCurrency is what a vendor's rates are quoted in when an entry does not
|
||||
// say. A vendor publishing one currency leaves no choice to make; DeepSeek
|
||||
// prices in both, and USD is what an entry that never chose has been read as.
|
||||
func DefaultCurrency(provider string) string {
|
||||
provider = strings.ToLower(strings.TrimSpace(provider))
|
||||
seen := map[string]bool{}
|
||||
for _, row := range officialRates {
|
||||
if row.Provider == provider {
|
||||
seen[row.Currency] = true
|
||||
}
|
||||
}
|
||||
if len(seen) == 1 {
|
||||
for currency := range seen {
|
||||
return currency
|
||||
}
|
||||
}
|
||||
return "USD"
|
||||
}
|
||||
|
||||
// officialRateRowsFor finds the rows for a model, which is more than one when a
|
||||
// vendor publishes the same rates under two billing modes.
|
||||
func officialRateRowsFor(provider, model, currency string) []officialRateRow {
|
||||
provider, currency = strings.ToLower(strings.TrimSpace(provider)), NormalizeCurrency(currency)
|
||||
model = RateModelFor(provider, model)
|
||||
var out []officialRateRow
|
||||
for _, row := range officialRates {
|
||||
if row.Provider == provider && row.Model == model && row.Currency == currency {
|
||||
out = append(out, row)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// CurrentRate is what a vendor's model costs in a currency today — the off-peak
|
||||
// card, which is what a caller charging one rate should quote. Nil when this
|
||||
// catalog prices neither. A model published under two billing modes quotes the
|
||||
// pay-as-you-go rate, which is the one an unsubscribed caller is charged.
|
||||
func CurrentRate(provider, model, currency string) *RateCard {
|
||||
for _, row := range officialRateRowsFor(provider, model, currency) {
|
||||
if row.BillingMode == BillingModeSubscriptionEquivalent {
|
||||
continue
|
||||
}
|
||||
if len(row.Rates) > 0 {
|
||||
rate := row.Rates[row.currentIndex()].Base
|
||||
return &rate
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// IsOfficialCard reports whether card is a rate this project has shipped for the
|
||||
// model in the card's own currency: any generation, its off-peak or its peak
|
||||
// card. A card that is none of them is the user's, or damaged, and cannot be
|
||||
// valued against the vendor's other-currency table.
|
||||
func IsOfficialCard(provider, model string, card RateCard) bool {
|
||||
for _, row := range officialRateRowsFor(provider, model, card.Currency) {
|
||||
if row.BillingMode == BillingModeSubscriptionEquivalent {
|
||||
continue
|
||||
}
|
||||
for _, generation := range row.Rates {
|
||||
if sameCard(generation.Base, card) || generation.Peak != nil && sameCard(*generation.Peak, card) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func sameCard(a, b RateCard) bool {
|
||||
return a.CacheHit == b.CacheHit && a.Input == b.Input && a.Output == b.Output && a.CacheWrite == b.CacheWrite
|
||||
}
|
||||
|
||||
// SupersededRates are the rates shipped for a model before the current one. A
|
||||
// stored price matching one came from this project rather than from its user, so
|
||||
// a refresh may replace it; anything else is theirs and is left alone.
|
||||
func SupersededRates(provider, model, currency string) []RateCard {
|
||||
var out []RateCard
|
||||
for _, row := range officialRateRowsFor(provider, model, currency) {
|
||||
if row.BillingMode == BillingModeSubscriptionEquivalent || len(row.Rates) < 2 {
|
||||
continue
|
||||
}
|
||||
for _, generation := range row.Rates[:row.currentIndex()] {
|
||||
out = append(out, generation.Base)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// officialCatalogRows publishes the current generation of every row, plus the
|
||||
// retired DeepSeek names, which bill at the flash rate under their own id.
|
||||
func officialCatalogRows() []CatalogEntry {
|
||||
rows := make([]CatalogEntry, 0, len(officialRates)+len(retiredDeepSeekFlashModels)*2)
|
||||
for _, row := range officialRates {
|
||||
if len(row.Rates) == 0 {
|
||||
continue
|
||||
}
|
||||
current := row.Rates[row.currentIndex()]
|
||||
rows = append(rows, row.entry(row.Model, current))
|
||||
if row.Provider != "deepseek" || row.Model != "deepseek-flash" {
|
||||
continue
|
||||
}
|
||||
for _, retired := range retiredDeepSeekFlashModels {
|
||||
rows = append(rows, row.entry(retired, current))
|
||||
}
|
||||
}
|
||||
return rows
|
||||
}
|
||||
|
||||
func (r officialRateRow) entry(model string, current rateGeneration) CatalogEntry {
|
||||
// Copied, so a caller holding a returned row cannot reach back into the
|
||||
// table every other caller reads.
|
||||
var peak *RateCard
|
||||
if current.Peak != nil {
|
||||
card := *current.Peak
|
||||
peak = &card
|
||||
}
|
||||
return CatalogEntry{
|
||||
Provider: r.Provider, Model: model, Currency: r.Currency,
|
||||
CacheHit: current.Base.CacheHit, Input: current.Base.Input, Output: current.Base.Output,
|
||||
CacheWrite: current.Base.CacheWrite,
|
||||
Peak: peak, Window: r.Window, DocURL: r.DocURL,
|
||||
BillingMode: r.BillingMode, Notes: r.Notes, CheckedOn: r.CheckedOn,
|
||||
}
|
||||
}
|
||||
@@ -1,12 +0,0 @@
|
||||
prompt = """Some providers bill tokens written into their prompt cache at a rate of their own. Add that to the pricing contract in this repository:
|
||||
|
||||
- `RateCard` gains a `CacheWrite` rate, per 1M tokens like the others. Zero means the card has no separate cache-write price, and cache-written tokens keep being charged at `Input` exactly as they are today.
|
||||
- When a card sets `CacheWrite`, `OriginalCostAmount` charges the billed cache-write units (`CacheWriteBilledTokens` when set, otherwise `CacheWriteTokens`) at `CacheWrite` instead of `Input`.
|
||||
- `PricingFingerprint` must tell apart two cards that differ only in `CacheWrite`, but a card without one must keep exactly the fingerprint it has today: stored ledgers are keyed by it.
|
||||
- Two cards that differ only in `CacheWrite` are different cards everywhere cards are compared, including official-card recognition and catalog matching.
|
||||
- `CatalogEntry` carries the rate too, and `RateCardFromCatalog` and `CatalogEntry.RatesAt` hand it back.
|
||||
- Add Anthropic to the official price book: `claude-sonnet-4-5`, pay-as-you-go in USD, cache hit 0.30, input 3, output 15 and cache write 3.75 per 1M tokens, documented at https://www.anthropic.com/pricing and checked 2026-10-19. `api.anthropic.com` is Anthropic's own endpoint.
|
||||
- `benchctl cost` gains a `-write` flag for cache-written tokens, and `benchctl rate` shows a card's cache-write rate when it has one.
|
||||
- Add tests for the new behaviour and update the README's pricing section."""
|
||||
class = "multi-file-feature"
|
||||
timeout_sec = 1200
|
||||
@@ -1,119 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
set -e
|
||||
export GOFLAGS=-mod=mod GOTOOLCHAIN=local
|
||||
cat > internal/contract/pricing/zz_grader_test.go <<'GO'
|
||||
package pricing
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestGraderCacheWriteCost(t *testing.T) {
|
||||
card := RateCard{CacheHit: 0.3, Input: 3, Output: 15, Currency: "USD"}
|
||||
priced := card
|
||||
priced.CacheWrite = 3.75
|
||||
cases := []struct {
|
||||
name string
|
||||
card RateCard
|
||||
usage UsageTokens
|
||||
want string
|
||||
}{
|
||||
{"no write, no rate", card, UsageTokens{CacheHitTokens: 200000, CacheMissTokens: 800000, CompletionTokens: 50000}, "3.21"},
|
||||
{"no write, rate set", priced, UsageTokens{CacheHitTokens: 200000, CacheMissTokens: 800000, CompletionTokens: 50000}, "3.21"},
|
||||
{"write at input", card, UsageTokens{CacheMissTokens: 1000000, CacheWriteTokens: 400000, CompletionTokens: 100000}, "4.5"},
|
||||
{"write at cache-write", priced, UsageTokens{CacheMissTokens: 1000000, CacheWriteTokens: 400000, CompletionTokens: 100000}, "4.8"},
|
||||
{"billed units at input", card, UsageTokens{CacheMissTokens: 1000000, CacheWriteTokens: 400000, CacheWriteBilledTokens: 500000}, "3.3"},
|
||||
{"billed units at cache-write", priced, UsageTokens{CacheMissTokens: 1000000, CacheWriteTokens: 400000, CacheWriteBilledTokens: 500000}, "3.675"},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
if got := OriginalCostAmount(tc.card, tc.usage).String(); got != tc.want {
|
||||
t.Errorf("%s: cost = %s, want %s", tc.name, got, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestGraderFingerprintKeepsStoredKeys(t *testing.T) {
|
||||
cny := RateCard{CacheHit: 0.02, Input: 1, Output: 4, Currency: "CNY"}
|
||||
usd := RateCard{CacheHit: 0.3, Input: 3, Output: 15, Currency: "USD"}
|
||||
if got := PricingFingerprint(cny); got != "ba96037de26c3f79" {
|
||||
t.Errorf("fingerprint of a card without CacheWrite moved: %s", got)
|
||||
}
|
||||
if got := PricingFingerprint(usd); got != "e16b53b4599e5fda" {
|
||||
t.Errorf("fingerprint of a card without CacheWrite moved: %s", got)
|
||||
}
|
||||
withWrite := usd
|
||||
withWrite.CacheWrite = 3.75
|
||||
if PricingFingerprint(withWrite) == PricingFingerprint(usd) {
|
||||
t.Error("cards differing only in CacheWrite share a fingerprint")
|
||||
}
|
||||
other := usd
|
||||
other.CacheWrite = 6
|
||||
if PricingFingerprint(withWrite) == PricingFingerprint(other) {
|
||||
t.Error("two different CacheWrite rates share a fingerprint")
|
||||
}
|
||||
}
|
||||
|
||||
func TestGraderCacheWriteIsPartOfCardIdentity(t *testing.T) {
|
||||
official := RateCard{CacheHit: 0.02, Input: 1, Output: 2, Currency: "CNY"}
|
||||
if !IsOfficialCard("mimo", "mimo-v2.5", official) {
|
||||
t.Fatal("the official mimo-v2.5 card is no longer recognised")
|
||||
}
|
||||
changed := official
|
||||
changed.CacheWrite = 1.25
|
||||
if IsOfficialCard("mimo", "mimo-v2.5", changed) {
|
||||
t.Error("a card with a cache-write rate the vendor never published is recognised as official")
|
||||
}
|
||||
if _, ok := MatchesCatalog("mimo", "mimo-v2.5", official); !ok {
|
||||
t.Fatal("the official mimo-v2.5 card no longer matches the catalog")
|
||||
}
|
||||
if _, ok := MatchesCatalog("mimo", "mimo-v2.5", changed); ok {
|
||||
t.Error("a card with an unpublished cache-write rate matches the catalog")
|
||||
}
|
||||
}
|
||||
|
||||
func TestGraderAnthropicIsPriced(t *testing.T) {
|
||||
want := RateCard{CacheHit: 0.30, Input: 3, Output: 15, CacheWrite: 3.75, Currency: "USD"}
|
||||
got := CurrentRate("anthropic", "claude-sonnet-4-5", "USD")
|
||||
if got == nil || *got != want {
|
||||
t.Fatalf("CurrentRate(anthropic, claude-sonnet-4-5) = %+v, want %+v", got, want)
|
||||
}
|
||||
if !IsOfficialCard("anthropic", "claude-sonnet-4-5", want) {
|
||||
t.Error("the Anthropic card is not recognised as official")
|
||||
}
|
||||
e, ok := LookupCatalog("anthropic", "claude-sonnet-4-5", "USD", "")
|
||||
if !ok || e.DocURL != "https://www.anthropic.com/pricing" || e.CheckedOn != "2026-10-19" || e.CacheWrite != 3.75 {
|
||||
t.Errorf("catalog entry = %+v, %v", e, ok)
|
||||
}
|
||||
if got := OfficialProviderForEndpoint("https://api.anthropic.com/v1/messages"); got != "anthropic" {
|
||||
t.Errorf("api.anthropic.com resolves to %q", got)
|
||||
}
|
||||
if got := DefaultCurrency("anthropic"); got != "USD" {
|
||||
t.Errorf("DefaultCurrency(anthropic) = %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGraderCatalogCarriesCacheWrite(t *testing.T) {
|
||||
e := CatalogEntry{Provider: "x", Model: "y", Currency: "USD", CacheHit: 0.1, Input: 1, Output: 2, CacheWrite: 1.25}
|
||||
if got := RateCardFromCatalog(e); got.CacheWrite != 1.25 || got.Input != 1 {
|
||||
t.Errorf("RateCardFromCatalog = %+v", got)
|
||||
}
|
||||
if got := e.RatesAt(time.Date(2026, 3, 4, 12, 0, 0, 0, time.UTC)); got.CacheWrite != 1.25 {
|
||||
t.Errorf("RatesAt = %+v", got)
|
||||
}
|
||||
}
|
||||
GO
|
||||
gofmt_out="$(gofmt -l .)"
|
||||
[ -z "$gofmt_out" ] || { echo "gofmt needed: $gofmt_out"; exit 1; }
|
||||
go vet ./...
|
||||
go test ./... >/dev/null
|
||||
out="$(go run ./cmd/benchctl cost -provider mimo -model mimo-v2.5 -miss 1000000 -write 400000)"
|
||||
[ "$out" = "1 CNY" ] || { echo "benchctl cost -write: got $out"; exit 1; }
|
||||
out="$(go run ./cmd/benchctl rate -provider anthropic -model claude-sonnet-4-5)"
|
||||
case "$out" in *"cache-write 3.75"*) ;; *) echo "benchctl rate does not show the cache-write rate: $out"; exit 1 ;; esac
|
||||
out="$(go run ./cmd/benchctl rate -provider mimo -model mimo-v2.5)"
|
||||
case "$out" in *"cache-write"*) echo "benchctl rate shows a cache-write rate the card does not have: $out"; exit 1 ;; esac
|
||||
grep -lE 'CacheWrite:' internal/contract/pricing/*_test.go | grep -qv zz_grader_test.go \
|
||||
|| { echo "no test of the pricing package sets a CacheWrite rate"; exit 1; }
|
||||
awk '/^## Pricing/{on=1; next} /^## /{on=0} on' README.md | grep -q 'CacheWrite' \
|
||||
|| { echo "README's pricing section does not mention CacheWrite"; exit 1; }
|
||||
-18
@@ -1,18 +0,0 @@
|
||||
---
|
||||
name: price-change
|
||||
description: Record a vendor's announced price change in the official price book without losing the rate it replaces.
|
||||
---
|
||||
|
||||
# Recording a price change
|
||||
|
||||
1. Find the vendor's rows in `officialRates` in
|
||||
`internal/contract/pricing/rates.go`. Each row is one model in one currency.
|
||||
2. **Append** a `rateGeneration` to the row's `Rates`; never edit an existing
|
||||
one. Set `From` to the first billed day (`YYYY-MM-DD`, UTC) when the change
|
||||
is announced ahead of time.
|
||||
3. Leave the earlier generation where it is: `SupersededRates` reads it so a
|
||||
stored price that matches it is recognised as ours and refreshed.
|
||||
4. Update the row's `CheckedOn` constant to the day you confirmed the page.
|
||||
5. Add a test beside `rates_test.go` that pins `CurrentRate` before and after
|
||||
the `From` day by replacing `today`.
|
||||
6. Run `make check`. `make pricecheck` needs the network and is run by hand.
|
||||
@@ -1,17 +0,0 @@
|
||||
.PHONY: check test vet fmt pricecheck
|
||||
|
||||
check: fmt vet test
|
||||
|
||||
fmt:
|
||||
@out="$$(gofmt -l .)"; if [ -n "$$out" ]; then echo "gofmt needed:"; echo "$$out"; exit 1; fi
|
||||
|
||||
vet:
|
||||
go vet ./...
|
||||
|
||||
test:
|
||||
go test ./...
|
||||
|
||||
# Reads each vendor's pricing page; this snapshot carries no network client.
|
||||
pricecheck:
|
||||
@echo "pricecheck: reaches each vendor's pricing page and needs the network; not available in this snapshot" >&2
|
||||
@exit 2
|
||||
@@ -1,69 +0,0 @@
|
||||
# Reasonix kernel snapshot
|
||||
|
||||
A trimmed slice of the Reasonix Go kernel: the contracts a benchmark report is
|
||||
priced and labelled with, a few leaf utilities, and `benchctl`, a small command
|
||||
that prints what those contracts answer.
|
||||
|
||||
```bash
|
||||
make check # gofmt, vet, test
|
||||
go run ./cmd/benchctl rate -model deepseek-flash -currency USD
|
||||
go run ./cmd/benchctl cost -provider mimo -model mimo-v2.5 -miss 1000000
|
||||
go run ./cmd/benchctl tariff -currency CNY -at 2026-10-12T02:00:00Z
|
||||
go run ./cmd/benchctl arm -ablate planner,evidence
|
||||
```
|
||||
|
||||
## Packages
|
||||
|
||||
| Package | Owns |
|
||||
| --- | --- |
|
||||
| `internal/contract/pricing` | the official price book, cost quotes, the spend ledger |
|
||||
| `internal/contract/ablation` | which subsystems a benchmark arm switches off, and the arm's name |
|
||||
| `internal/contract/agentgraph` | the vocabulary of one run's execution graph |
|
||||
| `internal/base/retrieval` | BM25 ranking over CJK-aware tokens |
|
||||
| `internal/base/rulelist` | splitting a one-line permission rule list |
|
||||
| `internal/base/tokencount` | a cheap token estimate |
|
||||
| `internal/base/neterr` | classifying transport errors |
|
||||
| `internal/base/nilutil` | typed-nil detection |
|
||||
| `cmd/benchctl` | prints rates, costs, peak schedules and arm names |
|
||||
|
||||
## Pricing
|
||||
|
||||
A `RateCard` is a price per million tokens in one currency:
|
||||
|
||||
| Field | Charges |
|
||||
| --- | --- |
|
||||
| `CacheHit` | prompt tokens served from the provider's cache |
|
||||
| `Input` | prompt tokens that missed the cache, including tokens written to it |
|
||||
| `Output` | completion tokens |
|
||||
|
||||
`OriginalCostAmount` turns a card and a `UsageTokens` breakdown into a
|
||||
fixed-point `Amount`; `PricingFingerprint` keys a card in the ledger, so a
|
||||
stored ledger entry stays attached to the card it was priced with.
|
||||
`CurrentRate` and `SupersededRates` answer from `officialRates`, which only
|
||||
ever grows: a price change appends a generation.
|
||||
|
||||
## Ablation
|
||||
|
||||
`ablation.Parse` reads a comma-separated list of modules to switch off, and
|
||||
`Set.Arm` names the result (`full` for the control). The switchable modules,
|
||||
in the order arm names use:
|
||||
|
||||
| Module | Off means |
|
||||
| --- | --- |
|
||||
| `evidence` | no evidence gate on completion claims |
|
||||
| `planner` | no two-model planning turn |
|
||||
| `subagent` | no delegation to workers |
|
||||
| `retrieval` | no retrieval over the folded transcript |
|
||||
| `compaction` | no context folding |
|
||||
| `upstream` | a dependency edge orders its endpoints without delivering the answer |
|
||||
| `recall-search` | recall can read a given address but cannot search for one |
|
||||
|
||||
The fold-index scale is a separate axis (`-fold-index`), not a module.
|
||||
|
||||
## Skills
|
||||
|
||||
Project skills live under `.reasonix/skills/<name>/SKILL.md`.
|
||||
|
||||
| Skill | Use it to |
|
||||
| --- | --- |
|
||||
| `price-change` | record a vendor's price change in the price book |
|
||||
@@ -1,51 +0,0 @@
|
||||
# Reasonix project memory
|
||||
|
||||
This checkout is a trimmed snapshot of the Reasonix kernel: the billing,
|
||||
ablation and graph contracts and a handful of leaf utilities, plus one small
|
||||
command that drives them. The conventions below are the project's own.
|
||||
|
||||
## Conventions
|
||||
|
||||
- Go kernel under `internal/`; each package owns one concern. A package's long
|
||||
explanation belongs in its `doc.go`, not spread across implementation files.
|
||||
- Judgements read structure, never wording: types, contracts, schemas. An
|
||||
error a caller must tell apart carries an identity — a sentinel, a typed
|
||||
code — never a sentence.
|
||||
- A metric is its definition, not its name. Before a number becomes a finding,
|
||||
state what it counts, from which anchor, over which population, and what it
|
||||
excludes.
|
||||
|
||||
## Comments
|
||||
|
||||
Default is none — the code is the truth. Write one only when the **why** is
|
||||
non-obvious: a hidden constraint, an invariant the type system cannot express,
|
||||
or an external-protocol quirk. A comment states the constraint that holds now;
|
||||
it does not narrate how the code got here. English, always.
|
||||
|
||||
- Declaration doc: ≤5 lines. Package comment: ≤8 lines, or ≤40 in a `doc.go`.
|
||||
- Every other comment: ≤3 lines. Struct-field and trailing `//`: 1 line.
|
||||
- One responsibility per file; 800 lines is the ceiling.
|
||||
|
||||
## Vendor rates
|
||||
|
||||
Official list prices live in one place: `officialRates` in
|
||||
`internal/contract/pricing/rates.go`, oldest generation first, the last being
|
||||
what a model charges today. A price change is an **append, never an edit** —
|
||||
the superseded rate is what lets an installed config be recognised as holding
|
||||
one of ours and brought forward. Callers read these through
|
||||
`pricing.CurrentRate` and `pricing.SupersededRates` and restate none of them.
|
||||
|
||||
`make pricecheck` compares that table against each vendor's own page. It needs
|
||||
the network, so it is not part of `make check`.
|
||||
|
||||
## Before every commit
|
||||
|
||||
```bash
|
||||
make check # gofmt -l, go vet ./..., go test ./...
|
||||
```
|
||||
|
||||
## Change discipline
|
||||
|
||||
List callers, consumers of the same state and sibling features before editing.
|
||||
Fix at the owning layer; a fix that breaks a neighbouring behaviour is a failed
|
||||
fix. Write tests first; they must fail without the change.
|
||||
@@ -1,115 +0,0 @@
|
||||
// Command benchctl prints what the pricing and ablation contracts answer for
|
||||
// one model or one benchmark arm, so a reviewer can check a rate or an arm name
|
||||
// without writing a test.
|
||||
package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
|
||||
"reasonix/internal/contract/ablation"
|
||||
"reasonix/internal/contract/pricing"
|
||||
)
|
||||
|
||||
var errUsage = errors.New("usage: benchctl rate|cost|tariff|arm [flags]")
|
||||
|
||||
func main() {
|
||||
if err := run(os.Args[1:], os.Stdout); err != nil {
|
||||
fmt.Fprintln(os.Stderr, "benchctl:", err)
|
||||
os.Exit(2)
|
||||
}
|
||||
}
|
||||
|
||||
func run(args []string, out io.Writer) error {
|
||||
if len(args) == 0 {
|
||||
return errUsage
|
||||
}
|
||||
switch args[0] {
|
||||
case "rate":
|
||||
return runRate(args[1:], out)
|
||||
case "cost":
|
||||
return runCost(args[1:], out)
|
||||
case "tariff":
|
||||
return runTariff(args[1:], out)
|
||||
case "arm":
|
||||
return runArm(args[1:], out)
|
||||
default:
|
||||
return fmt.Errorf("%w: unknown subcommand %q", errUsage, args[0])
|
||||
}
|
||||
}
|
||||
|
||||
func runRate(args []string, out io.Writer) error {
|
||||
fs := flag.NewFlagSet("rate", flag.ContinueOnError)
|
||||
fs.SetOutput(out)
|
||||
provider := fs.String("provider", "deepseek", "vendor")
|
||||
model := fs.String("model", "deepseek-flash", "model id")
|
||||
currency := fs.String("currency", "", "currency; empty reads the vendor's default")
|
||||
if err := fs.Parse(args); err != nil {
|
||||
return err
|
||||
}
|
||||
cur := *currency
|
||||
if cur == "" {
|
||||
cur = pricing.DefaultCurrency(*provider)
|
||||
}
|
||||
card := pricing.CurrentRate(*provider, *model, cur)
|
||||
if card == nil {
|
||||
return fmt.Errorf("no official rate for %s/%s in %s", *provider, *model, cur)
|
||||
}
|
||||
fmt.Fprintf(out, "%s/%s %s: cache-hit %g input %g output %g per 1M\n",
|
||||
*provider, *model, card.Currency, card.CacheHit, card.Input, card.Output)
|
||||
if entry, ok := pricing.LookupCatalog(*provider, *model, cur, ""); ok && entry.DocURL != "" {
|
||||
fmt.Fprintf(out, "source: %s\n", entry.DocURL)
|
||||
}
|
||||
fmt.Fprintf(out, "superseded generations: %d\n", len(pricing.SupersededRates(*provider, *model, cur)))
|
||||
return nil
|
||||
}
|
||||
|
||||
func runCost(args []string, out io.Writer) error {
|
||||
fs := flag.NewFlagSet("cost", flag.ContinueOnError)
|
||||
fs.SetOutput(out)
|
||||
provider := fs.String("provider", "deepseek", "vendor")
|
||||
model := fs.String("model", "deepseek-flash", "model id")
|
||||
currency := fs.String("currency", "", "currency; empty reads the vendor's default")
|
||||
hit := fs.Int("hit", 0, "cached prompt tokens")
|
||||
miss := fs.Int("miss", 0, "uncached prompt tokens")
|
||||
completion := fs.Int("completion", 0, "completion tokens")
|
||||
if err := fs.Parse(args); err != nil {
|
||||
return err
|
||||
}
|
||||
cur := *currency
|
||||
if cur == "" {
|
||||
cur = pricing.DefaultCurrency(*provider)
|
||||
}
|
||||
card := pricing.CurrentRate(*provider, *model, cur)
|
||||
if card == nil {
|
||||
return fmt.Errorf("no official rate for %s/%s in %s", *provider, *model, cur)
|
||||
}
|
||||
usage := pricing.UsageTokens{CacheHitTokens: *hit, CacheMissTokens: *miss, CompletionTokens: *completion}
|
||||
amount := pricing.OriginalCostAmount(*card, usage)
|
||||
fmt.Fprintf(out, "%s %s\n", amount, pricing.NormalizeCurrency(card.Currency))
|
||||
return nil
|
||||
}
|
||||
|
||||
func runArm(args []string, out io.Writer) error {
|
||||
fs := flag.NewFlagSet("arm", flag.ContinueOnError)
|
||||
fs.SetOutput(out)
|
||||
spec := fs.String("ablate", "", "comma-separated modules to switch off: "+ablation.ModuleList())
|
||||
foldIndex := fs.String("fold-index", "", "fold-index scale: default, half, quarter or off")
|
||||
list := fs.Bool("list", false, "list the switchable modules")
|
||||
if err := fs.Parse(args); err != nil {
|
||||
return err
|
||||
}
|
||||
if *list {
|
||||
fmt.Fprintln(out, ablation.ModuleList())
|
||||
return nil
|
||||
}
|
||||
set, err := ablation.ParseArm(*spec, *foldIndex)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Fprintln(out, set.Arm())
|
||||
return nil
|
||||
}
|
||||
@@ -1,65 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func runOut(t *testing.T, args ...string) string {
|
||||
t.Helper()
|
||||
var out bytes.Buffer
|
||||
if err := run(args, &out); err != nil {
|
||||
t.Fatalf("run %v: %v", args, err)
|
||||
}
|
||||
return out.String()
|
||||
}
|
||||
|
||||
func TestRateNamesTheCurrentCard(t *testing.T) {
|
||||
got := runOut(t, "rate", "-provider", "mimo", "-model", "mimo-v2.5")
|
||||
if !strings.Contains(got, "mimo/mimo-v2.5 CNY: cache-hit 0.02 input 1 output 2 per 1M") {
|
||||
t.Fatalf("rate output = %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRateRefusesAnUnpricedModel(t *testing.T) {
|
||||
var out bytes.Buffer
|
||||
if err := run([]string{"rate", "-provider", "nobody", "-model", "x"}, &out); err == nil {
|
||||
t.Fatal("an unpriced model must fail")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCostChargesTheCurrentCard(t *testing.T) {
|
||||
got := runOut(t, "cost", "-provider", "mimo", "-model", "mimo-v2.5", "-miss", "1000000", "-completion", "1000000")
|
||||
if strings.TrimSpace(got) != "3 CNY" {
|
||||
t.Fatalf("cost output = %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestArmNamesTheConfiguration(t *testing.T) {
|
||||
if got := strings.TrimSpace(runOut(t, "arm")); got != "full" {
|
||||
t.Fatalf("control arm = %q", got)
|
||||
}
|
||||
if got := strings.TrimSpace(runOut(t, "arm", "-ablate", "planner,evidence", "-fold-index", "half")); got != "no-evidence+no-planner+index-half" {
|
||||
t.Fatalf("arm = %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUnknownSubcommandIsAUsageError(t *testing.T) {
|
||||
var out bytes.Buffer
|
||||
if err := run([]string{"nope"}, &out); !errors.Is(err, errUsage) {
|
||||
t.Fatalf("err = %v, want errUsage", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTariffReadsTheVendorsSchedule(t *testing.T) {
|
||||
got := runOut(t, "tariff", "-provider", "mimo", "-model", "mimo-v2.5")
|
||||
if !strings.Contains(got, "official card: true") || !strings.Contains(got, "one rate around the clock") {
|
||||
t.Fatalf("tariff output = %q", got)
|
||||
}
|
||||
peak := runOut(t, "tariff", "-currency", "CNY", "-at", "2026-10-12T02:00:00Z")
|
||||
if !strings.Contains(peak, "peak=true until 2026-10-12T04:00:00Z") {
|
||||
t.Fatalf("deepseek tariff at a weekday peak = %q", peak)
|
||||
}
|
||||
}
|
||||
@@ -1,49 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"time"
|
||||
|
||||
"reasonix/internal/contract/pricing"
|
||||
)
|
||||
|
||||
func runTariff(args []string, out io.Writer) error {
|
||||
fs := flag.NewFlagSet("tariff", flag.ContinueOnError)
|
||||
fs.SetOutput(out)
|
||||
provider := fs.String("provider", "deepseek", "vendor")
|
||||
model := fs.String("model", "deepseek-flash", "model id")
|
||||
currency := fs.String("currency", "", "currency; empty reads the vendor's default")
|
||||
at := fs.String("at", "", "instant to read the schedule at, RFC 3339; empty is now")
|
||||
if err := fs.Parse(args); err != nil {
|
||||
return err
|
||||
}
|
||||
when := time.Now()
|
||||
if *at != "" {
|
||||
t, err := time.Parse(time.RFC3339, *at)
|
||||
if err != nil {
|
||||
return fmt.Errorf("-at: %w", err)
|
||||
}
|
||||
when = t
|
||||
}
|
||||
cur := *currency
|
||||
if cur == "" {
|
||||
cur = pricing.DefaultCurrency(*provider)
|
||||
}
|
||||
card := pricing.CurrentRate(*provider, *model, cur)
|
||||
if card == nil {
|
||||
return fmt.Errorf("no official rate for %s/%s in %s", *provider, *model, cur)
|
||||
}
|
||||
fmt.Fprintf(out, "official card: %v\n", pricing.IsOfficialCard(*provider, *model, *card))
|
||||
tariff, ok := pricing.LiveTariff(*provider, *model, *card, when)
|
||||
switch {
|
||||
case !ok:
|
||||
fmt.Fprintln(out, "schedule: one rate around the clock")
|
||||
case !tariff.Changes:
|
||||
fmt.Fprintf(out, "schedule: peak=%v, no change ahead\n", tariff.Peak)
|
||||
default:
|
||||
fmt.Fprintf(out, "schedule: peak=%v until %s\n", tariff.Peak, tariff.ChangesAt.UTC().Format(time.RFC3339))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -1,3 +0,0 @@
|
||||
module reasonix
|
||||
|
||||
go 1.26.0
|
||||
-7
@@ -1,7 +0,0 @@
|
||||
//go:build !windows
|
||||
|
||||
package neterr
|
||||
|
||||
import "syscall"
|
||||
|
||||
var resetErrors = []error{syscall.ECONNRESET, syscall.ECONNABORTED, syscall.EPIPE}
|
||||
-8
@@ -1,8 +0,0 @@
|
||||
package neterr
|
||||
|
||||
import "syscall"
|
||||
|
||||
// syscall.ECONNRESET and friends exist on Windows only as APPLICATION_ERROR
|
||||
// placeholders, so errors.Is against them never matches a real socket failure.
|
||||
// Winsock's own numbering carries the identities the runtime returns.
|
||||
var resetErrors = []error{syscall.WSAECONNRESET, syscall.WSAECONNABORTED, syscall.ERROR_BROKEN_PIPE}
|
||||
@@ -1,47 +0,0 @@
|
||||
// Package neterr answers what a transport error is from the identity the
|
||||
// operating system gave it, never from the sentence it rendered. A system
|
||||
// error's text arrives in the machine's own language and is reworded between
|
||||
// releases, so a word list that matches on an English host silently stops
|
||||
// matching elsewhere. The judgement lives here once because three packages
|
||||
// were each carrying their own copy of it.
|
||||
package neterr
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"net"
|
||||
)
|
||||
|
||||
// IsConnReset reports whether err is a connection-level drop (peer reset,
|
||||
// truncated body, closed socket) as opposed to a protocol or caller error. A
|
||||
// stream cut this way mid-body can be replayed from scratch, unlike a decode
|
||||
// or 4xx error. The common trigger is a local proxy (v2rayN/sing-box)
|
||||
// idle-closing a long-lived SSE connection during a reasoner's first-token gap.
|
||||
func IsConnReset(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
|
||||
return false
|
||||
}
|
||||
if errors.Is(err, io.ErrUnexpectedEOF) || errors.Is(err, io.EOF) ||
|
||||
errors.Is(err, net.ErrClosed) || isAny(err, resetErrors) {
|
||||
return true
|
||||
}
|
||||
var dnsErr *net.DNSError
|
||||
if errors.As(err, &dnsErr) {
|
||||
return false
|
||||
}
|
||||
var netErr net.Error
|
||||
return errors.As(err, &netErr)
|
||||
}
|
||||
|
||||
func isAny(err error, targets []error) bool {
|
||||
for _, target := range targets {
|
||||
if errors.Is(err, target) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
-43
@@ -1,43 +0,0 @@
|
||||
package neterr
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestIsConnReset(t *testing.T) {
|
||||
if IsConnReset(nil) {
|
||||
t.Error("nil is not a conn reset")
|
||||
}
|
||||
if IsConnReset(context.Canceled) || IsConnReset(context.DeadlineExceeded) {
|
||||
t.Error("ctx cancel/deadline must not look like a recoverable reset")
|
||||
}
|
||||
if IsConnReset(errors.New("decode stream: invalid character")) {
|
||||
t.Error("a plain protocol error must not be treated as a conn reset")
|
||||
}
|
||||
for _, err := range []error{
|
||||
io.ErrUnexpectedEOF,
|
||||
&net.OpError{Op: "read", Err: resetErrors[0]},
|
||||
fmt.Errorf("read stream: %w", &net.OpError{Op: "read", Err: errors.New("wsarecv: forcibly closed")}),
|
||||
} {
|
||||
if !IsConnReset(err) {
|
||||
t.Errorf("want conn reset for %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsConnResetExcludesDNS(t *testing.T) {
|
||||
for _, dns := range []*net.DNSError{
|
||||
{Err: "no such host", Name: "api.deepseek.com", IsNotFound: true},
|
||||
{Err: "i/o timeout", Name: "api.deepseek.com", IsTimeout: true, IsTemporary: true},
|
||||
} {
|
||||
err := fmt.Errorf("dial: %w", &net.OpError{Op: "dial", Err: dns})
|
||||
if IsConnReset(err) {
|
||||
t.Errorf("DNS failure %v is not a connection reset", dns)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,17 +0,0 @@
|
||||
package nilutil
|
||||
|
||||
import "reflect"
|
||||
|
||||
// IsNil reports whether v is nil or an interface holding a typed nil value.
|
||||
func IsNil(v any) bool {
|
||||
if v == nil {
|
||||
return true
|
||||
}
|
||||
rv := reflect.ValueOf(v)
|
||||
switch rv.Kind() {
|
||||
case reflect.Chan, reflect.Func, reflect.Interface, reflect.Map, reflect.Pointer, reflect.Slice:
|
||||
return rv.IsNil()
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
-31
@@ -1,31 +0,0 @@
|
||||
package nilutil
|
||||
|
||||
import "testing"
|
||||
|
||||
type sampleInterface interface {
|
||||
Do()
|
||||
}
|
||||
|
||||
type sampleImpl struct{}
|
||||
|
||||
func (*sampleImpl) Do() {}
|
||||
|
||||
func TestIsNilDetectsTypedNilInterface(t *testing.T) {
|
||||
var p *sampleImpl
|
||||
var i sampleInterface = p
|
||||
|
||||
if !IsNil(i) {
|
||||
t.Fatal("IsNil should detect typed nil interface")
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsNilRejectsConcreteValues(t *testing.T) {
|
||||
var p sampleInterface = &sampleImpl{}
|
||||
|
||||
if IsNil(p) {
|
||||
t.Fatal("IsNil should not reject a concrete interface value")
|
||||
}
|
||||
if IsNil("x") {
|
||||
t.Fatal("IsNil should not reject non-nil scalar values")
|
||||
}
|
||||
}
|
||||
-218
@@ -1,218 +0,0 @@
|
||||
package retrieval
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"math"
|
||||
"strings"
|
||||
"unicode"
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
// Tokens lowercases Latin words and splits CJK runs into overlapping bigrams
|
||||
// (a lone CJK rune stays a single term) — the standard CJK indexing unit
|
||||
// (Lucene's CJKAnalyzer, SQLite FTS): a bigram only matches a real two-rune
|
||||
// subsequence where per-rune unigrams matched scattered common characters.
|
||||
// Intentionally a local, dependency-free approximation of FTS matching.
|
||||
func Tokens(s string) []string {
|
||||
var out []string
|
||||
var b strings.Builder
|
||||
var prev rune
|
||||
cjkRun := 0
|
||||
flush := func() {
|
||||
if b.Len() == 0 {
|
||||
return
|
||||
}
|
||||
out = append(out, b.String())
|
||||
b.Reset()
|
||||
}
|
||||
endCJK := func() {
|
||||
if cjkRun == 1 {
|
||||
out = append(out, string(prev))
|
||||
}
|
||||
cjkRun = 0
|
||||
}
|
||||
for _, r := range s {
|
||||
switch {
|
||||
case isCJK(r):
|
||||
flush()
|
||||
if cjkRun > 0 {
|
||||
out = append(out, string([]rune{prev, r}))
|
||||
}
|
||||
prev = r
|
||||
cjkRun++
|
||||
case unicode.IsLetter(r) || unicode.IsDigit(r) || r == '_':
|
||||
endCJK()
|
||||
b.WriteRune(unicode.ToLower(r))
|
||||
default:
|
||||
flush()
|
||||
endCJK()
|
||||
}
|
||||
}
|
||||
flush()
|
||||
endCJK()
|
||||
return out
|
||||
}
|
||||
|
||||
func isCJK(r rune) bool {
|
||||
return unicode.In(r, unicode.Han, unicode.Hiragana, unicode.Katakana, unicode.Hangul)
|
||||
}
|
||||
|
||||
// Unique returns terms in first-seen order.
|
||||
func Unique(in []string) []string {
|
||||
seen := map[string]bool{}
|
||||
out := make([]string, 0, len(in))
|
||||
for _, s := range in {
|
||||
if s == "" || seen[s] {
|
||||
continue
|
||||
}
|
||||
seen[s] = true
|
||||
out = append(out, s)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Counts returns a term-frequency map.
|
||||
func Counts(terms []string) map[string]int {
|
||||
counts := map[string]int{}
|
||||
for _, term := range terms {
|
||||
counts[term]++
|
||||
}
|
||||
return counts
|
||||
}
|
||||
|
||||
// BM25Score scores a document against query terms.
|
||||
func BM25Score(counts map[string]int, length int, queryTerms []string, df map[string]int, totalDocs int, avgLen float64) float64 {
|
||||
const (
|
||||
k1 = 1.2
|
||||
b = 0.75
|
||||
)
|
||||
if length <= 0 || totalDocs <= 0 {
|
||||
return 0
|
||||
}
|
||||
if avgLen <= 0 {
|
||||
avgLen = 1
|
||||
}
|
||||
var score float64
|
||||
docLen := float64(length)
|
||||
for _, term := range queryTerms {
|
||||
tf := counts[term]
|
||||
if tf == 0 {
|
||||
continue
|
||||
}
|
||||
termDF := df[term]
|
||||
if termDF == 0 {
|
||||
continue
|
||||
}
|
||||
idf := math.Log(1 + (float64(totalDocs)-float64(termDF)+0.5)/(float64(termDF)+0.5))
|
||||
freq := float64(tf)
|
||||
score += idf * (freq * (k1 + 1)) / (freq + k1*(1-b+b*docLen/avgLen))
|
||||
}
|
||||
return score
|
||||
}
|
||||
|
||||
// DocumentFrequency counts how many documents contain each term.
|
||||
func DocumentFrequency(docs []map[string]int) map[string]int {
|
||||
df := map[string]int{}
|
||||
for _, counts := range docs {
|
||||
for term := range counts {
|
||||
df[term]++
|
||||
}
|
||||
}
|
||||
return df
|
||||
}
|
||||
|
||||
// KeepTopRelativeScore keeps the best item and drops trailing items whose score
|
||||
// falls below ratio * topScore. Callers must pass items already sorted best
|
||||
// first. This mirrors SQLite FTS/BM25 search UIs that over-fetch, then trim
|
||||
// common-word-only noise without imposing an absolute score threshold.
|
||||
func KeepTopRelativeScore[T any](items []T, ratio float64, score func(T) float64) []T {
|
||||
if len(items) == 0 || ratio <= 0 {
|
||||
return items
|
||||
}
|
||||
top := score(items[0])
|
||||
if top <= 0 {
|
||||
return items
|
||||
}
|
||||
cutoff := top * ratio
|
||||
out := items[:0]
|
||||
for i, item := range items {
|
||||
if i == 0 || score(item) >= cutoff {
|
||||
out = append(out, item)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// QueryTerms normalizes a search string and reports an error when nothing
|
||||
// searchable remains.
|
||||
func QueryTerms(query string) ([]string, error) {
|
||||
terms := Unique(Tokens(strings.TrimSpace(query)))
|
||||
if len(terms) == 0 {
|
||||
return nil, fmt.Errorf("query must contain at least one letter or number")
|
||||
}
|
||||
return terms, nil
|
||||
}
|
||||
|
||||
// MakeSnippet returns a whitespace-compacted excerpt centered near the query.
|
||||
func MakeSnippet(text, query string, terms []string, maxRunes int) string {
|
||||
text = CompactWhitespace(text)
|
||||
if maxRunes <= 0 || utf8.RuneCountInString(text) <= maxRunes {
|
||||
return text
|
||||
}
|
||||
lower := strings.ToLower(text)
|
||||
query = strings.ToLower(strings.TrimSpace(query))
|
||||
idx := -1
|
||||
if query != "" {
|
||||
idx = strings.Index(lower, query)
|
||||
}
|
||||
if idx < 0 {
|
||||
for _, term := range terms {
|
||||
runes := []rune(term)
|
||||
if len(runes) == 1 && !isCJK(runes[0]) {
|
||||
continue
|
||||
}
|
||||
if i := strings.Index(lower, term); i >= 0 {
|
||||
idx = i
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
if idx < 0 {
|
||||
idx = 0
|
||||
}
|
||||
return snippetAround(text, idx, maxRunes)
|
||||
}
|
||||
|
||||
func snippetAround(text string, byteIdx, maxRunes int) string {
|
||||
if byteIdx < 0 {
|
||||
byteIdx = 0
|
||||
}
|
||||
if byteIdx > len(text) {
|
||||
byteIdx = len(text)
|
||||
}
|
||||
for byteIdx > 0 && byteIdx < len(text) && !utf8.RuneStart(text[byteIdx]) {
|
||||
byteIdx--
|
||||
}
|
||||
runes := []rune(text)
|
||||
pos := utf8.RuneCountInString(text[:byteIdx])
|
||||
start := max(pos-maxRunes/2, 0)
|
||||
end := start + maxRunes
|
||||
if end > len(runes) {
|
||||
end = len(runes)
|
||||
start = max(end-maxRunes, 0)
|
||||
}
|
||||
prefix := ""
|
||||
suffix := ""
|
||||
if start > 0 {
|
||||
prefix = "..."
|
||||
}
|
||||
if end < len(runes) {
|
||||
suffix = "..."
|
||||
}
|
||||
return prefix + string(runes[start:end]) + suffix
|
||||
}
|
||||
|
||||
// CompactWhitespace collapses runs of whitespace into one ASCII space.
|
||||
func CompactWhitespace(s string) string {
|
||||
return strings.Join(strings.Fields(s), " ")
|
||||
}
|
||||
-77
@@ -1,77 +0,0 @@
|
||||
package retrieval
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
func TestTokensHandlesLatinAndCJK(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
in string
|
||||
want []string
|
||||
}{
|
||||
{"BM25 检索 cache-first", []string{"bm25", "检索", "cache", "first"}},
|
||||
{"数据库迁移", []string{"数据", "据库", "库迁", "迁移"}},
|
||||
{"库", []string{"库"}},
|
||||
{"用pnpm装依赖", []string{"用", "pnpm", "装依", "依赖"}},
|
||||
} {
|
||||
got := Tokens(tc.in)
|
||||
if strings.Join(got, ",") != strings.Join(tc.want, ",") {
|
||||
t.Fatalf("Tokens(%q) = %#v, want %#v", tc.in, got, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBM25ScoreRanksMatchingDocument(t *testing.T) {
|
||||
query := Unique(Tokens("prompt cache"))
|
||||
doc1 := Counts(Tokens("prompt cache cache stability"))
|
||||
doc2 := Counts(Tokens("dashboard colors"))
|
||||
df := DocumentFrequency([]map[string]int{doc1, doc2})
|
||||
score1 := BM25Score(doc1, 4, query, df, 2, 3)
|
||||
score2 := BM25Score(doc2, 2, query, df, 2, 3)
|
||||
if score1 <= score2 {
|
||||
t.Fatalf("matching score %.3f should exceed unrelated score %.3f", score1, score2)
|
||||
}
|
||||
}
|
||||
|
||||
func TestKeepTopRelativeScoreKeepsTopAndDropsWeakTail(t *testing.T) {
|
||||
items := []struct {
|
||||
name string
|
||||
score float64
|
||||
}{
|
||||
{name: "top", score: 10},
|
||||
{name: "near", score: 2},
|
||||
{name: "noise", score: 1.4},
|
||||
{name: "zero", score: 0},
|
||||
}
|
||||
got := KeepTopRelativeScore(items, 0.15, func(item struct {
|
||||
name string
|
||||
score float64
|
||||
}) float64 {
|
||||
return item.score
|
||||
})
|
||||
if len(got) != 2 || got[0].name != "top" || got[1].name != "near" {
|
||||
t.Fatalf("KeepTopRelativeScore() = %#v, want top and near", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMakeSnippetHandlesMultibyteBoundary(t *testing.T) {
|
||||
text := strings.Repeat("前缀", 80) + "稳定结论 synthesis cache " + strings.Repeat("后缀", 80)
|
||||
out := MakeSnippet(text, "synthesis cache", QueryTermsForTest(t, "synthesis cache"), 60)
|
||||
if !strings.Contains(out, "synthesis cache") {
|
||||
t.Fatalf("snippet missing query: %q", out)
|
||||
}
|
||||
if strings.ContainsRune(out, utf8.RuneError) {
|
||||
t.Fatalf("snippet contains replacement rune: %q", out)
|
||||
}
|
||||
}
|
||||
|
||||
func QueryTermsForTest(t *testing.T, query string) []string {
|
||||
t.Helper()
|
||||
terms, err := QueryTerms(query)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return terms
|
||||
}
|
||||
@@ -1,176 +0,0 @@
|
||||
// Retrieval V2, shadow-only: BM25F field weighting, code-symbol splitting,
|
||||
// and mixed CJK uni+bigrams. Nothing here serves the model — V2 rankings ride
|
||||
// the recall audit next to production's, and MemoryBench decides if V2 ever
|
||||
// takes over. Weights are candidates under measurement, not tuned truths.
|
||||
package retrieval
|
||||
|
||||
import (
|
||||
"math"
|
||||
"strings"
|
||||
"unicode"
|
||||
)
|
||||
|
||||
// TokensV2 extends Tokens with code-symbol structure and CJK unigrams:
|
||||
// CamelCase and snake_case words also emit their segments (FooBar → foobar,
|
||||
// foo, bar), and CJK runs emit unigrams alongside bigrams so one-character
|
||||
// queries still match. The V1 form of every token is preserved, so V2 recall
|
||||
// is a superset of V1's vocabulary.
|
||||
func TokensV2(s string) []string {
|
||||
var out []string
|
||||
var word []rune
|
||||
var prev rune
|
||||
cjkRun := 0
|
||||
flushWord := func() {
|
||||
if len(word) == 0 {
|
||||
return
|
||||
}
|
||||
lower := strings.ToLower(string(word))
|
||||
out = append(out, lower)
|
||||
for _, seg := range splitCodeSymbol(word) {
|
||||
if seg != lower {
|
||||
out = append(out, seg)
|
||||
}
|
||||
}
|
||||
word = word[:0]
|
||||
}
|
||||
endCJK := func() { cjkRun = 0 }
|
||||
for _, r := range s {
|
||||
switch {
|
||||
case isCJK(r):
|
||||
flushWord()
|
||||
out = append(out, string(r))
|
||||
if cjkRun > 0 {
|
||||
out = append(out, string([]rune{prev, r}))
|
||||
}
|
||||
prev = r
|
||||
cjkRun++
|
||||
case unicode.IsLetter(r) || unicode.IsDigit(r) || r == '_':
|
||||
endCJK()
|
||||
word = append(word, r)
|
||||
default:
|
||||
flushWord()
|
||||
endCJK()
|
||||
}
|
||||
}
|
||||
flushWord()
|
||||
return out
|
||||
}
|
||||
|
||||
// splitCodeSymbol yields the lowercase segments of a code identifier:
|
||||
// CamelCase humps, snake_case parts, and letter/digit boundaries.
|
||||
func splitCodeSymbol(word []rune) []string {
|
||||
var segments []string
|
||||
var seg []rune
|
||||
flush := func() {
|
||||
if len(seg) > 1 { // single letters are noise, not signal
|
||||
segments = append(segments, strings.ToLower(string(seg)))
|
||||
}
|
||||
seg = seg[:0]
|
||||
}
|
||||
for i, r := range word {
|
||||
boundary := r == '_' ||
|
||||
(i > 0 && unicode.IsUpper(r) && !unicode.IsUpper(word[i-1])) ||
|
||||
(i > 0 && unicode.IsDigit(r) != unicode.IsDigit(word[i-1]))
|
||||
if boundary {
|
||||
flush()
|
||||
}
|
||||
if r != '_' {
|
||||
seg = append(seg, r)
|
||||
}
|
||||
}
|
||||
flush()
|
||||
if len(segments) < 2 {
|
||||
return nil // no structure worth indexing beyond the whole word
|
||||
}
|
||||
return segments
|
||||
}
|
||||
|
||||
// FieldedDoc is one document split into weighted fields for BM25F.
|
||||
type FieldedDoc struct {
|
||||
ID string
|
||||
Fields map[string]string
|
||||
}
|
||||
|
||||
// v2FieldWeights follow the BM25F intuition: identity fields say what a fact
|
||||
// IS, the body says everything it mentions. Candidates under measurement.
|
||||
var v2FieldWeights = map[string]float64{
|
||||
"name": 2.5, "title": 2.5, "keywords": 2.0, "subject": 2.0,
|
||||
"description": 1.5, "body": 1.0,
|
||||
}
|
||||
|
||||
// V2Hit is one shadow-ranked document.
|
||||
type V2Hit struct {
|
||||
ID string
|
||||
Score float64
|
||||
}
|
||||
|
||||
// RankV2 scores docs against a query with BM25F over TokensV2: term
|
||||
// frequencies accumulate per field scaled by field weight, then the standard
|
||||
// saturation applies once per term (the F in BM25F), with length
|
||||
// normalization over the weighted document length.
|
||||
func RankV2(query string, docs []FieldedDoc) []V2Hit {
|
||||
queryTerms := Unique(TokensV2(query))
|
||||
if len(queryTerms) == 0 || len(docs) == 0 {
|
||||
return nil
|
||||
}
|
||||
type indexed struct {
|
||||
id string
|
||||
tf map[string]float64
|
||||
length float64
|
||||
}
|
||||
corpus := make([]indexed, 0, len(docs))
|
||||
df := map[string]int{}
|
||||
var totalLen float64
|
||||
for _, doc := range docs {
|
||||
ix := indexed{id: doc.ID, tf: map[string]float64{}}
|
||||
seen := map[string]bool{}
|
||||
for field, text := range doc.Fields {
|
||||
weight, ok := v2FieldWeights[field]
|
||||
if !ok {
|
||||
weight = 1.0
|
||||
}
|
||||
for _, term := range TokensV2(text) {
|
||||
ix.tf[term] += weight
|
||||
ix.length += weight
|
||||
seen[term] = true
|
||||
}
|
||||
}
|
||||
for term := range seen {
|
||||
df[term]++
|
||||
}
|
||||
totalLen += ix.length
|
||||
corpus = append(corpus, ix)
|
||||
}
|
||||
avgLen := totalLen / float64(len(corpus))
|
||||
if avgLen <= 0 {
|
||||
avgLen = 1
|
||||
}
|
||||
const k1, b = 1.2, 0.75
|
||||
var hits []V2Hit
|
||||
for _, doc := range corpus {
|
||||
var score float64
|
||||
for _, term := range queryTerms {
|
||||
tf := doc.tf[term]
|
||||
if tf == 0 || df[term] == 0 {
|
||||
continue
|
||||
}
|
||||
idf := math.Log(1 + (float64(len(corpus))-float64(df[term])+0.5)/(float64(df[term])+0.5))
|
||||
score += idf * (tf * (k1 + 1)) / (tf + k1*(1-b+b*doc.length/avgLen))
|
||||
}
|
||||
if score > 0 {
|
||||
hits = append(hits, V2Hit{ID: doc.id, Score: score})
|
||||
}
|
||||
}
|
||||
SortHitsDesc(hits)
|
||||
return hits
|
||||
}
|
||||
|
||||
// SortHitsDesc orders shadow hits best-first with a stable ID tiebreak.
|
||||
func SortHitsDesc(hits []V2Hit) {
|
||||
for i := 1; i < len(hits); i++ {
|
||||
for j := i; j > 0 && (hits[j].Score > hits[j-1].Score ||
|
||||
(hits[j].Score == hits[j-1].Score && hits[j].ID < hits[j-1].ID)); j-- {
|
||||
hits[j], hits[j-1] = hits[j-1], hits[j]
|
||||
}
|
||||
}
|
||||
}
|
||||
-47
@@ -1,47 +0,0 @@
|
||||
package retrieval
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestTokensV2CodeSymbolsAndCJK(t *testing.T) {
|
||||
got := strings.Join(TokensV2("RetryPolicy.MaxBackoffMs ticket_1827 数据库"), " ")
|
||||
for _, want := range []string{"retrypolicy", "retry", "policy", "maxbackoffms", "max", "backoff", "ticket_1827", "ticket", "1827", "数", "据", "库", "数据", "据库"} {
|
||||
if !strings.Contains(" "+got+" ", " "+want+" ") {
|
||||
t.Fatalf("TokensV2 missing %q in %q", want, got)
|
||||
}
|
||||
}
|
||||
if strings.Contains(" "+got+" ", " ms ") {
|
||||
// "Ms" splits to a single meaningful segment only when len > 1; "ms" is
|
||||
// fine — assert single letters never appear instead.
|
||||
t.Log("ms segment present (acceptable)")
|
||||
}
|
||||
for _, tok := range TokensV2("A B C") {
|
||||
if len(tok) == 0 {
|
||||
t.Fatal("empty token emitted")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRankV2FieldWeightsBeatBodyMentions(t *testing.T) {
|
||||
docs := []FieldedDoc{
|
||||
{ID: "titled", Fields: map[string]string{"title": "Package manager", "body": "We migrated to pnpm."}},
|
||||
{ID: "mention", Fields: map[string]string{"title": "Sprint notes", "body": "Someone asked about the package manager once; also weather, lunch, and a package of stickers."}},
|
||||
}
|
||||
hits := RankV2("package manager", docs)
|
||||
if len(hits) != 2 || hits[0].ID != "titled" {
|
||||
t.Fatalf("field-weighted doc must outrank body mentions: %+v", hits)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRankV2FindsCodeSymbolBySegment(t *testing.T) {
|
||||
docs := []FieldedDoc{
|
||||
{ID: "sym", Fields: map[string]string{"body": "Configure RetryPolicy.MaxBackoffMs in policy/retry.toml."}},
|
||||
{ID: "other", Fields: map[string]string{"body": "Unrelated fact about deploys."}},
|
||||
}
|
||||
hits := RankV2("max backoff configuration", docs)
|
||||
if len(hits) == 0 || hits[0].ID != "sym" {
|
||||
t.Fatalf("segment query must find the camel-case symbol: %+v", hits)
|
||||
}
|
||||
}
|
||||
@@ -1,7 +0,0 @@
|
||||
// Package rulelist splits a list of permission rules written on one line.
|
||||
//
|
||||
// Rules are separated by commas or whitespace, and a separator inside
|
||||
// parentheses belongs to the rule: `Bash(git add:*) Read` is two rules. Skill
|
||||
// front matter and the --allowed-tools flag share it, so one spelling means
|
||||
// one thing in both.
|
||||
package rulelist
|
||||
-56
@@ -1,56 +0,0 @@
|
||||
package rulelist
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"strings"
|
||||
"unicode"
|
||||
)
|
||||
|
||||
var (
|
||||
// ErrUnclosed: an opening parenthesis is never closed.
|
||||
ErrUnclosed = errors.New("unclosed '('")
|
||||
// ErrUnexpectedClose: a closing parenthesis has no opening one.
|
||||
ErrUnexpectedClose = errors.New("unexpected ')'")
|
||||
)
|
||||
|
||||
// Split returns the rules in value, in order, trimmed and non-empty.
|
||||
func Split(value string) ([]string, error) {
|
||||
var rules []string
|
||||
start, depth := -1, 0
|
||||
flush := func(end int) {
|
||||
if start < 0 {
|
||||
return
|
||||
}
|
||||
if rule := strings.TrimSpace(value[start:end]); rule != "" {
|
||||
rules = append(rules, rule)
|
||||
}
|
||||
start = -1
|
||||
}
|
||||
for i, r := range value {
|
||||
switch r {
|
||||
case '(':
|
||||
if start < 0 {
|
||||
start = i
|
||||
}
|
||||
depth++
|
||||
case ')':
|
||||
if depth == 0 {
|
||||
return nil, ErrUnexpectedClose
|
||||
}
|
||||
depth--
|
||||
default:
|
||||
if depth == 0 && (r == ',' || unicode.IsSpace(r)) {
|
||||
flush(i)
|
||||
continue
|
||||
}
|
||||
if start < 0 {
|
||||
start = i
|
||||
}
|
||||
}
|
||||
}
|
||||
if depth != 0 {
|
||||
return nil, ErrUnclosed
|
||||
}
|
||||
flush(len(value))
|
||||
return rules, nil
|
||||
}
|
||||
-40
@@ -1,40 +0,0 @@
|
||||
package rulelist
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"reflect"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestSplitSeparatesOnCommasAndWhitespaceOutsideParentheses(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
in string
|
||||
want []string
|
||||
}{
|
||||
{"", nil},
|
||||
{" ", nil},
|
||||
{"Read", []string{"Read"}},
|
||||
{"read_file, grep,bash", []string{"read_file", "grep", "bash"}},
|
||||
{"Read Grep Bash", []string{"Read", "Grep", "Bash"}},
|
||||
{"Bash(git add:*) Read", []string{"Bash(git add:*)", "Read"}},
|
||||
{"Bash(git add:*, git commit:*), Read", []string{"Bash(git add:*, git commit:*)", "Read"}},
|
||||
{"bash,", []string{"bash"}},
|
||||
} {
|
||||
got, err := Split(tc.in)
|
||||
if err != nil {
|
||||
t.Fatalf("Split(%q): %v", tc.in, err)
|
||||
}
|
||||
if !reflect.DeepEqual(got, tc.want) {
|
||||
t.Errorf("Split(%q) = %q, want %q", tc.in, got, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSplitReportsUnbalancedParenthesesByIdentity(t *testing.T) {
|
||||
if _, err := Split("Bash(git add"); !errors.Is(err, ErrUnclosed) {
|
||||
t.Errorf("unclosed = %v", err)
|
||||
}
|
||||
if _, err := Split("Bash)"); !errors.Is(err, ErrUnexpectedClose) {
|
||||
t.Errorf("unexpected close = %v", err)
|
||||
}
|
||||
}
|
||||
-25
@@ -1,25 +0,0 @@
|
||||
// Package tokencount estimates how many tokens a string costs a model, without
|
||||
// carrying a tokenizer. Anything that has to size text before a provider counts
|
||||
// it — a cache-shape diagnostic, a tool call's context footprint — needs one
|
||||
// answer to that question rather than a private heuristic per caller.
|
||||
package tokencount
|
||||
|
||||
import "unicode/utf8"
|
||||
|
||||
// Text estimates the token count of s. The split is by script because the two
|
||||
// behave nothing alike — Latin text and code trend near four bytes per token,
|
||||
// CJK closer to one token per character — and either rule applied to the whole
|
||||
// string is wrong by ~4x on the other half. A real tokenizer would be exact and
|
||||
// would mean a vocabulary per model, re-run over whole transcripts every turn.
|
||||
func Text(s string) int {
|
||||
narrow, wide := 0, 0
|
||||
for _, r := range s {
|
||||
if r < utf8.RuneSelf {
|
||||
narrow++
|
||||
continue
|
||||
}
|
||||
wide++
|
||||
}
|
||||
// Round up: a long run of short strings must not sum to nothing.
|
||||
return (narrow+3)/4 + wide
|
||||
}
|
||||
-27
@@ -1,27 +0,0 @@
|
||||
package tokencount
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestTextDoesNotUndercountCJK(t *testing.T) {
|
||||
// The reason this package exists: bytes/4 reads a Chinese transcript as a
|
||||
// third of its real size, so a planner sizing context off it compacts late.
|
||||
const cjk = "把这个仓库跑一遍测试,把失败的那几个定位到具体文件"
|
||||
runes := len([]rune(cjk))
|
||||
if got := Text(cjk); got != runes {
|
||||
t.Fatalf("Text(cjk) = %d, want one token per rune (%d)", got, runes)
|
||||
}
|
||||
if byBytes := len(cjk) / 4; Text(cjk) <= byBytes {
|
||||
t.Fatalf("Text(cjk) = %d, must exceed the bytes/4 reading (%d)", Text(cjk), byBytes)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTextRoundsASCIIUpAndKeepsEmptyFree(t *testing.T) {
|
||||
if got := Text(""); got != 0 {
|
||||
t.Fatalf("Text(\"\") = %d, want 0", got)
|
||||
}
|
||||
// Five bytes is more than one token's worth; truncating would let a long
|
||||
// run of short strings sum to nothing.
|
||||
if got := Text("abcde"); got != 2 {
|
||||
t.Fatalf("Text(\"abcde\") = %d, want 2", got)
|
||||
}
|
||||
}
|
||||
-200
@@ -1,200 +0,0 @@
|
||||
// Package ablation switches individual Reasonix subsystems off so a benchmark
|
||||
// can attribute a change in solve rate to one of them.
|
||||
package ablation
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type Module string
|
||||
|
||||
const (
|
||||
Evidence Module = "evidence"
|
||||
Planner Module = "planner"
|
||||
Subagent Module = "subagent"
|
||||
Retrieval Module = "retrieval"
|
||||
Compaction Module = "compaction"
|
||||
// Upstream off means a fleet dependency edge orders its endpoints without
|
||||
// delivering the dependency's answer, so a dependent must re-derive it.
|
||||
Upstream Module = "upstream"
|
||||
// RecallSearch off leaves recall able to read an address it was given and
|
||||
// unable to look one up, which is what the tool could do before the folded
|
||||
// region became searchable.
|
||||
RecallSearch Module = "recall-search"
|
||||
)
|
||||
|
||||
// FoldIndexScale sizes the model-visible fold index against its shipped budget.
|
||||
// It is a bench axis, not a module: the question it answers is what an ambient
|
||||
// address list is worth, and four steps answer that where a free-form fraction
|
||||
// would turn a measurement into a tuning knob.
|
||||
type FoldIndexScale string
|
||||
|
||||
const (
|
||||
FoldIndexDefault FoldIndexScale = ""
|
||||
FoldIndexHalf FoldIndexScale = "half"
|
||||
FoldIndexQuarter FoldIndexScale = "quarter"
|
||||
FoldIndexOff FoldIndexScale = "off"
|
||||
)
|
||||
|
||||
// Ratio scales a budget. The default returns it unchanged.
|
||||
func (f FoldIndexScale) Ratio() float64 {
|
||||
switch f {
|
||||
case FoldIndexHalf:
|
||||
return 0.5
|
||||
case FoldIndexQuarter:
|
||||
return 0.25
|
||||
case FoldIndexOff:
|
||||
return 0
|
||||
default:
|
||||
return 1
|
||||
}
|
||||
}
|
||||
|
||||
// ParseFoldIndexScale reads a bench flag value. "" and "default" ship as-is.
|
||||
func ParseFoldIndexScale(spec string) (FoldIndexScale, error) {
|
||||
switch strings.ToLower(strings.TrimSpace(spec)) {
|
||||
case "", "default", "full":
|
||||
return FoldIndexDefault, nil
|
||||
case string(FoldIndexHalf):
|
||||
return FoldIndexHalf, nil
|
||||
case string(FoldIndexQuarter):
|
||||
return FoldIndexQuarter, nil
|
||||
case string(FoldIndexOff), "none", "0":
|
||||
return FoldIndexOff, nil
|
||||
default:
|
||||
return "", fmt.Errorf("unknown fold-index scale %q (want default, half, quarter, or off)", spec)
|
||||
}
|
||||
}
|
||||
|
||||
// Modules returns every switchable module in the order arm names use.
|
||||
func Modules() []Module {
|
||||
return []Module{Evidence, Planner, Subagent, Retrieval, Compaction, Upstream, RecallSearch}
|
||||
}
|
||||
|
||||
// Set is the group of modules disabled for a run. The zero value is the
|
||||
// control arm: everything on.
|
||||
type Set struct {
|
||||
off map[Module]bool
|
||||
// foldIndex is an axis rather than a switch, so it lives beside the module
|
||||
// set instead of inside it. It still reaches Arm(): a run whose index was
|
||||
// quartered must not report the same identity as the control.
|
||||
foldIndex FoldIndexScale
|
||||
}
|
||||
|
||||
// WithFoldIndex returns this set at the given index scale.
|
||||
func (s Set) WithFoldIndex(scale FoldIndexScale) Set {
|
||||
s.foldIndex = scale
|
||||
return s
|
||||
}
|
||||
|
||||
// FoldIndex is the scale this run sizes its fold index at.
|
||||
func (s Set) FoldIndex() FoldIndexScale { return s.foldIndex }
|
||||
|
||||
// Parse reads a spec such as "evidence,planner". "" and "none" mean the control
|
||||
// arm; "all" disables every module.
|
||||
func Parse(spec string) (Set, error) {
|
||||
spec = strings.TrimSpace(spec)
|
||||
if spec == "" || strings.EqualFold(spec, "none") {
|
||||
return Set{}, nil
|
||||
}
|
||||
if strings.EqualFold(spec, "all") {
|
||||
return New(Modules()...), nil
|
||||
}
|
||||
known := map[Module]bool{}
|
||||
for _, m := range Modules() {
|
||||
known[m] = true
|
||||
}
|
||||
var mods []Module
|
||||
for _, field := range strings.FieldsFunc(spec, func(r rune) bool { return r == ',' || r == ' ' }) {
|
||||
m := Module(strings.ToLower(strings.TrimSpace(field)))
|
||||
if !known[m] {
|
||||
return Set{}, fmt.Errorf("unknown ablation module %q (want %s, or none/all)", field, ModuleList())
|
||||
}
|
||||
mods = append(mods, m)
|
||||
}
|
||||
return New(mods...), nil
|
||||
}
|
||||
|
||||
// ParseArm reads both bench axes at once: the module spec and the fold-index
|
||||
// scale. One call so a frontend cannot wire the first and forget the second,
|
||||
// which would silently run an index arm under the control arm's name.
|
||||
func ParseArm(modules, foldIndex string) (Set, error) {
|
||||
set, err := Parse(modules)
|
||||
if err != nil {
|
||||
return Set{}, err
|
||||
}
|
||||
scale, err := ParseFoldIndexScale(foldIndex)
|
||||
if err != nil {
|
||||
return Set{}, err
|
||||
}
|
||||
return set.WithFoldIndex(scale), nil
|
||||
}
|
||||
|
||||
// ModuleList names every switchable module for flag help and error text, so
|
||||
// those cannot drift from Modules() the way a hand-kept list does.
|
||||
func ModuleList() string { return joinModules(Modules(), ", ") }
|
||||
|
||||
// New returns a Set with the given modules disabled.
|
||||
func New(mods ...Module) Set {
|
||||
if len(mods) == 0 {
|
||||
return Set{}
|
||||
}
|
||||
off := make(map[Module]bool, len(mods))
|
||||
for _, m := range mods {
|
||||
off[m] = true
|
||||
}
|
||||
return Set{off: off}
|
||||
}
|
||||
|
||||
func (s Set) Off(m Module) bool { return s.off[m] }
|
||||
|
||||
func (s Set) Empty() bool { return len(s.off) == 0 && s.foldIndex == FoldIndexDefault }
|
||||
|
||||
// Arm is the published name of this configuration: "full" for the control arm,
|
||||
// otherwise "no-evidence+no-planner". Stable across runs so results from
|
||||
// different machines group by the same key.
|
||||
func (s Set) Arm() string {
|
||||
if s.Empty() {
|
||||
return "full"
|
||||
}
|
||||
parts := make([]string, 0, len(s.off)+1)
|
||||
for _, m := range s.disabled() {
|
||||
parts = append(parts, "no-"+string(m))
|
||||
}
|
||||
if s.foldIndex != FoldIndexDefault {
|
||||
parts = append(parts, "index-"+string(s.foldIndex))
|
||||
}
|
||||
return strings.Join(parts, "+")
|
||||
}
|
||||
|
||||
// String round-trips back through Parse. It carries modules only — the fold
|
||||
// index axis has its own flag and its own parser.
|
||||
func (s Set) String() string {
|
||||
if s.Empty() {
|
||||
return "none"
|
||||
}
|
||||
return joinModules(s.disabled(), ",")
|
||||
}
|
||||
|
||||
func (s Set) disabled() []Module {
|
||||
order := map[Module]int{}
|
||||
for i, m := range Modules() {
|
||||
order[m] = i
|
||||
}
|
||||
out := make([]Module, 0, len(s.off))
|
||||
for m := range s.off {
|
||||
out = append(out, m)
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool { return order[out[i]] < order[out[j]] })
|
||||
return out
|
||||
}
|
||||
|
||||
func joinModules(mods []Module, sep string) string {
|
||||
parts := make([]string, len(mods))
|
||||
for i, m := range mods {
|
||||
parts[i] = string(m)
|
||||
}
|
||||
return strings.Join(parts, sep)
|
||||
}
|
||||
Loaded 100 of 5496 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user