mirror of
https://github.com/esengine/DeepSeek-Reasonix.git
synced 2026-11-07 17:10:43 -10:00
Compare commits
2
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
7943fac193 | ||
|
|
a6cf72080f |
No files matched your search
@@ -10,7 +10,6 @@
|
||||
|
||||
# User-facing guides and references.
|
||||
/docs/ @SivanCola @esengine
|
||||
/docs/THEME_AUTHOR_GUIDE.md @esengine @SivanCola
|
||||
/sdk/ @SivanCola @esengine
|
||||
/README.md @SivanCola @esengine
|
||||
/README.zh-CN.md @SivanCola @esengine
|
||||
@@ -37,7 +36,6 @@
|
||||
/.github/workflows/studio-certum-signing-smoke.yml @esengine @SivanCola
|
||||
/.github/actions/setup-certum/ @esengine @SivanCola
|
||||
/cmd/signpath-contract/ @esengine @SivanCola
|
||||
/scripts/install-smoke.ps1 @esengine @SivanCola
|
||||
/scripts/sign-certum.ps1 @esengine @SivanCola
|
||||
/scripts/verify-windows-authenticode.ps1 @esengine @SivanCola
|
||||
/scripts/verify-windows-package.ps1 @esengine @SivanCola
|
||||
|
||||
@@ -6,9 +6,3 @@ contact_links:
|
||||
- name: Questions & discussions
|
||||
url: https://github.com/esengine/DeepSeek-Reasonix/discussions
|
||||
about: For usage questions, ideas, and general discussion — please use Discussions instead of opening an issue.
|
||||
- name: Discord
|
||||
url: https://discord.gg/XF78rEME2D
|
||||
about: Bilingual community chat for setup help, workflow showcases, and feature ideas.
|
||||
- name: QQ group (DeepSeek-Reasonix官方群, 1093562660)
|
||||
url: https://qm.qq.com/q/i59b0z2R8s
|
||||
about: Chinese-language community chat for setup help and everyday questions.
|
||||
@@ -8,34 +8,6 @@ only, on `main-v2`.
|
||||
|
||||
-
|
||||
|
||||
## Cause
|
||||
|
||||
-
|
||||
|
||||
## Blast radius
|
||||
|
||||
-
|
||||
|
||||
## Neighbouring behaviours tested
|
||||
|
||||
-
|
||||
|
||||
## Why this layer
|
||||
|
||||
-
|
||||
|
||||
## UI changes
|
||||
|
||||
<!--
|
||||
Delete this section if the PR touches nothing under `desktop/frontend-next/src`
|
||||
or the website pages.
|
||||
-->
|
||||
|
||||
- Screenshots, before and after: light and dark, a wide window and a phone-width one (about 390 px). Attach them to this description.
|
||||
- Entries moved, hidden or removed: none / list each one.
|
||||
- Hiding or removing a visible control needs a linked issue and a release-notes line naming it. Do not retire UI with `display: none !important` or a feature flag.
|
||||
- If a row of `docs/STUDIO_PARITY.md` changes, update it here.
|
||||
|
||||
## Issues
|
||||
|
||||
<!--
|
||||
@@ -54,7 +26,3 @@ the source-to-target contract here: easing syntax, time units, callbacks,
|
||||
cancellation, reduced-motion behavior, and failure fallback. Verification must
|
||||
assert that the target API was actually called; a mock that silently skips it
|
||||
does not count.
|
||||
|
||||
## Disclosure
|
||||
|
||||
- [ ] AI assistance was used for this change (optional; for transparency only, it does not change how the change is reviewed)
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 138 KiB |
@@ -1,19 +0,0 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<svg id="_图层_2" data-name="图层 2" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 145.55 113.29">
|
||||
<defs>
|
||||
<style>
|
||||
.cls-1 {
|
||||
fill: #ffffff;
|
||||
stroke: #ffffff;
|
||||
stroke-miterlimit: 10;
|
||||
}
|
||||
</style>
|
||||
</defs>
|
||||
<g id="_图层_1-2" data-name="图层 1">
|
||||
<g>
|
||||
<path class="cls-1" d="M144.68,38.49l-.06-.23c-.88-3.28-2.5-5.94-4.58-8.06.14,5.65-2.96,11.02-6.22,16.66l-.39.68c-2.26,3.91-4.66,7.94-6.57,11.1l-.86,1.38c-3.36,5.44-6.27,10.14-12.18,14.18-8.34,5.87-18.2,5.81-26.92,5.76-2.81-.02-5.48-.03-7.95.14l-.35.02c-3.22.06-5.96,1.57-8.17,4.49l-.14.18c-.86,1.06-1.7,2.26-2.58,3.54-3.43,4.92-7.69,11.04-16.17,12.42-4.37.86-9.98.84-14.94.83h-1.95c-.52,0-1.06,0-1.61,0-6.95,0-16.08-.89-21.94-6.55-.15-.15-.3-.3-.44-.45.5,4.73,2.33,8.64,5.44,11.65,5.86,5.66,14.98,6.55,21.94,6.55.55,0,1.09,0,1.61-.01h1.93c4.96.02,10.58.04,14.96-.83,8.48-1.37,12.74-7.5,16.17-12.42.88-1.27,1.72-2.48,2.58-3.54l.14-.18c2.21-2.92,4.95-4.43,8.17-4.49l.35-.02c2.47-.17,5.13-.16,7.95-.14,8.72.05,18.58.11,26.91-5.76,5.92-4.03,8.82-8.73,12.19-14.17l.86-1.39c1.91-3.15,4.3-7.18,6.57-11.09l.39-.69c3.81-6.6,7.41-12.84,5.86-19.57ZM120.9,23.02c-.28,0-.56,0-.83,0-9.68-.19-24.03-.09-35.57-.01l-2.04.02c-.93.01-1.82.02-2.67.03-8.36.08-14.4.13-23.33,3.82l-.27.12c-10.76,4.68-16.91,12.16-22.83,21.95-5.53,8.76-12.62,20.57-16.32,26.79-.41.69-.77,1.3-1.09,1.84l-.49.84c-1.49,2.53-3.23,5.49-4.21,8.95,3.98,1.9,8.52,2.65,12.71,2.91.27-.77.64-1.56,1.07-2.38.48-.94,1.05-1.9,1.63-2.89l.49-.84c.96-1.62,2.38-3.99,4.05-6.78,3.82-6.36,8.98-14.88,13.19-21.55l.07-.11c5.02-8.31,9.2-13.45,16.91-16.81l.11-.05c6.57-2.7,10.54-2.74,18.43-2.81.87-.01,1.78-.02,2.69-.03l1.99-.02c9.17-.06,20.13-.14,29.01-.07,2.26.01,4.39.04,6.32.08h.12s.11,0,.11,0c1.25-.01,2.91.11,4.61.46,1.03.22,2.08.52,3.05.93.21-.35.41-.71.62-1.06l.39-.68c1.92-3.33,3.79-6.57,4.97-9.82-4.08-1.92-8.7-2.76-12.89-2.82Z"/>
|
||||
<path class="cls-1" d="M115.07,11.81c-9.7-.19-24.07-.09-35.62-.01h-1.99c-.93.03-1.82.04-2.67.04-8.36.08-14.4.14-23.33,3.83l-.27.12c-10.76,4.67-16.91,12.16-22.83,21.95-6.13,9.71-14.19,23.21-17.41,28.63l-.5.84c-2.08,3.55-4.68,7.97-4.94,13.39v.2s0,.21,0,.21c.01.81.06,1.61.15,2.38.14.15.29.3.44.45,1.53,1.47,3.28,2.63,5.15,3.52,3.98,1.9,8.52,2.65,12.71,2.91,1.41.09,2.78.12,4.08.12.55,0,1.09-.01,1.61-.01h1.95c4.95.01,10.57.03,14.94-.83,8.48-1.38,12.74-7.5,16.17-12.42.88-1.28,1.72-2.48,2.58-3.54l.14-.18c2.21-2.92,4.95-4.44,8.17-4.49l.35-.02c2.47-.17,5.14-.16,7.95-.14,8.71.05,18.58.1,26.92-5.76,5.91-4.04,8.82-8.74,12.18-14.18l.86-1.39c1.74-2.86,3.87-6.45,5.95-10.03.21-.35.41-.71.62-1.06l.39-.68c1.92-3.33,3.79-6.57,4.97-9.82.82-2.26,1.31-4.53,1.25-6.84-5.18-5.29-13.22-7.25-19.97-7.19ZM122.56,40.37l-.39.67c-2.21,3.81-4.55,7.77-6.39,10.79l-.84,1.36c-3.01,4.87-4.83,7.81-8.48,10.29l-.1.07c-4.94,3.49-11.95,3.45-19.38,3.41-2.88-.02-5.87-.04-8.79.16-7.1.19-13.51,3.58-18.07,9.57-1.13,1.4-2.12,2.83-3.08,4.21-2.92,4.18-4.71,6.57-7.64,7.02l-.31.06c-3.09.63-8.28.61-12.45.6h-2.16c-3.85.07-7.01-.16-9.45-.69-2.24-.48-3.87-1.22-4.89-2.2-.66-.64-1.54-1.81-1.63-4.65.11-1.35.71-2.82,1.52-4.35.48-.94,1.05-1.9,1.63-2.89l.49-.84c3.17-5.33,11.19-18.75,17.24-28.33l.07-.11c5.02-8.32,9.2-13.45,16.92-16.81l.1-.05c6.57-2.7,10.54-2.74,18.43-2.82.87,0,1.78,0,2.69-.03h1.94c11.52-.09,25.86-.19,35.38,0h.23c1.25-.01,2.92.11,4.61.46,3.15.66,6.4,2.12,7.27,5.02.2,1.2-.89,3.62-2.27,6.18-.7,1.31-1.48,2.65-2.2,3.9ZM120.07,23.01c-9.68-.19-24.03-.09-35.57-.01l-2.04.02c-.93.01-1.82.02-2.67.03-8.36.08-14.4.13-23.33,3.82l-.27.12c-10.76,4.68-16.91,12.16-22.83,21.95-5.53,8.76-12.62,20.57-16.32,26.79-.78-.35-1.41-.77-1.9-1.24-.66-.64-1.54-1.81-1.63-4.65.18-2.18,1.62-4.64,3.15-7.25l.49-.83c3.17-5.32,11.18-18.73,17.24-28.33l.07-.12c5.02-8.31,9.2-13.45,16.92-16.81l.1-.04c6.57-2.7,10.54-2.74,18.43-2.82.87-.01,1.78-.01,2.69-.03h1.99c11.5-.09,25.82-.19,35.33,0h.23c3.57-.04,10.55,1.02,11.88,5.48.14.84-.35,2.27-1.13,3.93-.28,0-.56,0-.83,0Z"/>
|
||||
<path class="cls-1" d="M134.68,16.09l-.06-.23c-3.07-11.45-15.08-15.33-24.55-15.25-9.68-.19-24.03-.09-35.57-.01h-2.04c-.93.03-1.82.04-2.67.04-8.36.08-14.4.14-23.33,3.83l-.27.12c-10.76,4.67-16.91,12.16-22.83,21.95-6.14,9.73-14.2,23.22-17.41,28.62l-.49.85c-2.09,3.55-4.69,7.96-4.95,13.39v.2s0,.21,0,.21c.09,5.57,1.82,10.13,5.15,13.58.14.15.29.3.44.45,1.53,1.47,3.28,2.63,5.15,3.52,3.98,1.9,8.52,2.65,12.71,2.91,1.41.09,2.78.12,4.08.12.55,0,1.09-.01,1.61-.01h1.95c4.95.01,10.57.03,14.94-.83,8.48-1.38,12.74-7.5,16.17-12.42.88-1.28,1.72-2.48,2.58-3.54l.14-.18c2.21-2.92,4.95-4.44,8.17-4.49l.35-.02c2.47-.17,5.14-.16,7.95-.14,8.71.05,18.58.1,26.92-5.76,5.91-4.04,8.82-8.74,12.18-14.18l.86-1.39c1.74-2.86,3.87-6.45,5.95-10.03.21-.35.41-.71.62-1.06l.39-.68c1.92-3.33,3.79-6.57,4.97-9.82.82-2.26,1.31-4.53,1.25-6.84-.02-.96-.14-1.93-.36-2.91ZM119.76,25.27c-.7,1.31-1.48,2.65-2.2,3.9l-.39.67c-1.19,2.04-2.41,4.13-3.57,6.09-1.01,1.7-1.97,3.31-2.82,4.7l-.84,1.36c-3.01,4.87-4.83,7.81-8.48,10.29l-.1.07c-4.94,3.49-11.96,3.45-19.38,3.41-2.89-.02-5.87-.04-8.79.16-7.1.18-13.51,3.58-18.07,9.57-1.13,1.4-2.12,2.83-3.08,4.21-2.92,4.18-4.71,6.57-7.64,7.02l-.31.06c-3.09.63-8.27.61-12.45.6h-2.16c-3.85.07-7.01-.16-9.45-.69-1.16-.25-2.16-.57-2.99-.96-.78-.35-1.41-.77-1.9-1.24-.66-.64-1.54-1.81-1.63-4.65.18-2.18,1.62-4.64,3.15-7.25l.49-.83c3.17-5.32,11.18-18.73,17.24-28.33l.07-.12c5.02-8.31,9.2-13.45,16.92-16.81l.1-.04c6.57-2.7,10.54-2.74,18.43-2.82.87-.01,1.78-.01,2.69-.03h1.99c11.5-.09,25.82-.19,35.33,0h.23c3.57-.04,10.55,1.02,11.88,5.48.14.84-.35,2.27-1.13,3.93-.34.72-.73,1.48-1.14,2.25Z"/>
|
||||
</g>
|
||||
</g>
|
||||
</svg>
|
||||
|
Before Width: | Height: | Size: 5.4 KiB |
@@ -1,19 +0,0 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<svg id="_图层_2" data-name="图层 2" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 145.55 113.29">
|
||||
<defs>
|
||||
<style>
|
||||
.cls-1 {
|
||||
fill: #111111;
|
||||
stroke: #111111;
|
||||
stroke-miterlimit: 10;
|
||||
}
|
||||
</style>
|
||||
</defs>
|
||||
<g id="_图层_1-2" data-name="图层 1">
|
||||
<g>
|
||||
<path class="cls-1" d="M144.68,38.49l-.06-.23c-.88-3.28-2.5-5.94-4.58-8.06.14,5.65-2.96,11.02-6.22,16.66l-.39.68c-2.26,3.91-4.66,7.94-6.57,11.1l-.86,1.38c-3.36,5.44-6.27,10.14-12.18,14.18-8.34,5.87-18.2,5.81-26.92,5.76-2.81-.02-5.48-.03-7.95.14l-.35.02c-3.22.06-5.96,1.57-8.17,4.49l-.14.18c-.86,1.06-1.7,2.26-2.58,3.54-3.43,4.92-7.69,11.04-16.17,12.42-4.37.86-9.98.84-14.94.83h-1.95c-.52,0-1.06,0-1.61,0-6.95,0-16.08-.89-21.94-6.55-.15-.15-.3-.3-.44-.45.5,4.73,2.33,8.64,5.44,11.65,5.86,5.66,14.98,6.55,21.94,6.55.55,0,1.09,0,1.61-.01h1.93c4.96.02,10.58.04,14.96-.83,8.48-1.37,12.74-7.5,16.17-12.42.88-1.27,1.72-2.48,2.58-3.54l.14-.18c2.21-2.92,4.95-4.43,8.17-4.49l.35-.02c2.47-.17,5.13-.16,7.95-.14,8.72.05,18.58.11,26.91-5.76,5.92-4.03,8.82-8.73,12.19-14.17l.86-1.39c1.91-3.15,4.3-7.18,6.57-11.09l.39-.69c3.81-6.6,7.41-12.84,5.86-19.57ZM120.9,23.02c-.28,0-.56,0-.83,0-9.68-.19-24.03-.09-35.57-.01l-2.04.02c-.93.01-1.82.02-2.67.03-8.36.08-14.4.13-23.33,3.82l-.27.12c-10.76,4.68-16.91,12.16-22.83,21.95-5.53,8.76-12.62,20.57-16.32,26.79-.41.69-.77,1.3-1.09,1.84l-.49.84c-1.49,2.53-3.23,5.49-4.21,8.95,3.98,1.9,8.52,2.65,12.71,2.91.27-.77.64-1.56,1.07-2.38.48-.94,1.05-1.9,1.63-2.89l.49-.84c.96-1.62,2.38-3.99,4.05-6.78,3.82-6.36,8.98-14.88,13.19-21.55l.07-.11c5.02-8.31,9.2-13.45,16.91-16.81l.11-.05c6.57-2.7,10.54-2.74,18.43-2.81.87-.01,1.78-.02,2.69-.03l1.99-.02c9.17-.06,20.13-.14,29.01-.07,2.26.01,4.39.04,6.32.08h.12s.11,0,.11,0c1.25-.01,2.91.11,4.61.46,1.03.22,2.08.52,3.05.93.21-.35.41-.71.62-1.06l.39-.68c1.92-3.33,3.79-6.57,4.97-9.82-4.08-1.92-8.7-2.76-12.89-2.82Z"/>
|
||||
<path class="cls-1" d="M115.07,11.81c-9.7-.19-24.07-.09-35.62-.01h-1.99c-.93.03-1.82.04-2.67.04-8.36.08-14.4.14-23.33,3.83l-.27.12c-10.76,4.67-16.91,12.16-22.83,21.95-6.13,9.71-14.19,23.21-17.41,28.63l-.5.84c-2.08,3.55-4.68,7.97-4.94,13.39v.2s0,.21,0,.21c.01.81.06,1.61.15,2.38.14.15.29.3.44.45,1.53,1.47,3.28,2.63,5.15,3.52,3.98,1.9,8.52,2.65,12.71,2.91,1.41.09,2.78.12,4.08.12.55,0,1.09-.01,1.61-.01h1.95c4.95.01,10.57.03,14.94-.83,8.48-1.38,12.74-7.5,16.17-12.42.88-1.28,1.72-2.48,2.58-3.54l.14-.18c2.21-2.92,4.95-4.44,8.17-4.49l.35-.02c2.47-.17,5.14-.16,7.95-.14,8.71.05,18.58.1,26.92-5.76,5.91-4.04,8.82-8.74,12.18-14.18l.86-1.39c1.74-2.86,3.87-6.45,5.95-10.03.21-.35.41-.71.62-1.06l.39-.68c1.92-3.33,3.79-6.57,4.97-9.82.82-2.26,1.31-4.53,1.25-6.84-5.18-5.29-13.22-7.25-19.97-7.19ZM122.56,40.37l-.39.67c-2.21,3.81-4.55,7.77-6.39,10.79l-.84,1.36c-3.01,4.87-4.83,7.81-8.48,10.29l-.1.07c-4.94,3.49-11.95,3.45-19.38,3.41-2.88-.02-5.87-.04-8.79.16-7.1.19-13.51,3.58-18.07,9.57-1.13,1.4-2.12,2.83-3.08,4.21-2.92,4.18-4.71,6.57-7.64,7.02l-.31.06c-3.09.63-8.28.61-12.45.6h-2.16c-3.85.07-7.01-.16-9.45-.69-2.24-.48-3.87-1.22-4.89-2.2-.66-.64-1.54-1.81-1.63-4.65.11-1.35.71-2.82,1.52-4.35.48-.94,1.05-1.9,1.63-2.89l.49-.84c3.17-5.33,11.19-18.75,17.24-28.33l.07-.11c5.02-8.32,9.2-13.45,16.92-16.81l.1-.05c6.57-2.7,10.54-2.74,18.43-2.82.87,0,1.78,0,2.69-.03h1.94c11.52-.09,25.86-.19,35.38,0h.23c1.25-.01,2.92.11,4.61.46,3.15.66,6.4,2.12,7.27,5.02.2,1.2-.89,3.62-2.27,6.18-.7,1.31-1.48,2.65-2.2,3.9ZM120.07,23.01c-9.68-.19-24.03-.09-35.57-.01l-2.04.02c-.93.01-1.82.02-2.67.03-8.36.08-14.4.13-23.33,3.82l-.27.12c-10.76,4.68-16.91,12.16-22.83,21.95-5.53,8.76-12.62,20.57-16.32,26.79-.78-.35-1.41-.77-1.9-1.24-.66-.64-1.54-1.81-1.63-4.65.18-2.18,1.62-4.64,3.15-7.25l.49-.83c3.17-5.32,11.18-18.73,17.24-28.33l.07-.12c5.02-8.31,9.2-13.45,16.92-16.81l.1-.04c6.57-2.7,10.54-2.74,18.43-2.82.87-.01,1.78-.01,2.69-.03h1.99c11.5-.09,25.82-.19,35.33,0h.23c3.57-.04,10.55,1.02,11.88,5.48.14.84-.35,2.27-1.13,3.93-.28,0-.56,0-.83,0Z"/>
|
||||
<path class="cls-1" d="M134.68,16.09l-.06-.23c-3.07-11.45-15.08-15.33-24.55-15.25-9.68-.19-24.03-.09-35.57-.01h-2.04c-.93.03-1.82.04-2.67.04-8.36.08-14.4.14-23.33,3.83l-.27.12c-10.76,4.67-16.91,12.16-22.83,21.95-6.14,9.73-14.2,23.22-17.41,28.62l-.49.85c-2.09,3.55-4.69,7.96-4.95,13.39v.2s0,.21,0,.21c.09,5.57,1.82,10.13,5.15,13.58.14.15.29.3.44.45,1.53,1.47,3.28,2.63,5.15,3.52,3.98,1.9,8.52,2.65,12.71,2.91,1.41.09,2.78.12,4.08.12.55,0,1.09-.01,1.61-.01h1.95c4.95.01,10.57.03,14.94-.83,8.48-1.38,12.74-7.5,16.17-12.42.88-1.28,1.72-2.48,2.58-3.54l.14-.18c2.21-2.92,4.95-4.44,8.17-4.49l.35-.02c2.47-.17,5.14-.16,7.95-.14,8.71.05,18.58.1,26.92-5.76,5.91-4.04,8.82-8.74,12.18-14.18l.86-1.39c1.74-2.86,3.87-6.45,5.95-10.03.21-.35.41-.71.62-1.06l.39-.68c1.92-3.33,3.79-6.57,4.97-9.82.82-2.26,1.31-4.53,1.25-6.84-.02-.96-.14-1.93-.36-2.91ZM119.76,25.27c-.7,1.31-1.48,2.65-2.2,3.9l-.39.67c-1.19,2.04-2.41,4.13-3.57,6.09-1.01,1.7-1.97,3.31-2.82,4.7l-.84,1.36c-3.01,4.87-4.83,7.81-8.48,10.29l-.1.07c-4.94,3.49-11.96,3.45-19.38,3.41-2.89-.02-5.87-.04-8.79.16-7.1.18-13.51,3.58-18.07,9.57-1.13,1.4-2.12,2.83-3.08,4.21-2.92,4.18-4.71,6.57-7.64,7.02l-.31.06c-3.09.63-8.27.61-12.45.6h-2.16c-3.85.07-7.01-.16-9.45-.69-1.16-.25-2.16-.57-2.99-.96-.78-.35-1.41-.77-1.9-1.24-.66-.64-1.54-1.81-1.63-4.65.18-2.18,1.62-4.64,3.15-7.25l.49-.83c3.17-5.32,11.18-18.73,17.24-28.33l.07-.12c5.02-8.31,9.2-13.45,16.92-16.81l.1-.04c6.57-2.7,10.54-2.74,18.43-2.82.87-.01,1.78-.01,2.69-.03h1.99c11.5-.09,25.82-.19,35.33,0h.23c3.57-.04,10.55,1.02,11.88,5.48.14.84-.35,2.27-1.13,3.93-.34.72-.73,1.48-1.14,2.25Z"/>
|
||||
</g>
|
||||
</g>
|
||||
</svg>
|
||||
|
Before Width: | Height: | Size: 5.4 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 2.4 KiB |
+29
-54
@@ -26,9 +26,7 @@ jobs:
|
||||
# merges. Gated jobs skip only on an explicit `false` output: wrapped in
|
||||
# `always()`, a failed `changes` job (or a missing output) makes them run
|
||||
# the full matrix instead of silently passing required checks as skipped.
|
||||
# Pushes to main-v2 always run everything. Pull requests run the macOS and
|
||||
# Windows legs only when `platform` is not 'false'; the rule lives in
|
||||
# scripts/ci-platform-scope.sh and fails toward running them.
|
||||
# Pushes to main-v2 always run everything.
|
||||
changes:
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
@@ -36,14 +34,13 @@ jobs:
|
||||
desktop: ${{ steps.filter.outputs.desktop }}
|
||||
sdk: ${{ steps.filter.outputs.sdk }}
|
||||
scripts: ${{ steps.filter.outputs.scripts }}
|
||||
platform: ${{ steps.filter.outputs.platform }}
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- id: filter
|
||||
run: |
|
||||
code=true; desktop=true; sdk=true; scripts=true; platform=true
|
||||
code=true; desktop=true; sdk=true; scripts=true
|
||||
base=""
|
||||
if [ "${{ github.event_name }}" = "pull_request" ]; then
|
||||
base="${{ github.event.pull_request.base.sha }}"
|
||||
@@ -51,7 +48,7 @@ jobs:
|
||||
base="${{ github.event.before }}"
|
||||
fi
|
||||
if [ -n "$base" ] && git cat-file -e "$base^{commit}" 2>/dev/null; then
|
||||
files=$(git diff --name-only --no-renames "$base" HEAD)
|
||||
files=$(git diff --name-only "$base" HEAD)
|
||||
if [ -n "$files" ]; then
|
||||
code=false; desktop=false; sdk=false; scripts=false
|
||||
# Root-module CI: desktop/ is a separate module, so only the
|
||||
@@ -65,11 +62,10 @@ jobs:
|
||||
# both paths must trigger the sdk job.
|
||||
if echo "$files" | grep -qE '^(sdk/|internal/ext/extension/)'; then sdk=true; fi
|
||||
# The release-note tests render every committed Studio note; npm/ holds the publish tests.
|
||||
if echo "$files" | grep -qE '^(scripts/|npm/|release-notes/studio/|\.github/workflows/release-studio\.yml$)'; then scripts=true; fi
|
||||
platform=$(git diff -z --name-only --no-renames "$base" HEAD | bash scripts/ci-platform-scope.sh)
|
||||
if echo "$files" | grep -qE '^(scripts/|npm/|release-notes/studio/)'; then scripts=true; fi
|
||||
fi
|
||||
fi
|
||||
{ echo "code=$code"; echo "desktop=$desktop"; echo "sdk=$sdk"; echo "scripts=$scripts"; echo "platform=$platform"; } >> "$GITHUB_OUTPUT"
|
||||
{ echo "code=$code"; echo "desktop=$desktop"; echo "sdk=$sdk"; echo "scripts=$scripts"; } >> "$GITHUB_OUTPUT"
|
||||
|
||||
release-notes-scripts:
|
||||
needs: changes
|
||||
@@ -81,14 +77,10 @@ jobs:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7
|
||||
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
|
||||
with:
|
||||
node-version: "22"
|
||||
- uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7
|
||||
with:
|
||||
go-version-file: go.mod
|
||||
cache: true
|
||||
- run: node --test scripts/linux-install-smoke.test.mjs scripts/studio-artifacts.test.mjs scripts/ci-platform-scope.test.mjs scripts/cli-archive-build.test.mjs scripts/release-credits.test.mjs scripts/update-contributors.test.mjs scripts/update-acknowledgments.test.mjs scripts/markdown-refs.test.mjs scripts/studio-release-notes.test.mjs scripts/publish-cli-pointer.test.mjs scripts/studio-release-flags.test.mjs scripts/studio-notes-publish.test.mjs scripts/withdraw-version.test.mjs scripts/brew-cask-guard.test.mjs scripts/feedback-sync.test.mjs npm/publish.test.mjs
|
||||
- run: node --test scripts/release-credits.test.mjs scripts/markdown-refs.test.mjs scripts/studio-release-notes.test.mjs scripts/publish-cli-pointer.test.mjs npm/publish.test.mjs
|
||||
|
||||
# The ruleset requires the per-OS check names (test (ubuntu-latest) etc.).
|
||||
# A matrix job skipped at job level reports no per-leg checks at all, so
|
||||
@@ -97,9 +89,6 @@ jobs:
|
||||
# detector reports the diff is unrelated, every step skips and each leg
|
||||
# reports success in seconds. `always()` also keeps the legs alive when
|
||||
# the changes job itself fails (fail-open: an empty output != 'false').
|
||||
# A pull request with no platform-relevant change moves the macOS and
|
||||
# Windows legs onto an ubuntu runner with every step skipped: the check
|
||||
# names the ruleset requires still report, without a scarce runner.
|
||||
test:
|
||||
needs: changes
|
||||
if: always()
|
||||
@@ -107,9 +96,9 @@ jobs:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
os: [ubuntu-latest, macos-latest, windows-latest]
|
||||
runs-on: ${{ github.event_name == 'pull_request' && needs.changes.outputs.platform == 'false' && 'ubuntu-latest' || matrix.os }}
|
||||
runs-on: ${{ matrix.os }}
|
||||
env:
|
||||
RUN_STEPS: ${{ github.event_name != 'pull_request' || (needs.changes.outputs.code != 'false' && (matrix.os == 'ubuntu-latest' || needs.changes.outputs.platform != 'false')) }}
|
||||
RUN_STEPS: ${{ github.event_name != 'pull_request' || needs.changes.outputs.code != 'false' }}
|
||||
steps:
|
||||
- if: env.RUN_STEPS == 'true'
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
@@ -186,9 +175,12 @@ jobs:
|
||||
# budget above normal 7-9 minute runner variance; each package test binary
|
||||
# remains independently bounded by Go's per-package timeout below.
|
||||
#
|
||||
# The per-package timeout is 18m: internal/assembly/boot took 478-717s
|
||||
# (12 push runs) on the Windows runner, so 18m is 1.5x the worst. A hang
|
||||
# still trips the step budget (25m smoke, 35m full).
|
||||
# That timeout is 8m, not 3m: across four consecutive runs of the same
|
||||
# code, internal/assembly/boot measured 133s, 146s, 162s and then hit 180s, and
|
||||
# internal/runtime/agent 82s, 92s, 144s and 180s — a 2.2x spread from runner
|
||||
# variance alone, with four heavy packages sharing four vCPUs. At 3m the
|
||||
# step failed on slow runners without a code change; a hang still trips
|
||||
# the 15 minute job budget above.
|
||||
#
|
||||
# Every root-module `go test` that runs on Windows passes -count=1. With
|
||||
# the result cache on, go test records each file and environment access
|
||||
@@ -199,11 +191,11 @@ jobs:
|
||||
# internal/base/testenv holds every such step to it.
|
||||
- name: test (Windows smoke)
|
||||
if: env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request'
|
||||
timeout-minutes: 25
|
||||
timeout-minutes: 15
|
||||
env:
|
||||
REASONIX_RELEASE_CACHE_GUARD: "1"
|
||||
WINDOWS_SANDBOX_WAIT_MS: "20000"
|
||||
run: go test -count=1 -p 4 -timeout=18m ./internal/runtime/agent/... ./internal/platform/appidentity/... ./internal/assembly/boot/... ./internal/state/checkpoint/... ./internal/frontend/cli/... ./internal/session/control/... ./internal/base/filelock/... ./internal/base/fileutil/... ./internal/ext/hook/... ./internal/state/instruction/... ./internal/ext/mcplaunch/... ./internal/platform/notify/... ./internal/base/proc/... ./internal/platform/remote/... ./internal/platform/repair/... ./internal/safety/sandbox/... ./internal/base/sysproxy/... ./internal/state/workspacelease/... ./cmd/...
|
||||
run: go test -count=1 -p 4 -timeout=8m ./internal/runtime/agent/... ./internal/platform/appidentity/... ./internal/assembly/boot/... ./internal/state/checkpoint/... ./internal/frontend/cli/... ./internal/session/control/... ./internal/base/filelock/... ./internal/base/fileutil/... ./internal/ext/hook/... ./internal/state/instruction/... ./internal/ext/mcplaunch/... ./internal/platform/notify/... ./internal/base/proc/... ./internal/platform/remote/... ./internal/platform/repair/... ./internal/safety/sandbox/... ./internal/base/sysproxy/... ./internal/state/workspacelease/... ./cmd/...
|
||||
|
||||
# The general Windows PR smoke list intentionally omits internal/contract/tool.
|
||||
# Keep the session-temp portability contract covered without widening the
|
||||
@@ -222,12 +214,14 @@ jobs:
|
||||
timeout-minutes: 5
|
||||
run: go test -count=1 -timeout=3m -run '^TestBashPowerShellExecuteDetailedContract$|^TestBashPowerShell51PreflightRejectsAndAndDetailed$|^TestBashPowerShellOutputIsUTF8$|^TestBashPowerShellSurfacesNonZeroExit$|^TestBashPowerShellRejectsChaining$' ./internal/tools/builtin
|
||||
|
||||
# Full push suite uses the same 18m package timeout as Windows smoke and a
|
||||
# 35m step budget above it. -count=1 as at the smoke step. A failure prints
|
||||
# the failed tests' output and the 15 slowest boot tests.
|
||||
# Full push suite still uses the same 8m package budget as Windows smoke:
|
||||
# agent/boot/control already hit 180s package timeouts under runner load,
|
||||
# so 3m fails on slow machines without a code regression. It runs with
|
||||
# -count=1 for the reason given at the smoke step. A hang still dies here
|
||||
# rather than on a package.
|
||||
- name: test (full)
|
||||
if: env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name != 'pull_request'
|
||||
timeout-minutes: 35
|
||||
timeout-minutes: 30
|
||||
env:
|
||||
# Run the prompt-cache prefix-stability guard (TestCacheHit*) in CI: a
|
||||
# regression there silently tanks the cache hit rate the project is
|
||||
@@ -236,25 +230,7 @@ jobs:
|
||||
# Bound sandbox helper children in Windows tests so a failed OS-level
|
||||
# launch cannot pin the Actions step after Go's package timeout fires.
|
||||
WINDOWS_SANDBOX_WAIT_MS: "20000"
|
||||
shell: bash
|
||||
run: |
|
||||
go test -count=1 -p 4 -timeout=18m -json ./... | tee "$RUNNER_TEMP/full.jsonl" |
|
||||
jq -rj --unbuffered 'select(.Action == "output" and .Test == null) | .Output'
|
||||
|
||||
- name: test (full) failure detail
|
||||
if: failure() && env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name != 'pull_request'
|
||||
shell: bash
|
||||
run: |
|
||||
f="$RUNNER_TEMP/full.jsonl"
|
||||
[ -s "$f" ] || exit 0
|
||||
echo "::group::output of failed tests"
|
||||
jq -rn '[inputs | select(.Test != null)] | group_by(.Package + "/" + .Test)
|
||||
| map(select(any(.[]; .Action == "fail")))[]
|
||||
| map(select(.Action == "output") | .Output) | join("")' "$f" || true
|
||||
echo "::endgroup::"
|
||||
echo "slowest internal/assembly/boot tests (seconds, name):"
|
||||
jq -rn '[inputs | select((.Package | endswith("/internal/assembly/boot")) and .Test != null and .Action == "pass")]
|
||||
| sort_by(-.Elapsed)[:15][] | "\(.Elapsed)\t\(.Test)"' "$f" || true
|
||||
run: go test -count=1 -p 4 -timeout=8m ./...
|
||||
|
||||
race:
|
||||
needs: changes
|
||||
@@ -309,9 +285,9 @@ jobs:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
os: [ubuntu-latest, macos-latest, windows-latest]
|
||||
runs-on: ${{ github.event_name == 'pull_request' && needs.changes.outputs.platform == 'false' && 'ubuntu-latest' || matrix.os }}
|
||||
runs-on: ${{ matrix.os }}
|
||||
env:
|
||||
RUN_STEPS: ${{ github.event_name != 'pull_request' || (needs.changes.outputs.sdk != 'false' && (matrix.os == 'ubuntu-latest' || needs.changes.outputs.platform != 'false')) }}
|
||||
RUN_STEPS: ${{ github.event_name != 'pull_request' || needs.changes.outputs.sdk != 'false' }}
|
||||
defaults:
|
||||
run:
|
||||
working-directory: sdk/go
|
||||
@@ -416,7 +392,7 @@ jobs:
|
||||
# compile it against a real macOS toolchain.
|
||||
desktop-macos:
|
||||
needs: changes
|
||||
if: always() && (github.event_name != 'pull_request' || (needs.changes.outputs.desktop != 'false' && needs.changes.outputs.platform != 'false'))
|
||||
if: always() && (github.event_name != 'pull_request' || needs.changes.outputs.desktop != 'false')
|
||||
runs-on: macos-latest
|
||||
defaults:
|
||||
run:
|
||||
@@ -444,7 +420,7 @@ jobs:
|
||||
# (//go:build windows), so the ubuntu leg above never compiles them.
|
||||
desktop-windows:
|
||||
needs: changes
|
||||
if: always() && (github.event_name != 'pull_request' || (needs.changes.outputs.desktop != 'false' && needs.changes.outputs.platform != 'false'))
|
||||
if: always() && (github.event_name != 'pull_request' || needs.changes.outputs.desktop != 'false')
|
||||
runs-on: windows-latest
|
||||
defaults:
|
||||
run:
|
||||
@@ -515,7 +491,6 @@ jobs:
|
||||
run: |
|
||||
go run github.com/rhysd/actionlint/cmd/[email protected] \
|
||||
-ignore 'label "windows-11-arm" is unknown' \
|
||||
-ignore 'label "ubuntu-22.04-arm" is unknown' \
|
||||
.github/workflows/release-studio.yml \
|
||||
.github/workflows/studio-certum-signing-smoke.yml \
|
||||
.github/workflows/ci.yml
|
||||
@@ -555,7 +530,7 @@ jobs:
|
||||
run: go test -coverprofile=coverage.out -covermode=atomic ./...
|
||||
|
||||
- name: upload coverage
|
||||
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
|
||||
with:
|
||||
name: coverage-report
|
||||
path: coverage.out
|
||||
|
||||
@@ -43,20 +43,20 @@ jobs:
|
||||
if: matrix.branch != ''
|
||||
run: echo "sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT"
|
||||
- name: Initialize CodeQL
|
||||
uses: github/codeql-action/init@24c54180a607b1449ed407dd24f251e4e9147c8d # v4.38.3
|
||||
uses: github/codeql-action/init@2892aa5e19bbd11bc0cff5427e3b750a04d9e3c2 # v4.38.2
|
||||
with:
|
||||
languages: ${{ matrix.language }}
|
||||
build-mode: ${{ matrix.language == 'go' && 'autobuild' || 'none' }}
|
||||
- name: Perform CodeQL Analysis
|
||||
if: matrix.branch == ''
|
||||
uses: github/codeql-action/analyze@24c54180a607b1449ed407dd24f251e4e9147c8d # v4.38.3
|
||||
uses: github/codeql-action/analyze@2892aa5e19bbd11bc0cff5427e3b750a04d9e3c2 # v4.38.2
|
||||
with:
|
||||
category: "/language:${{ matrix.language }}"
|
||||
# Results from a checked-out line are filed under that line's ref, not the
|
||||
# default branch the schedule happened to run from.
|
||||
- name: Perform CodeQL Analysis (${{ matrix.branch }})
|
||||
if: matrix.branch != ''
|
||||
uses: github/codeql-action/analyze@24c54180a607b1449ed407dd24f251e4e9147c8d # v4.38.3
|
||||
uses: github/codeql-action/analyze@2892aa5e19bbd11bc0cff5427e3b750a04d9e3c2 # v4.38.2
|
||||
with:
|
||||
category: "/language:${{ matrix.language }}"
|
||||
ref: refs/heads/${{ matrix.branch }}
|
||||
|
||||
@@ -24,7 +24,7 @@ jobs:
|
||||
with:
|
||||
version: 10
|
||||
run_install: false
|
||||
- uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7
|
||||
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
|
||||
with:
|
||||
node-version: "24"
|
||||
cache: pnpm
|
||||
|
||||
@@ -20,7 +20,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
- uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7
|
||||
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
|
||||
with:
|
||||
node-version: '22'
|
||||
cache: npm
|
||||
@@ -36,6 +36,16 @@ jobs:
|
||||
env:
|
||||
CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
|
||||
run: npm run migrate:diagnostics-v2
|
||||
- name: Apply and verify feedback D1 migration
|
||||
working-directory: workers/crash-report
|
||||
env:
|
||||
CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
|
||||
run: |
|
||||
npx wrangler d1 execute reasonix-crash --remote --file=migrate-feedback.sql
|
||||
out=$(npx wrangler d1 execute reasonix-crash --remote --json --command "SELECT name FROM sqlite_master WHERE name IN ('feedback','feedback_status_created','feedback_install_created')")
|
||||
for name in feedback feedback_status_created feedback_install_created; do
|
||||
printf '%s' "$out" | grep -q "\"$name\"" || { echo "missing $name after migration"; exit 1; }
|
||||
done
|
||||
- name: Deploy
|
||||
working-directory: workers/crash-report
|
||||
env:
|
||||
|
||||
@@ -24,7 +24,7 @@ jobs:
|
||||
with:
|
||||
version: 10
|
||||
run_install: false
|
||||
- uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7
|
||||
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
|
||||
with:
|
||||
node-version: "24"
|
||||
cache: pnpm
|
||||
|
||||
@@ -1,44 +0,0 @@
|
||||
name: Deploy ops events relay
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: deploy-ops-events
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
deploy:
|
||||
if: github.ref == 'refs/heads/platform'
|
||||
runs-on: ubuntu-latest
|
||||
environment: production
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
- uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7
|
||||
with:
|
||||
node-version: '22'
|
||||
cache: npm
|
||||
cache-dependency-path: workers/ops-events/package-lock.json
|
||||
- name: Validate
|
||||
working-directory: workers/ops-events
|
||||
run: |
|
||||
npm ci
|
||||
npm run typecheck
|
||||
npm test
|
||||
- name: Deploy
|
||||
working-directory: workers/ops-events
|
||||
env:
|
||||
CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
|
||||
run: npx wrangler deploy
|
||||
- name: Sync secrets
|
||||
working-directory: workers/ops-events
|
||||
env:
|
||||
CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
|
||||
GITHUB_WEBHOOK_SECRET: ${{ secrets.OPS_GITHUB_WEBHOOK_SECRET }}
|
||||
OPS_LISTEN_TOKEN: ${{ secrets.OPS_LISTEN_TOKEN }}
|
||||
OPS_EMIT_TOKEN: ${{ secrets.OPS_EMIT_TOKEN }}
|
||||
run: |
|
||||
node -e 'process.stdout.write(JSON.stringify({GITHUB_WEBHOOK_SECRET:process.env.GITHUB_WEBHOOK_SECRET||null,OPS_LISTEN_TOKEN:process.env.OPS_LISTEN_TOKEN||null,OPS_EMIT_TOKEN:process.env.OPS_EMIT_TOKEN||null}))' | npx wrangler secret bulk
|
||||
@@ -23,12 +23,12 @@ jobs:
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
|
||||
- uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413 # v6.1.0
|
||||
- uses: pnpm/action-setup@0ebf47130e4866e96fce0953f49152a61190b271 # v6.0.9
|
||||
with:
|
||||
version: 10
|
||||
run_install: false
|
||||
|
||||
- uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7
|
||||
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
|
||||
with:
|
||||
node-version: "24"
|
||||
cache: pnpm
|
||||
|
||||
@@ -1,77 +0,0 @@
|
||||
name: Sync in-app feedback
|
||||
|
||||
# Turns feedback submitted from Reasonix Studio into issues and reports issue
|
||||
# state (in progress, fixed in a tag, won't fix, duplicate) back to the worker,
|
||||
# and mirrors the reporter's replies onto their issue as bot comments.
|
||||
# Text from the public reaches the script only through the worker's JSON and
|
||||
# the GitHub API, never through a shell command line or an expression.
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: '*/10 * * * *'
|
||||
- cron: '5 3 * * *'
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
dry_run:
|
||||
description: Print what would happen without creating issues or writing back
|
||||
required: false
|
||||
default: false
|
||||
type: boolean
|
||||
full:
|
||||
description: Re-read every open entry instead of only recently changed issues
|
||||
required: false
|
||||
default: false
|
||||
type: boolean
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
issues: write
|
||||
|
||||
concurrency:
|
||||
group: feedback-sync
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
sync:
|
||||
if: github.repository == 'esengine/DeepSeek-Reasonix'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 8
|
||||
steps:
|
||||
- name: Check secret
|
||||
id: gate
|
||||
env:
|
||||
FEEDBACK_ADMIN_TOKEN: ${{ secrets.FEEDBACK_ADMIN_TOKEN }}
|
||||
run: |
|
||||
if [ -z "$FEEDBACK_ADMIN_TOKEN" ]; then
|
||||
echo "::notice::FEEDBACK_ADMIN_TOKEN is not set; feedback sync skipped"
|
||||
echo "run=false" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "run=true" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
- if: steps.gate.outputs.run == 'true'
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
with:
|
||||
# git tag --contains needs full history; the tag API would cost a request per entry.
|
||||
fetch-depth: 0
|
||||
persist-credentials: false
|
||||
- if: steps.gate.outputs.run == 'true'
|
||||
uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7
|
||||
with:
|
||||
node-version: '22'
|
||||
- name: Sync
|
||||
if: steps.gate.outputs.run == 'true'
|
||||
env:
|
||||
FEEDBACK_ADMIN_TOKEN: ${{ secrets.FEEDBACK_ADMIN_TOKEN }}
|
||||
# crash.reasonix.io answers runner IPs with a managed challenge; workers.dev serves only the converter's admin paths.
|
||||
FEEDBACK_BASE_URL: https://reasonix-crash-report.reasonix.workers.dev
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
DRY_RUN: ${{ github.event.inputs.dry_run || 'false' }}
|
||||
FULL: ${{ github.event.inputs.full || 'false' }}
|
||||
SCHEDULE: ${{ github.event.schedule }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
args=()
|
||||
if [ "$DRY_RUN" = "true" ]; then args+=(--dry-run); fi
|
||||
# The daily schedule sweeps everything the change window could have missed.
|
||||
if [ "$FULL" = "true" ] || [ "$SCHEDULE" = "5 3 * * *" ]; then args+=(--full); fi
|
||||
node scripts/feedback-sync.mjs "${args[@]}"
|
||||
@@ -5,12 +5,10 @@ run-name: Release Studio ${{ inputs.tag || github.ref_name }}
|
||||
# (studio-v*), its own environment gate, and its own rollback catalog at
|
||||
# dl.reasonix.io/studio/versions.json.
|
||||
#
|
||||
# What it must never touch: the root dl.reasonix.io/versions.json, which is the
|
||||
# desktop line's catalog. Studio artifacts land under the tag prefix, which
|
||||
# cannot collide because the tag name itself differs. The highest stable tag
|
||||
# is the repository's GitHub latest and a candidate is a prerelease; the GitHub
|
||||
# release carries no root latest.json, because the 1.x updaters' last-resort
|
||||
# URL is releases/latest/download/latest.json and must keep answering 404.
|
||||
# What it must never touch: the repository-wide "latest" release (every Studio
|
||||
# release is a prerelease) and the root dl.reasonix.io/versions.json, which is
|
||||
# the desktop line's catalog. Studio artifacts land under the tag prefix, which
|
||||
# cannot collide because the tag name itself differs.
|
||||
#
|
||||
# Windows artifacts are Authenticode-signed with a Certum SimplySign cloud
|
||||
# certificate in two stages: every PE file in the application tree, then the
|
||||
@@ -76,7 +74,8 @@ jobs:
|
||||
fi
|
||||
version="${tag#studio-}"
|
||||
# Semver's prerelease segment, group 4, read from the grammar rather
|
||||
# than the spelling.
|
||||
# than the spelling. Every Studio release carries GitHub's prerelease
|
||||
# flag, so the tag-level distinction needs a name of its own.
|
||||
candidate=false
|
||||
if [[ "${version#v}" =~ $semver ]] && [ -n "${BASH_REMATCH[4]}" ]; then
|
||||
candidate=true
|
||||
@@ -149,21 +148,10 @@ jobs:
|
||||
# vocabularies disagree (x64 / amd64) and a reviewer has to see both.
|
||||
# Absent means the runner's own, which is what Windows and Linux build.
|
||||
# artifact is the exact name publish downloads this leg's packages by.
|
||||
# goarch and unpacked belong to the Windows legs: the release
|
||||
# architecture the package is named for, and the directory
|
||||
# electron-builder leaves its bundle in (win-unpacked for x64 alone).
|
||||
include:
|
||||
- platform: windows/amd64
|
||||
runner: windows-latest
|
||||
goarch: amd64
|
||||
unpacked: win-unpacked
|
||||
artifact: studio-build-windows-amd64
|
||||
- platform: windows/arm64
|
||||
runner: windows-11-arm
|
||||
ebarch: arm64
|
||||
goarch: arm64
|
||||
unpacked: win-arm64-unpacked
|
||||
artifact: studio-build-windows-arm64
|
||||
- platform: darwin/amd64
|
||||
runner: macos-latest
|
||||
ebarch: x64
|
||||
@@ -175,9 +163,6 @@ jobs:
|
||||
- platform: linux/amd64
|
||||
runner: ubuntu-22.04
|
||||
artifact: studio-build-linux-amd64
|
||||
- platform: linux/arm64
|
||||
runner: ubuntu-22.04-arm
|
||||
artifact: studio-build-linux-arm64
|
||||
steps:
|
||||
# Real-time scanning inspects every file the job writes, and packaging
|
||||
# writes a quarter of a gigabyte twice. The runner is thrown away after
|
||||
@@ -205,7 +190,7 @@ jobs:
|
||||
version: 10
|
||||
run_install: false
|
||||
|
||||
- uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7
|
||||
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
|
||||
with:
|
||||
node-version: "24"
|
||||
cache: pnpm
|
||||
@@ -213,22 +198,12 @@ jobs:
|
||||
desktop/frontend-next/pnpm-lock.yaml
|
||||
desktop/electron/pnpm-lock.yaml
|
||||
|
||||
# electron-builder shells out to dpkg-deb through fakeroot for the .deb and
|
||||
# to rpmbuild for the .rpm, and carries its own NSIS -- the toolchain the
|
||||
# Wails line installed by hand (GTK, WebKitGTK, nfpm, a pinned NSIS) is
|
||||
# not this shell's.
|
||||
# electron-builder shells out to dpkg-deb through fakeroot for the .deb,
|
||||
# and carries its own NSIS -- the toolchain the Wails line installed by
|
||||
# hand (GTK, WebKitGTK, nfpm, a pinned NSIS) is not this shell's.
|
||||
- name: Install Linux packaging deps
|
||||
if: runner.os == 'Linux'
|
||||
run: sudo apt-get update && sudo apt-get install -y fakeroot rpm cpio xvfb xauth policykit-1
|
||||
|
||||
# The fpm electron-builder carries is an x86_64 program, so on an ARM64
|
||||
# runner the .deb and .rpm are built with one installed from RubyGems.
|
||||
- name: Use a native fpm on ARM64 Linux
|
||||
if: runner.os == 'Linux' && runner.arch == 'ARM64'
|
||||
run: |
|
||||
sudo apt-get install -y ruby ruby-dev build-essential
|
||||
sudo gem install --no-document fpm -v 1.16.0
|
||||
echo "USE_SYSTEM_FPM=true" >> "$GITHUB_ENV"
|
||||
run: sudo apt-get update && sudo apt-get install -y fakeroot
|
||||
|
||||
# The same Developer ID and notary credentials the desktop line uses: one
|
||||
# certificate signs any bundle identifier its team owns, so Studio does not
|
||||
@@ -291,7 +266,6 @@ jobs:
|
||||
# kernel on a remote machine, and those routes name a release by it.
|
||||
env:
|
||||
REASONIX_VERSION: ${{ needs.resolve.outputs.version }}
|
||||
REASONIX_GOARCH: ${{ matrix.goarch }}
|
||||
run: pnpm install --frozen-lockfile && pnpm run build:host
|
||||
|
||||
# The helper the Linux self-update runs under pkexec, staged where the
|
||||
@@ -314,14 +288,6 @@ jobs:
|
||||
if [ -n "${{ matrix.ebarch }}" ]; then arch="--${{ matrix.ebarch }}"; fi
|
||||
pnpm exec electron-builder --dir $arch -c.extraMetadata.version="${version#v}"
|
||||
|
||||
# An emulated x64 program runs on ARM64 Windows, so a bundle that starts
|
||||
# proves nothing about its architecture: every PE image in it has to be
|
||||
# built for the architecture the package is named for.
|
||||
- name: Every PE image is built for the package's architecture
|
||||
if: runner.os == 'Windows'
|
||||
shell: bash
|
||||
run: node desktop/electron/packaging/pe.js "dist/${{ matrix.unpacked }}" "${{ matrix.goarch }}"
|
||||
|
||||
# Notarization happens to the bundle, so it sits between the two packaging
|
||||
# steps the way Authenticode does on Windows. Stapling here is what lets a
|
||||
# first launch work offline; a ticket Apple has but the bundle does not is
|
||||
@@ -372,31 +338,14 @@ jobs:
|
||||
pnpm exec electron-builder --prepackaged "../../$app" $arch \
|
||||
-c.extraMetadata.version="${version#v}"
|
||||
|
||||
- name: Verify both Linux package formats
|
||||
if: runner.os == 'Linux'
|
||||
shell: bash
|
||||
run: |
|
||||
platform="${{ matrix.platform }}"
|
||||
arch="${platform#linux/}"
|
||||
bash scripts/verify-linux-package.sh "dist/ReasonixStudio-linux-$arch.deb" "dist/ReasonixStudio-linux-$arch.rpm" "$arch"
|
||||
|
||||
- name: Accept the installed Linux package
|
||||
if: runner.os == 'Linux'
|
||||
shell: bash
|
||||
run: |
|
||||
platform="${{ matrix.platform }}"
|
||||
arch="${platform#linux/}"
|
||||
version="${{ needs.resolve.outputs.version }}"
|
||||
bash scripts/linux-install-smoke.sh "dist/ReasonixStudio-linux-$arch.deb" "$arch" "${version#v}"
|
||||
|
||||
# With signing on, this leg only hands the bundle on: the signing jobs
|
||||
# run nothing but the signer, and windows-package packages in between.
|
||||
- name: Upload the unsigned Windows bundle for signing
|
||||
if: runner.os == 'Windows' && env.SIGN_WINDOWS == 'true'
|
||||
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: unsigned-studio-windows-${{ matrix.goarch }}-bundle
|
||||
path: dist/${{ matrix.unpacked }}
|
||||
name: unsigned-studio-windows-bundle
|
||||
path: dist/win-unpacked
|
||||
if-no-files-found: error
|
||||
retention-days: 1
|
||||
|
||||
@@ -407,26 +356,19 @@ jobs:
|
||||
if: runner.os == 'Windows' && env.SIGN_WINDOWS != 'true'
|
||||
working-directory: desktop/electron
|
||||
shell: bash
|
||||
env:
|
||||
# The arm64 installer has to be packed with the BCJ filter: 7-Zip
|
||||
# applies an ARM64 filter to ARM64 executables by default, the NSIS
|
||||
# extractor cannot decode it, and the installer then exits 0 with
|
||||
# every executable missing.
|
||||
ELECTRON_BUILDER_7Z_FILTER: ${{ matrix.goarch == 'arm64' && 'BCJ' || '' }}
|
||||
run: |
|
||||
version="${{ needs.resolve.outputs.version }}"
|
||||
arch="--${{ matrix.ebarch || 'x64' }}"
|
||||
rm -rf ../../zip-src ../../dist-zip
|
||||
cp -r ../../dist/${{ matrix.unpacked }} ../../zip-src
|
||||
pnpm exec electron-builder --prepackaged ../../zip-src --win zip $arch \
|
||||
cp -r ../../dist/win-unpacked ../../zip-src
|
||||
pnpm exec electron-builder --prepackaged ../../zip-src --win zip \
|
||||
-c.extraMetadata.version="${version#v}" -c.directories.output=../../dist-zip &
|
||||
zip_pid=$!
|
||||
pnpm exec electron-builder --prepackaged ../../dist/${{ matrix.unpacked }} --win nsis $arch \
|
||||
pnpm exec electron-builder --prepackaged ../../dist/win-unpacked --win nsis \
|
||||
-c.extraMetadata.version="${version#v}"
|
||||
wait "$zip_pid"
|
||||
mv ../../dist-zip/ReasonixStudio-windows-${{ matrix.goarch }}.zip ../../dist/
|
||||
mv ../../dist-zip/ReasonixStudio-windows-amd64.zip ../../dist/
|
||||
|
||||
- uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
|
||||
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
if: runner.os != 'Windows' || env.SIGN_WINDOWS != 'true'
|
||||
with:
|
||||
name: ${{ matrix.artifact }}
|
||||
@@ -440,36 +382,22 @@ jobs:
|
||||
# package check run between them on their own runners with no secrets. Both
|
||||
# signing jobs share one group with the Studio smoke test, so no two runs
|
||||
# drive the same SimplySign account at once.
|
||||
#
|
||||
# Each signing stage is a matrix over the Windows architectures with one tree
|
||||
# per job. max-parallel: 1 keeps the legs strictly serial, which the shared
|
||||
# certum-signing group needs: a group holds one running and one waiting job,
|
||||
# so two legs started together would have the waiting one cancelled by the
|
||||
# next arrival. A failed leg reruns alone; the others' results are kept.
|
||||
windows-sign-payload:
|
||||
name: sign the Windows PE files (Certum, ${{ matrix.arch }})
|
||||
name: sign the Windows PE files (Certum)
|
||||
needs: [resolve, signing-contract, build]
|
||||
if: vars.STUDIO_SIGNING_ENABLED == 'true'
|
||||
runs-on: windows-2022
|
||||
timeout-minutes: 30
|
||||
strategy:
|
||||
fail-fast: false
|
||||
max-parallel: 1
|
||||
matrix:
|
||||
arch: [amd64, arm64]
|
||||
environment: studio-release
|
||||
permissions:
|
||||
contents: read
|
||||
concurrency:
|
||||
group: certum-signing
|
||||
cancel-in-progress: false
|
||||
# One output per architecture, because matrix legs share a job's output map:
|
||||
# a leg writes only its own and an empty value never replaces another's.
|
||||
# Recorded here because no later job can write this job's outputs: the
|
||||
# installer signing job refuses a payload artifact that no longer hashes to it.
|
||||
outputs:
|
||||
payload-digest-amd64: ${{ matrix.arch == 'amd64' && steps.verify.outputs.digest || '' }}
|
||||
payload-digest-arm64: ${{ matrix.arch == 'arm64' && steps.verify.outputs.digest || '' }}
|
||||
payload-digest: ${{ steps.verify.outputs.digest }}
|
||||
steps:
|
||||
# Only the workflow's own commit is checked out: these jobs run the signer
|
||||
# and its checks, never anything the release tag builds.
|
||||
@@ -508,9 +436,9 @@ jobs:
|
||||
}
|
||||
if ($failed) { exit 1 }
|
||||
|
||||
- uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
|
||||
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
name: unsigned-studio-windows-${{ matrix.arch }}-bundle
|
||||
name: unsigned-studio-windows-bundle
|
||||
path: signing/app
|
||||
|
||||
# Stage one: every PE file in the application tree that Microsoft has
|
||||
@@ -557,15 +485,14 @@ jobs:
|
||||
$ErrorActionPreference = 'Stop'
|
||||
./scripts/verify-windows-authenticode.ps1 `
|
||||
-PayloadDirectory signing/app `
|
||||
-Architecture ${{ matrix.arch }} `
|
||||
-DigestPath "$env:RUNNER_TEMP/payload-digest" `
|
||||
-ExpectedThumbprint $env:CERTUM_KEY_ID `
|
||||
-ExpectedSubject $env:EXPECTED_SUBJECT
|
||||
"digest=$(Get-Content -Raw "$env:RUNNER_TEMP/payload-digest")" >> $env:GITHUB_OUTPUT
|
||||
|
||||
- uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
|
||||
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: signed-studio-windows-payload-${{ matrix.arch }}
|
||||
name: signed-studio-windows-payload
|
||||
path: signing/app
|
||||
if-no-files-found: error
|
||||
retention-days: 1
|
||||
@@ -573,20 +500,10 @@ jobs:
|
||||
# No environment and no secrets: electron-builder and its dependencies run
|
||||
# here, on a runner that never held a signing session.
|
||||
windows-package:
|
||||
name: package Windows from the signed tree (${{ matrix.arch }})
|
||||
name: package Windows from the signed tree
|
||||
needs: [resolve, windows-sign-payload]
|
||||
runs-on: windows-latest
|
||||
timeout-minutes: 30
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- arch: amd64
|
||||
ebarch: x64
|
||||
unpacked: win-unpacked
|
||||
- arch: arm64
|
||||
ebarch: arm64
|
||||
unpacked: win-arm64-unpacked
|
||||
permissions:
|
||||
contents: read
|
||||
steps:
|
||||
@@ -605,7 +522,7 @@ jobs:
|
||||
version: 10
|
||||
run_install: false
|
||||
|
||||
- uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7
|
||||
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
|
||||
with:
|
||||
node-version: "24"
|
||||
cache: pnpm
|
||||
@@ -617,10 +534,10 @@ jobs:
|
||||
run: pnpm install --frozen-lockfile
|
||||
|
||||
# The signed payload is the whole bundle, so it is packaged as it came.
|
||||
- uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
|
||||
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
name: signed-studio-windows-payload-${{ matrix.arch }}
|
||||
path: dist/${{ matrix.unpacked }}
|
||||
name: signed-studio-windows-payload
|
||||
path: dist/win-unpacked
|
||||
|
||||
# Both packages come from the signed tree, so the archive carries the
|
||||
# signed files too rather than a second unsigned copy. The zip is packed
|
||||
@@ -628,27 +545,24 @@ jobs:
|
||||
- name: Package Windows from the signed bundle
|
||||
working-directory: desktop/electron
|
||||
shell: bash
|
||||
env:
|
||||
# See the build job: the arm64 installer must be packed with BCJ.
|
||||
ELECTRON_BUILDER_7Z_FILTER: ${{ matrix.arch == 'arm64' && 'BCJ' || '' }}
|
||||
run: |
|
||||
version="${{ needs.resolve.outputs.version }}"
|
||||
rm -rf ../../zip-src ../../dist-zip
|
||||
cp -r ../../dist/${{ matrix.unpacked }} ../../zip-src
|
||||
pnpm exec electron-builder --prepackaged ../../zip-src --win zip --${{ matrix.ebarch }} \
|
||||
cp -r ../../dist/win-unpacked ../../zip-src
|
||||
pnpm exec electron-builder --prepackaged ../../zip-src --win zip \
|
||||
-c.extraMetadata.version="${version#v}" -c.directories.output=../../dist-zip &
|
||||
zip_pid=$!
|
||||
pnpm exec electron-builder --prepackaged ../../dist/${{ matrix.unpacked }} --win nsis --${{ matrix.ebarch }} \
|
||||
pnpm exec electron-builder --prepackaged ../../dist/win-unpacked --win nsis \
|
||||
-c.extraMetadata.version="${version#v}"
|
||||
wait "$zip_pid"
|
||||
mv ../../dist-zip/ReasonixStudio-windows-${{ matrix.arch }}.zip ../../dist/
|
||||
mv ../../dist-zip/ReasonixStudio-windows-amd64.zip ../../dist/
|
||||
|
||||
- uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
|
||||
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: packaged-studio-windows-${{ matrix.arch }}
|
||||
name: packaged-studio-windows
|
||||
path: |
|
||||
dist/ReasonixStudio-windows-${{ matrix.arch }}-installer.exe
|
||||
dist/ReasonixStudio-windows-${{ matrix.arch }}.zip
|
||||
dist/ReasonixStudio-windows-amd64-installer.exe
|
||||
dist/ReasonixStudio-windows-amd64.zip
|
||||
if-no-files-found: error
|
||||
retention-days: 1
|
||||
|
||||
@@ -657,35 +571,29 @@ jobs:
|
||||
# secrets, never in a job that holds the Certum credentials. Its outputs are
|
||||
# the hashes of the two files it checked, which the signing job requires.
|
||||
windows-verify-package:
|
||||
name: check the Windows packages against the signed tree (${{ matrix.arch }})
|
||||
name: check the Windows packages against the signed tree
|
||||
needs: [resolve, windows-sign-payload, windows-package]
|
||||
runs-on: windows-2022
|
||||
timeout-minutes: 30
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
arch: [amd64, arm64]
|
||||
permissions:
|
||||
contents: read
|
||||
outputs:
|
||||
installer-sha256-amd64: ${{ matrix.arch == 'amd64' && steps.check.outputs.installer-sha256 || '' }}
|
||||
installer-sha256-arm64: ${{ matrix.arch == 'arm64' && steps.check.outputs.installer-sha256 || '' }}
|
||||
archive-sha256-amd64: ${{ matrix.arch == 'amd64' && steps.check.outputs.archive-sha256 || '' }}
|
||||
archive-sha256-arm64: ${{ matrix.arch == 'arm64' && steps.check.outputs.archive-sha256 || '' }}
|
||||
installer-sha256: ${{ steps.check.outputs.installer-sha256 }}
|
||||
archive-sha256: ${{ steps.check.outputs.archive-sha256 }}
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
with:
|
||||
ref: ${{ github.workflow_sha }}
|
||||
persist-credentials: false
|
||||
|
||||
- uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
|
||||
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
name: signed-studio-windows-payload-${{ matrix.arch }}
|
||||
name: signed-studio-windows-payload
|
||||
path: payload
|
||||
|
||||
- uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
|
||||
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
name: packaged-studio-windows-${{ matrix.arch }}
|
||||
name: packaged-studio-windows
|
||||
path: dist
|
||||
|
||||
# 7-Zip is the runner image's, not the tag's.
|
||||
@@ -693,26 +601,20 @@ jobs:
|
||||
id: check
|
||||
shell: pwsh
|
||||
env:
|
||||
PAYLOAD_DIGEST: ${{ needs.windows-sign-payload.outputs[format('payload-digest-{0}', matrix.arch)] }}
|
||||
PAYLOAD_DIGEST: ${{ needs.windows-sign-payload.outputs.payload-digest }}
|
||||
run: |
|
||||
./scripts/verify-windows-package.ps1 `
|
||||
-PayloadDirectory payload `
|
||||
-ExpectedPayloadDigest $env:PAYLOAD_DIGEST `
|
||||
-InstallerPath "dist/ReasonixStudio-windows-${{ matrix.arch }}-installer.exe" `
|
||||
-PortableArchivePath "dist/ReasonixStudio-windows-${{ matrix.arch }}.zip" `
|
||||
-OutputPath $env:GITHUB_OUTPUT `
|
||||
-Architecture ${{ matrix.arch }}
|
||||
-InstallerPath "dist/ReasonixStudio-windows-amd64-installer.exe" `
|
||||
-PortableArchivePath "dist/ReasonixStudio-windows-amd64.zip" `
|
||||
-OutputPath $env:GITHUB_OUTPUT
|
||||
|
||||
windows-sign-installer:
|
||||
name: sign the Windows installer (Certum, ${{ matrix.arch }})
|
||||
name: sign the Windows installer (Certum)
|
||||
needs: [resolve, windows-package, windows-verify-package]
|
||||
runs-on: windows-2022
|
||||
timeout-minutes: 30
|
||||
strategy:
|
||||
fail-fast: false
|
||||
max-parallel: 1
|
||||
matrix:
|
||||
arch: [amd64, arm64]
|
||||
environment: studio-release
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -721,8 +623,7 @@ jobs:
|
||||
cancel-in-progress: false
|
||||
# The signed installer's hash, which publish requires of what it downloads.
|
||||
outputs:
|
||||
installer-sha256-amd64: ${{ matrix.arch == 'amd64' && steps.signed.outputs.installer-sha256 || '' }}
|
||||
installer-sha256-arm64: ${{ matrix.arch == 'arm64' && steps.signed.outputs.installer-sha256 || '' }}
|
||||
installer-sha256: ${{ steps.signed.outputs.installer-sha256 }}
|
||||
steps:
|
||||
# Only the workflow's own commit is checked out: these jobs run the signer
|
||||
# and its checks, never anything the release tag builds.
|
||||
@@ -761,9 +662,9 @@ jobs:
|
||||
}
|
||||
if ($failed) { exit 1 }
|
||||
|
||||
- uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
|
||||
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
name: packaged-studio-windows-${{ matrix.arch }}
|
||||
name: packaged-studio-windows
|
||||
path: dist
|
||||
|
||||
# Only the two files windows-verify-package checked, byte for byte. This
|
||||
@@ -772,13 +673,13 @@ jobs:
|
||||
- name: Require the checked packages
|
||||
shell: pwsh
|
||||
env:
|
||||
INSTALLER_SHA256: ${{ needs.windows-verify-package.outputs[format('installer-sha256-{0}', matrix.arch)] }}
|
||||
ARCHIVE_SHA256: ${{ needs.windows-verify-package.outputs[format('archive-sha256-{0}', matrix.arch)] }}
|
||||
INSTALLER_SHA256: ${{ needs.windows-verify-package.outputs.installer-sha256 }}
|
||||
ARCHIVE_SHA256: ${{ needs.windows-verify-package.outputs.archive-sha256 }}
|
||||
run: |
|
||||
$ErrorActionPreference = 'Stop'
|
||||
$expected = [ordered]@{
|
||||
'ReasonixStudio-windows-${{ matrix.arch }}-installer.exe' = $env:INSTALLER_SHA256
|
||||
'ReasonixStudio-windows-${{ matrix.arch }}.zip' = $env:ARCHIVE_SHA256
|
||||
'ReasonixStudio-windows-amd64-installer.exe' = $env:INSTALLER_SHA256
|
||||
'ReasonixStudio-windows-amd64.zip' = $env:ARCHIVE_SHA256
|
||||
}
|
||||
$present = @(Get-ChildItem -LiteralPath dist -Recurse -Force | ForEach-Object { $_.Name })
|
||||
if (Compare-Object -CaseSensitive @($expected.Keys) $present) { throw "dist holds $($present -join ', ')" }
|
||||
@@ -800,7 +701,7 @@ jobs:
|
||||
shell: pwsh
|
||||
env:
|
||||
CERTUM_KEY_ID: ${{ secrets.CERTUM_KEY_ID }}
|
||||
run: ./scripts/sign-certum.ps1 -FilePath dist/ReasonixStudio-windows-${{ matrix.arch }}-installer.exe
|
||||
run: ./scripts/sign-certum.ps1 -FilePath dist/ReasonixStudio-windows-amd64-installer.exe
|
||||
|
||||
# The session signs for any process on the runner while it is up.
|
||||
- name: Disconnect from Certum
|
||||
@@ -816,7 +717,7 @@ jobs:
|
||||
EXPECTED_SUBJECT: ${{ vars.STUDIO_SIGNING_SUBJECT }}
|
||||
run: |
|
||||
$ErrorActionPreference = 'Stop'
|
||||
$installer = 'dist/ReasonixStudio-windows-${{ matrix.arch }}-installer.exe'
|
||||
$installer = 'dist/ReasonixStudio-windows-amd64-installer.exe'
|
||||
./scripts/verify-windows-authenticode.ps1 `
|
||||
-FilePath $installer `
|
||||
-ExpectedThumbprint $env:CERTUM_KEY_ID `
|
||||
@@ -824,62 +725,13 @@ jobs:
|
||||
$hash = (Get-FileHash -Algorithm SHA256 -LiteralPath $installer).Hash.ToLowerInvariant()
|
||||
"installer-sha256=$hash" >> $env:GITHUB_OUTPUT
|
||||
|
||||
- uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
|
||||
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: studio-windows-signed-${{ matrix.arch }}
|
||||
name: studio-windows-signed
|
||||
path: dist/ReasonixStudio-*
|
||||
if-no-files-found: error
|
||||
retention-days: 7
|
||||
|
||||
# Installs each Windows installer where it is meant to run (the same script
|
||||
# the Studio workflow runs on pull requests, against the unsigned installer).
|
||||
# The package check unpacks with the runner's modern 7-Zip, which reads
|
||||
# archives the NSIS extractor cannot, so only running the installer shows that
|
||||
# it installs every executable. No environment and no secrets.
|
||||
windows-install-smoke:
|
||||
name: install the Windows package (${{ matrix.arch }})
|
||||
needs: [resolve, build, windows-sign-installer]
|
||||
if: >-
|
||||
${{ !cancelled() && needs.resolve.result == 'success' && needs.build.result == 'success' &&
|
||||
(needs.windows-sign-installer.result == 'success' ||
|
||||
(vars.STUDIO_SIGNING_ENABLED != 'true' && needs.windows-sign-installer.result == 'skipped')) }}
|
||||
runs-on: ${{ matrix.runner }}
|
||||
timeout-minutes: 30
|
||||
permissions:
|
||||
contents: read
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- arch: amd64
|
||||
runner: windows-latest
|
||||
- arch: arm64
|
||||
runner: windows-11-arm
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
with:
|
||||
ref: ${{ needs.resolve.outputs.sha }}
|
||||
persist-credentials: false
|
||||
|
||||
- uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
|
||||
if: needs.windows-sign-installer.result == 'success'
|
||||
with:
|
||||
name: studio-windows-signed-${{ matrix.arch }}
|
||||
path: dist
|
||||
|
||||
- uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
|
||||
if: needs.windows-sign-installer.result != 'success'
|
||||
with:
|
||||
name: studio-build-windows-${{ matrix.arch }}
|
||||
path: dist
|
||||
|
||||
- name: Install, check and uninstall
|
||||
shell: pwsh
|
||||
run: |
|
||||
./scripts/install-smoke.ps1 `
|
||||
-Installer "dist/ReasonixStudio-windows-${{ matrix.arch }}-installer.exe" `
|
||||
-Architecture ${{ matrix.arch }}
|
||||
|
||||
# The kernel a Studio installs on another machine. This line publishes it
|
||||
# under its own tag, because the install routes name a release by the version
|
||||
# the local build reports — and a line that ships no archive leaves every one
|
||||
@@ -905,18 +757,14 @@ jobs:
|
||||
- name: Cross-compile and archive
|
||||
env:
|
||||
VERSION: ${{ needs.resolve.outputs.version }}
|
||||
SHA: ${{ needs.resolve.outputs.sha }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
mkdir -p dist
|
||||
built="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||||
pkg=reasonix/internal/tools/productdocs
|
||||
for target in darwin/amd64 darwin/arm64 linux/amd64 linux/arm64 windows/amd64 windows/arm64; do
|
||||
os="${target%/*}"; arch="${target#*/}"; ext=""
|
||||
[ "$os" = windows ] && ext=".exe"
|
||||
CGO_ENABLED=0 GOOS="$os" GOARCH="$arch" go build -trimpath \
|
||||
-ldflags "-s -w -X main.version=$VERSION -X main.gitCommit=${SHA:0:12} -X main.buildTimeUTC=$built -X $pkg.linkedVersion=$VERSION -X $pkg.linkedRevision=$SHA" \
|
||||
-o "reasonix$ext" ./cmd/reasonix
|
||||
-ldflags "-s -w -X main.version=$VERSION" -o "reasonix$ext" ./cmd/reasonix
|
||||
if [ "$os" = windows ]; then
|
||||
zip -qj "dist/reasonix-$os-$arch.zip" "reasonix$ext"
|
||||
else
|
||||
@@ -941,7 +789,7 @@ jobs:
|
||||
[ -f "dist/$want" ] || { echo "::error::no dist/$want"; exit 1; }
|
||||
done
|
||||
|
||||
- uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
|
||||
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: cli-archives
|
||||
path: dist/*
|
||||
@@ -950,13 +798,12 @@ jobs:
|
||||
|
||||
publish:
|
||||
name: sign, publish and mirror
|
||||
needs: [resolve, build, cli, windows-sign-payload, windows-package, windows-verify-package, windows-sign-installer, windows-install-smoke]
|
||||
needs: [resolve, build, cli, windows-sign-payload, windows-package, windows-verify-package, windows-sign-installer]
|
||||
# The Windows signing jobs may be skipped only while the switch is off; with
|
||||
# it on, any of them failing, cancelled or absent blocks the release.
|
||||
if: >-
|
||||
${{ !cancelled() && needs.resolve.result == 'success' && needs.build.result == 'success' &&
|
||||
needs.cli.result == 'success' && needs.windows-install-smoke.result == 'success' &&
|
||||
((needs.windows-sign-payload.result == 'success' &&
|
||||
needs.cli.result == 'success' && ((needs.windows-sign-payload.result == 'success' &&
|
||||
needs.windows-package.result == 'success' && needs.windows-verify-package.result == 'success' &&
|
||||
needs.windows-sign-installer.result == 'success') ||
|
||||
(vars.STUDIO_SIGNING_ENABLED != 'true' && needs.windows-sign-payload.result == 'skipped' &&
|
||||
@@ -1000,88 +847,63 @@ jobs:
|
||||
# Every package by its exact name, never a pattern: any job without
|
||||
# secrets can upload an artifact, and a pattern would publish it. Windows
|
||||
# comes from the installer signing job when it ran, else the build leg.
|
||||
- uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
|
||||
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
name: studio-build-darwin-amd64
|
||||
path: staging/studio-build-darwin-amd64
|
||||
|
||||
- uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
|
||||
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
name: studio-build-darwin-arm64
|
||||
path: staging/studio-build-darwin-arm64
|
||||
|
||||
- uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
|
||||
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
name: studio-build-linux-amd64
|
||||
path: staging/studio-build-linux-amd64
|
||||
|
||||
- uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
|
||||
with:
|
||||
name: studio-build-linux-arm64
|
||||
path: staging/studio-build-linux-arm64
|
||||
|
||||
- uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
|
||||
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
if: needs.windows-sign-installer.result == 'success'
|
||||
with:
|
||||
name: studio-windows-signed-amd64
|
||||
path: staging/studio-windows-amd64
|
||||
|
||||
- uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
|
||||
if: needs.windows-sign-installer.result == 'success'
|
||||
with:
|
||||
name: studio-windows-signed-arm64
|
||||
path: staging/studio-windows-arm64
|
||||
name: studio-windows-signed
|
||||
path: staging/studio-windows
|
||||
|
||||
# Any job can upload an artifact, so the signed Windows packages are
|
||||
# taken only if they are the bytes the signing jobs recorded: each
|
||||
# installer as signed, each zip as windows-verify-package checked it.
|
||||
# taken only if they are the bytes the signing jobs recorded: the
|
||||
# installer as signed, the zip as windows-verify-package checked it.
|
||||
- name: Require the recorded Windows packages
|
||||
if: needs.windows-sign-installer.result == 'success'
|
||||
env:
|
||||
INSTALLER_SHA256_AMD64: ${{ needs.windows-sign-installer.outputs[format('installer-sha256-{0}', 'amd64')] }}
|
||||
ARCHIVE_SHA256_AMD64: ${{ needs.windows-verify-package.outputs[format('archive-sha256-{0}', 'amd64')] }}
|
||||
INSTALLER_SHA256_ARM64: ${{ needs.windows-sign-installer.outputs[format('installer-sha256-{0}', 'arm64')] }}
|
||||
ARCHIVE_SHA256_ARM64: ${{ needs.windows-verify-package.outputs[format('archive-sha256-{0}', 'arm64')] }}
|
||||
INSTALLER_SHA256: ${{ needs.windows-sign-installer.outputs.installer-sha256 }}
|
||||
ARCHIVE_SHA256: ${{ needs.windows-verify-package.outputs.archive-sha256 }}
|
||||
run: |
|
||||
check() {
|
||||
local file="$1/$2" want="$3" got
|
||||
local file="staging/studio-windows/$1" want="$2" got
|
||||
if [[ ! "$want" =~ ^[0-9a-f]{64}$ ]]; then
|
||||
echo "::error title=studio-signing.package-hash-missing::no recorded SHA-256 for $2"
|
||||
echo "::error title=studio-signing.package-hash-missing::no recorded SHA-256 for $1"
|
||||
return 1
|
||||
fi
|
||||
got="$(sha256sum "$file" | awk '{print $1}')" || return 1
|
||||
if [ "$got" != "$want" ]; then
|
||||
echo "::error title=studio-signing.package-hash-mismatch::$2 is $got, the signing jobs recorded $want"
|
||||
echo "::error title=studio-signing.package-hash-mismatch::$1 is $got, the signing jobs recorded $want"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
for arch in amd64 arm64; do
|
||||
dir="staging/studio-windows-$arch"
|
||||
files="$(cd "$dir" && find . -type f | sed 's|^\./||' | LC_ALL=C sort | tr '\n' ' ')"
|
||||
if [ "$files" != "ReasonixStudio-windows-$arch-installer.exe ReasonixStudio-windows-$arch.zip " ]; then
|
||||
echo "::error title=studio-signing.package-set-mismatch::studio-windows-signed-$arch holds: $files"
|
||||
exit 1
|
||||
fi
|
||||
upper="${arch^^}"
|
||||
installer="INSTALLER_SHA256_$upper"
|
||||
archive="ARCHIVE_SHA256_$upper"
|
||||
check "$dir" "ReasonixStudio-windows-$arch-installer.exe" "${!installer}"
|
||||
check "$dir" "ReasonixStudio-windows-$arch.zip" "${!archive}"
|
||||
done
|
||||
files="$(cd staging/studio-windows && find . -type f | sed 's|^\./||' | LC_ALL=C sort | tr '\n' ' ')"
|
||||
if [ "$files" != "ReasonixStudio-windows-amd64-installer.exe ReasonixStudio-windows-amd64.zip " ]; then
|
||||
echo "::error title=studio-signing.package-set-mismatch::studio-windows-signed holds: $files"
|
||||
exit 1
|
||||
fi
|
||||
check ReasonixStudio-windows-amd64-installer.exe "$INSTALLER_SHA256"
|
||||
check ReasonixStudio-windows-amd64.zip "$ARCHIVE_SHA256"
|
||||
|
||||
- uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
|
||||
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
if: needs.windows-sign-installer.result != 'success'
|
||||
with:
|
||||
name: studio-build-windows-amd64
|
||||
path: staging/studio-windows-amd64
|
||||
path: staging/studio-windows
|
||||
|
||||
- uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
|
||||
if: needs.windows-sign-installer.result != 'success'
|
||||
with:
|
||||
name: studio-build-windows-arm64
|
||||
path: staging/studio-windows-arm64
|
||||
|
||||
- uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
|
||||
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
name: cli-archives
|
||||
path: staging/cli-archives
|
||||
@@ -1102,7 +924,30 @@ jobs:
|
||||
done < <(find staging/studio-* -type f -name 'ReasonixStudio-*' -print0
|
||||
find staging/cli-archives -type f -print0)
|
||||
ls -la dist
|
||||
bash scripts/verify-studio-artifacts.sh dist
|
||||
# Two Windows artifacts (installer, portable zip), two per macOS
|
||||
# architecture (dmg, zip) and the Linux .deb. Fewer means an artifact
|
||||
# was lost between jobs, and a partial release is worse than a failed
|
||||
# one.
|
||||
count=$(find dist -type f -name 'ReasonixStudio-*' | wc -l)
|
||||
if [ "$count" -lt 7 ]; then
|
||||
echo "::error::expected 7 artifacts (Windows installer + zip, macOS dmg + zip per architecture, Linux .deb), found $count"
|
||||
exit 1
|
||||
fi
|
||||
# Each platform's update channel, and there is exactly one per
|
||||
# platform: dpkg on Linux, the next installer on Windows, and the
|
||||
# bundle swapped out of the archive on macOS. Missing one degrades the
|
||||
# release silently instead of failing it — it strands that platform on
|
||||
# the version it is running.
|
||||
# Each macOS architecture by name, not darwin-*: a glob passes on
|
||||
# whichever one shipped, and the missing one is a platform left on
|
||||
# the version it is running with nothing saying so.
|
||||
for required in 'ReasonixStudio-linux-*.deb' 'ReasonixStudio-windows-*-installer.exe' \
|
||||
'ReasonixStudio-darwin-amd64.zip' 'ReasonixStudio-darwin-arm64.zip'; do
|
||||
find dist -type f -name "$required" | grep -q . || {
|
||||
echo "::error::no $required in the release; its packager did not run"
|
||||
exit 1
|
||||
}
|
||||
done
|
||||
|
||||
# The same minisign key the desktop line uses, because the public key that
|
||||
# verifies it is compiled into the client (update.PublicKey).
|
||||
@@ -1145,14 +990,12 @@ jobs:
|
||||
if [ "$HAS_R2" = "true" ]; then delta_dir=../delta; fi
|
||||
go run ./cmd/studio-manifest ../dist "${{ needs.resolve.outputs.version }}" "${{ needs.resolve.outputs.tag }}" $delta_dir
|
||||
|
||||
# The manifest stays out of the GitHub release (it is mirrored to R2
|
||||
# under <tag>/ below), and a stable release takes the repository's latest
|
||||
# only when it is the highest stable studio tag.
|
||||
# Always a prerelease: this line is not the public product line, and
|
||||
# GitHub never promotes a prerelease to the repository-wide "latest".
|
||||
- name: Publish GitHub release
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
TAG: ${{ needs.resolve.outputs.tag }}
|
||||
CANDIDATE: ${{ needs.resolve.outputs.candidate }}
|
||||
VERSION: ${{ needs.resolve.outputs.version }}
|
||||
SIGNED_WINDOWS: ${{ needs.windows-sign-installer.result == 'success' }}
|
||||
run: |
|
||||
@@ -1190,23 +1033,18 @@ jobs:
|
||||
fi
|
||||
{
|
||||
printf '%s\n\n' '---'
|
||||
printf "**macOS** 已用 Developer ID 签名并公证,打开时不需要清除隔离属性。%s**Linux** Debian 系从 \`.deb\` 安装,Fedora、RHEL、openSUSE 等从 \`.rpm\` 安装(\`sudo dnf install ./ReasonixStudio-linux-amd64.rpm\`)。x86_64 用文件名带 amd64 的包,aarch64(统信 UOS、麒麟等)用带 arm64 的包。\n\n" "$windows"
|
||||
printf "%s\n" "三个平台都能自更新:Linux 的 \`.deb\` 走自己的包(\`.rpm\` 的自更新尚未提供,请手动下载新包),Windows 运行下一个安装器,macOS 替换自身 bundle。"
|
||||
printf "**macOS** 已用 Developer ID 签名并公证,打开时不需要清除隔离属性。%s**Linux** 从 \`.deb\` 安装。\n\n" "$windows"
|
||||
printf '%s\n' '三个平台都能自更新:Linux 走自己的包,Windows 运行下一个安装器,macOS 替换自身 bundle。'
|
||||
} >> "$notes"
|
||||
assets=()
|
||||
for f in dist/*; do
|
||||
[ "$(basename "$f")" = latest.json ] || assets+=("$f")
|
||||
done
|
||||
if gh release view "$TAG" >/dev/null 2>&1; then
|
||||
gh release edit "$TAG" --notes-file "$notes"
|
||||
gh release upload "$TAG" "${assets[@]}" --clobber
|
||||
gh release upload "$TAG" dist/* --clobber
|
||||
else
|
||||
decided="$(git tag -l 'studio-v*' | bash scripts/studio-release-flags.sh "$TAG" "$CANDIDATE")"
|
||||
mapfile -t flags <<<"$decided"
|
||||
gh release create "$TAG" "${assets[@]}" \
|
||||
gh release create "$TAG" dist/* \
|
||||
--title "Reasonix Studio $VERSION" \
|
||||
--notes-file "$notes" \
|
||||
"${flags[@]}"
|
||||
--prerelease \
|
||||
--latest=false
|
||||
fi
|
||||
|
||||
- name: Mirror to R2 and update the Studio catalog
|
||||
@@ -1250,22 +1088,6 @@ jobs:
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# The notes object goes up before the catalog entry that names it, so
|
||||
# no client reads a pointer to an object that is not there yet. Notes
|
||||
# are optional: a failure here publishes the version without a pointer
|
||||
# rather than keeping the new version out of the catalog.
|
||||
notes_key="studio/notes/${VERSION#v}.md"
|
||||
notes_arg=()
|
||||
if aws s3 cp "$RUNNER_TEMP/version-notes.md" "s3://${R2_BUCKET}/${notes_key}" \
|
||||
--endpoint-url "$endpoint" \
|
||||
--content-type "text/markdown; charset=utf-8" \
|
||||
--cache-control "public, max-age=31536000, immutable" &&
|
||||
aws s3api head-object --bucket "$R2_BUCKET" --key "$notes_key" --endpoint-url "$endpoint" >/dev/null; then
|
||||
notes_arg=(20 "https://dl.reasonix.io/${notes_key}")
|
||||
else
|
||||
echo "::warning::release notes for ${VERSION} were not uploaded to the mirror; the catalog entry is published without notes"
|
||||
fi
|
||||
|
||||
# Studio's own catalog. Writing the ROOT versions.json here would hand
|
||||
# desktop users Studio releases, so the prefix is not optional.
|
||||
current=/tmp/studio-versions-current.json
|
||||
@@ -1277,7 +1099,7 @@ jobs:
|
||||
echo "no existing Studio catalog; starting a new one"
|
||||
fi
|
||||
bash scripts/update-versions-index.sh "$source" "$VERSION" "$TAG" studio \
|
||||
"$(date -u +%Y-%m-%dT%H:%M:%SZ)" "${notes_arg[@]}" > "$next"
|
||||
"$(date -u +%Y-%m-%dT%H:%M:%SZ)" > "$next"
|
||||
jq -e --arg v "$VERSION" '.versions | map(select(.version == $v)) | length == 1' "$next" >/dev/null
|
||||
aws s3 cp "$next" "s3://${R2_BUCKET}/studio/versions.json" \
|
||||
--endpoint-url "$endpoint" \
|
||||
@@ -1389,13 +1211,13 @@ jobs:
|
||||
|
||||
# The CLI's install channels, npm's `reasonix` and the Homebrew tap. The 1.x
|
||||
# line owns them until this one takes the CLI over; the repository variable
|
||||
# STUDIO_PUBLISHES_CLI=true hands them here. They run once cli-pointer has
|
||||
# created the CLI release, because the cask points at its archives. A candidate
|
||||
# goes to npm's `next` tag and leaves the tap alone: brew has no prerelease
|
||||
# channel, so a candidate cask would become the default install.
|
||||
# STUDIO_PUBLISHES_CLI=true hands them here. They run once the release
|
||||
# exists, because the cask points at its archives. A candidate goes to npm's
|
||||
# `next` tag and leaves the tap alone: brew has no prerelease channel, so a
|
||||
# candidate cask would become the default install.
|
||||
cli-channels:
|
||||
name: publish the CLI to npm and Homebrew
|
||||
needs: [resolve, publish, cli-gate, cli-tag, cli-pointer]
|
||||
needs: [resolve, publish, cli-gate, cli-tag]
|
||||
if: vars.STUDIO_PUBLISHES_CLI == 'true'
|
||||
runs-on: ubuntu-latest
|
||||
# id-token lets npm publish --provenance attach a Sigstore attestation; the
|
||||
@@ -1411,7 +1233,7 @@ jobs:
|
||||
with:
|
||||
go-version-file: go.mod
|
||||
cache: true
|
||||
- uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7
|
||||
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
|
||||
with:
|
||||
node-version: "24"
|
||||
registry-url: https://registry.npmjs.org
|
||||
@@ -1420,7 +1242,7 @@ jobs:
|
||||
NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
|
||||
VERSION: ${{ needs.resolve.outputs.version }}
|
||||
run: node npm/build.mjs "$VERSION" --publish
|
||||
- uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
|
||||
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
if: needs.resolve.outputs.candidate != 'true'
|
||||
with:
|
||||
name: cli-archives
|
||||
@@ -1434,18 +1256,6 @@ jobs:
|
||||
run: |
|
||||
set -euo pipefail
|
||||
git clone --depth 1 "https://x-access-token:${TAP_TOKEN}@github.com/esengine/homebrew-reasonix.git" tap
|
||||
decision="$(node scripts/brew-cask-guard.mjs tap/Casks/reasonix.rb "${VERSION#v}")"
|
||||
case "$decision" in
|
||||
write) ;;
|
||||
skip)
|
||||
echo "the tap's cask is newer than ${VERSION#v}; leaving it"
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
echo "::error::unexpected cask guard decision: '$decision'"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
bash scripts/render-brew-cask.sh "${VERSION#v}" "$TAG" cli-archives/SHA256SUMS > tap/Casks/reasonix.rb
|
||||
cd tap
|
||||
git add Casks/reasonix.rb
|
||||
@@ -1474,7 +1284,7 @@ jobs:
|
||||
with:
|
||||
ref: ${{ needs.resolve.outputs.sha }}
|
||||
persist-credentials: false
|
||||
- uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
|
||||
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
name: cli-archives
|
||||
path: cli-archives
|
||||
|
||||
@@ -43,7 +43,7 @@ jobs:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7
|
||||
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
|
||||
with:
|
||||
node-version: '22'
|
||||
- name: Test sweeper
|
||||
|
||||
@@ -46,7 +46,7 @@ jobs:
|
||||
version: 10
|
||||
run_install: false
|
||||
|
||||
- uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7
|
||||
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
|
||||
with:
|
||||
node-version: "24"
|
||||
cache: pnpm
|
||||
@@ -99,7 +99,7 @@ jobs:
|
||||
-c.extraMetadata.version="${{ steps.version.outputs.version }}"
|
||||
sha256sum ../../dist/ReasonixStudio-windows-amd64-installer.exe | tee -a "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
- uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
|
||||
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: studio-selftest-windows-${{ steps.version.outputs.version }}
|
||||
path: dist/ReasonixStudio-windows-amd64-installer.exe
|
||||
|
||||
+60
-320
@@ -33,7 +33,7 @@ jobs:
|
||||
version: 10
|
||||
run_install: false
|
||||
|
||||
- uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7
|
||||
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
|
||||
with:
|
||||
node-version: "24"
|
||||
cache: pnpm
|
||||
@@ -81,98 +81,9 @@ jobs:
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Real-Chromium guards that run without a kernel. Only the ones that passed
|
||||
# three consecutive local runs are wired; the rest stay manual (see
|
||||
# desktop/frontend-next/perf/README.md). Skipped unless the SPA changed.
|
||||
browser-guards:
|
||||
name: browser guards
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
defaults:
|
||||
run:
|
||||
working-directory: desktop/frontend-next
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: SPA changed?
|
||||
id: spa
|
||||
working-directory: .
|
||||
run: |
|
||||
if [ "${{ github.event_name }}" = "pull_request" ] &&
|
||||
git diff --quiet "origin/${{ github.base_ref }}...HEAD" -- desktop/frontend-next; then
|
||||
echo "run=false" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "run=true" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
- if: steps.spa.outputs.run == 'true'
|
||||
uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413 # v6.1.0
|
||||
with:
|
||||
version: 10
|
||||
run_install: false
|
||||
|
||||
- if: steps.spa.outputs.run == 'true'
|
||||
uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7
|
||||
with:
|
||||
node-version: "24"
|
||||
cache: pnpm
|
||||
cache-dependency-path: desktop/frontend-next/pnpm-lock.yaml
|
||||
|
||||
- if: steps.spa.outputs.run == 'true'
|
||||
name: Install
|
||||
run: pnpm install --frozen-lockfile
|
||||
|
||||
- if: steps.spa.outputs.run == 'true'
|
||||
name: Build the perf harness and install Chromium
|
||||
run: |
|
||||
pnpm exec vite build --config vite.perf.config.ts
|
||||
pnpm exec playwright install --with-deps chromium
|
||||
|
||||
- if: steps.spa.outputs.run == 'true'
|
||||
name: Guards
|
||||
run: |
|
||||
python3 -m http.server 4399 --directory dist-perf >/dev/null 2>&1 &
|
||||
for _ in $(seq 30); do curl -sf http://localhost:4399/perf.html >/dev/null && break; sleep 1; done
|
||||
node perf/intrinsic.mjs
|
||||
node perf/firstscreen-fonts.mjs
|
||||
node perf/contrast.mjs
|
||||
node perf/stickyfold.mjs
|
||||
node perf/crumb.mjs
|
||||
node perf/hoverradius.mjs
|
||||
node perf/modelrows.mjs
|
||||
node perf/explorersize.mjs
|
||||
node perf/rowhit.mjs
|
||||
node perf/sessquick.mjs
|
||||
node perf/sessionrows.mjs
|
||||
node perf/wheel.mjs
|
||||
node perf/branch-tooltip.mjs
|
||||
node perf/branch-menu.mjs
|
||||
node perf/image-preview.mjs
|
||||
node perf/toast-anchor.mjs
|
||||
node perf/addfolder-rest.mjs
|
||||
node perf/scroll-jump.mjs
|
||||
node perf/add-project-selects.mjs
|
||||
node perf/setup-defer.mjs
|
||||
node perf/search-toggle.mjs
|
||||
node perf/narrow.mjs
|
||||
node perf/settings-rows.mjs
|
||||
node perf/narrow-steps.mjs
|
||||
node perf/narrow-rail.mjs
|
||||
node perf/selfcheck.mjs branch-tooltip
|
||||
|
||||
# A pull request with no platform-relevant change (scripts/ci-platform-scope.sh)
|
||||
# moves the Windows and macOS legs onto an ubuntu runner with every step
|
||||
# skipped, so the check names still report without a scarce runner. A push to
|
||||
# studio, or a failed `scope`, runs all three.
|
||||
desktop:
|
||||
name: desktop (${{ matrix.os }})
|
||||
needs: scope
|
||||
if: always()
|
||||
runs-on: ${{ github.event_name == 'pull_request' && needs.scope.outputs.platform == 'false' && matrix.os != 'ubuntu-22.04' && 'ubuntu-latest' || matrix.os }}
|
||||
env:
|
||||
RUN_STEPS: ${{ github.event_name != 'pull_request' || needs.scope.outputs.platform != 'false' || matrix.os == 'ubuntu-22.04' }}
|
||||
runs-on: ${{ matrix.os }}
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
@@ -181,11 +92,9 @@ jobs:
|
||||
run:
|
||||
working-directory: desktop
|
||||
steps:
|
||||
- if: env.RUN_STEPS == 'true'
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
|
||||
- if: env.RUN_STEPS == 'true'
|
||||
uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7
|
||||
- uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7
|
||||
with:
|
||||
go-version-file: desktop/go.mod
|
||||
cache: true
|
||||
@@ -198,97 +107,20 @@ jobs:
|
||||
# carries build dependencies of its own.
|
||||
# Same command as `make studio-test`; this job is already in desktop/.
|
||||
- name: go test
|
||||
if: env.RUN_STEPS == 'true'
|
||||
run: go test ./...
|
||||
|
||||
- if: env.RUN_STEPS == 'true'
|
||||
uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7
|
||||
with:
|
||||
node-version: "24"
|
||||
|
||||
- name: CUA process and packaging contracts
|
||||
if: env.RUN_STEPS == 'true'
|
||||
timeout-minutes: 2
|
||||
run: node --test --test-name-pattern=CUA electron/test/unit.mjs
|
||||
|
||||
- name: Install Linux native CUA fixture dependencies
|
||||
if: env.RUN_STEPS == 'true' && runner.os == 'Linux'
|
||||
run: sudo apt-get update && sudo apt-get install -y xvfb openbox dbus-x11 libgtk-3-dev at-spi2-core
|
||||
|
||||
- name: Native CUA application acceptance
|
||||
if: env.RUN_STEPS == 'true'
|
||||
timeout-minutes: 6
|
||||
shell: bash
|
||||
run: |
|
||||
if [ "$RUNNER_OS" = Linux ]; then
|
||||
dbus-run-session -- xvfb-run -a sh -c 'openbox >/tmp/reasonix-openbox.log 2>&1 & export NO_AT_BRIDGE=0 GTK_MODULES=gail:atk-bridge; node electron/test/cua-native.mjs'
|
||||
else
|
||||
node electron/test/cua-native.mjs
|
||||
fi
|
||||
|
||||
# The host sits in the main module and nothing else builds it, so its
|
||||
# dependency closure is checked here: it must stay CGO-free and
|
||||
# platform-neutral or the Electron bundle cannot ship it. Cross-compiling
|
||||
# is host-independent, so once is enough.
|
||||
- name: host stays pure Go and platform-neutral
|
||||
if: env.RUN_STEPS == 'true' && runner.os == 'Linux'
|
||||
if: runner.os == 'Linux'
|
||||
working-directory: .
|
||||
run: |
|
||||
for os in windows linux darwin; do
|
||||
for arch in amd64 arm64; do
|
||||
CGO_ENABLED=0 GOOS=$os GOARCH=$arch go build -o /dev/null ./cmd/reasonix-studio-host
|
||||
done
|
||||
CGO_ENABLED=0 GOOS=$os GOARCH=amd64 go build -o /dev/null ./cmd/reasonix-studio-host
|
||||
done
|
||||
|
||||
# The ARM64 leg queues on a scarcer runner pool, so a pull request runs it
|
||||
# only when it touches what the leg verifies; a push to studio always does.
|
||||
# The macOS and Windows package legs follow `platform`, the same rule as the
|
||||
# desktop matrix above.
|
||||
scope:
|
||||
name: package matrix
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
matrix: ${{ steps.pick.outputs.matrix }}
|
||||
arm64: ${{ steps.pick.outputs.arm64 }}
|
||||
platform: ${{ steps.pick.outputs.platform }}
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- id: pick
|
||||
env:
|
||||
EVENT: ${{ github.event_name }}
|
||||
BASE_SHA: ${{ github.event.pull_request.base.sha }}
|
||||
run: |
|
||||
arm64=true
|
||||
platform=true
|
||||
if [ "$EVENT" = pull_request ]; then
|
||||
arm64=false
|
||||
platform=$(git diff -z --name-only --no-renames "$BASE_SHA" HEAD | bash scripts/ci-platform-scope.sh)
|
||||
if git diff --name-only --no-renames "$BASE_SHA" HEAD |
|
||||
grep -qE '^(desktop/electron/|desktop/packaging/|desktop/cmd/update-helper/|desktop/cmd/studio-manifest/|desktop/go\.(mod|sum)$|cmd/reasonix-studio-host/|internal/platform/(update|appupdate)/|go\.(mod|sum)$|scripts/(linux-install-smoke|verify-linux-package|verify-studio-artifacts)\.sh$|scripts/(windows-signing-lib|verify-windows-package|verify-windows-authenticode|install-smoke)\.ps1$|\.github/workflows/(studio|release-studio)\.yml$)'; then
|
||||
arm64=true
|
||||
fi
|
||||
fi
|
||||
legs='[
|
||||
{"os":"macos-latest","arch":"arm64","platform":"darwin"},
|
||||
{"os":"ubuntu-22.04","arch":"amd64","platform":"linux"},
|
||||
{"os":"windows-latest","arch":"amd64","platform":"windows","ebarch":"x64"}
|
||||
]'
|
||||
if [ "$platform" = false ]; then
|
||||
legs="$(jq -c 'map(select(.platform == "linux"))' <<<"$legs")"
|
||||
elif [ "$arm64" = true ]; then
|
||||
legs="$(jq -c '. + [{"os":"windows-11-arm","arch":"arm64","platform":"windows","ebarch":"arm64"},{"os":"ubuntu-22.04-arm","arch":"arm64","platform":"linux"}]' <<<"$legs")"
|
||||
else
|
||||
legs="$(jq -c . <<<"$legs")"
|
||||
fi
|
||||
{
|
||||
echo "arm64=$arm64"
|
||||
echo "platform=$platform"
|
||||
echo "matrix={\"include\":$legs}"
|
||||
} >> "$GITHUB_OUTPUT"
|
||||
|
||||
# The packaging contract, on real artifacts rather than on the config that
|
||||
# describes them. Unsigned on purpose: signing and notarization are a
|
||||
# distribution concern and belong to the release workflow, and treating their
|
||||
@@ -296,12 +128,20 @@ jobs:
|
||||
# cross-compile while only Windows had ever produced a package.
|
||||
package:
|
||||
name: package (${{ matrix.os }})
|
||||
needs: scope
|
||||
runs-on: ${{ matrix.os }}
|
||||
timeout-minutes: 45
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix: ${{ fromJSON(needs.scope.outputs.matrix) }}
|
||||
matrix:
|
||||
include:
|
||||
- os: macos-latest
|
||||
arch: arm64
|
||||
platform: darwin
|
||||
- os: ubuntu-22.04
|
||||
arch: amd64
|
||||
platform: linux
|
||||
- os: windows-latest
|
||||
arch: amd64
|
||||
platform: windows
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
@@ -319,40 +159,24 @@ jobs:
|
||||
version: 10
|
||||
run_install: false
|
||||
|
||||
- uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7
|
||||
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
|
||||
with:
|
||||
node-version: "24"
|
||||
cache: pnpm
|
||||
cache-dependency-path: desktop/electron/pnpm-lock.yaml
|
||||
|
||||
# electron-builder shells out to dpkg-deb through fakeroot for the .deb and
|
||||
# to rpmbuild for the .rpm.
|
||||
# electron-builder shells out to dpkg-deb through fakeroot for the .deb.
|
||||
- name: Install Linux packaging deps
|
||||
if: runner.os == 'Linux'
|
||||
run: sudo apt-get update && sudo apt-get install -y fakeroot rpm cpio xvfb xauth policykit-1
|
||||
|
||||
# The fpm electron-builder carries is an x86_64 program, so on an ARM64
|
||||
# runner the .deb and .rpm are built with one installed from RubyGems.
|
||||
- name: Use a native fpm on ARM64 Linux
|
||||
if: runner.os == 'Linux' && runner.arch == 'ARM64'
|
||||
run: |
|
||||
sudo apt-get install -y ruby ruby-dev build-essential
|
||||
sudo gem install --no-document fpm -v 1.16.0
|
||||
echo "USE_SYSTEM_FPM=true" >> "$GITHUB_ENV"
|
||||
run: sudo apt-get update && sudo apt-get install -y fakeroot
|
||||
|
||||
- name: Build the SPA
|
||||
working-directory: desktop/frontend-next
|
||||
run: pnpm install --frozen-lockfile && pnpm build
|
||||
|
||||
# The Windows legs name their architecture instead of taking the
|
||||
# toolchain's, so a runner whose Go or Node is the other architecture
|
||||
# fails here rather than shipping a mislabelled package.
|
||||
- name: Build the kernel the shell speaks to
|
||||
working-directory: desktop/electron
|
||||
env:
|
||||
REASONIX_GOARCH: ${{ matrix.platform == 'windows' && matrix.arch || '' }}
|
||||
REASONIX_VERSION: v0.0.0-ci
|
||||
run: pnpm install --frozen-lockfile && pnpm test && pnpm run build:host
|
||||
run: pnpm install --frozen-lockfile && pnpm run build:host
|
||||
|
||||
# The .deb's absolute-path payload. A no-op off Linux, so it is not
|
||||
# guarded here: the one place it must not be forgotten is the one place
|
||||
@@ -367,8 +191,7 @@ jobs:
|
||||
working-directory: desktop/electron
|
||||
env:
|
||||
CSC_IDENTITY_AUTO_DISCOVERY: "false"
|
||||
ELECTRON_BUILDER_7Z_FILTER: ${{ matrix.platform == 'windows' && matrix.arch == 'arm64' && 'BCJ' || '' }}
|
||||
run: pnpm run build:dist ${{ matrix.ebarch && format('--{0}', matrix.ebarch) || '' }} -c.extraMetadata.version=0.0.0-ci
|
||||
run: pnpm run build:dist
|
||||
|
||||
# The release vocabulary is GOARCH. electron-builder's own is not, and the
|
||||
# rename that reconciles them runs here rather than in the manifest.
|
||||
@@ -384,99 +207,6 @@ jobs:
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# An emulated x64 program runs on ARM64 Windows, so a package that merely
|
||||
# starts proves nothing about its architecture: every PE image in the
|
||||
# bundle has to be built for the one the package is named for.
|
||||
- name: Every PE image is built for the package's architecture
|
||||
if: matrix.platform == 'windows'
|
||||
run: |
|
||||
dir=$(find dist -maxdepth 1 -type d -name 'win*-unpacked' | head -1)
|
||||
test -n "$dir"
|
||||
node desktop/electron/packaging/pe.js "$dir" "${{ matrix.arch }}"
|
||||
|
||||
# An update starts the installer with --updated, and that is what keeps the
|
||||
# shortcuts (and the taskbar pin hanging off them) from being deleted and
|
||||
# recreated. Installed twice over itself the way an update does it, the
|
||||
# desktop shortcut has to still be the same file, pointing at a path with
|
||||
# no version in it, or every update costs the user their pin.
|
||||
- name: An update keeps the shortcuts and they name no version
|
||||
if: matrix.platform == 'windows'
|
||||
shell: pwsh
|
||||
run: |
|
||||
$installer = (Get-ChildItem dist -Filter 'ReasonixStudio-windows-*-installer.exe' | Select-Object -First 1).FullName
|
||||
$dir = Join-Path $env:RUNNER_TEMP 'rx-update-guard'
|
||||
Start-Process $installer -ArgumentList '/S', "/D=$dir" -Wait
|
||||
$lnk = Join-Path ([Environment]::GetFolderPath('Desktop')) 'Reasonix Studio.lnk'
|
||||
$target = (New-Object -ComObject WScript.Shell).CreateShortcut($lnk).TargetPath
|
||||
if ($target -ne (Join-Path $dir 'Reasonix Studio.exe')) { throw "the desktop shortcut targets $target, not the stable executable" }
|
||||
$mark = [datetime]'2020-01-01'
|
||||
(Get-Item $lnk).LastWriteTime = $mark
|
||||
Start-Process $installer -ArgumentList '/S', '--updated', "/D=$dir" -Wait
|
||||
if ((Get-Item $lnk).LastWriteTime -ne $mark) { throw 'an update rewrote the desktop shortcut' }
|
||||
Start-Process (Join-Path $dir 'Uninstall Reasonix Studio.exe') -ArgumentList '/S', '/currentuser' -Wait
|
||||
|
||||
# The release packages a signed copy of the bundle and then reads both
|
||||
# packages back against it. The same recipe on an unsigned copy, so the
|
||||
# packaging contract (the installer's one application archive, the pinned
|
||||
# elevate.exe, every PE image's architecture) is exercised on pull
|
||||
# requests and not first by a release.
|
||||
- name: The packages read back against the bundle they were made from
|
||||
if: matrix.platform == 'windows'
|
||||
shell: bash
|
||||
env:
|
||||
ELECTRON_BUILDER_7Z_FILTER: ${{ matrix.arch == 'arm64' && 'BCJ' || '' }}
|
||||
run: |
|
||||
rm -rf payload nsis-src packed zip-src
|
||||
cp -r "$(find dist -maxdepth 1 -type d -name 'win*-unpacked' | head -1)" payload
|
||||
rm -f payload/resources/elevate.exe
|
||||
# The NSIS run writes elevate.exe into the tree it packs, so each
|
||||
# packager gets a copy and payload stays what was signed.
|
||||
cp -r payload zip-src
|
||||
cp -r payload nsis-src
|
||||
cd desktop/electron
|
||||
pnpm exec electron-builder --prepackaged ../../zip-src --win zip --${{ matrix.ebarch }} \
|
||||
-c.extraMetadata.version=0.0.0-ci -c.directories.output=../../packed-zip
|
||||
pnpm exec electron-builder --prepackaged ../../nsis-src --win nsis --${{ matrix.ebarch }} \
|
||||
-c.extraMetadata.version=0.0.0-ci -c.directories.output=../../packed
|
||||
mv ../../packed-zip/ReasonixStudio-windows-${{ matrix.arch }}.zip ../../packed/
|
||||
ls -la ../../packed
|
||||
|
||||
- name: verify-windows-package.ps1 accepts them
|
||||
if: matrix.platform == 'windows'
|
||||
shell: pwsh
|
||||
run: |
|
||||
$ErrorActionPreference = 'Stop'
|
||||
. ./scripts/windows-signing-lib.ps1
|
||||
$digest = Get-ManifestDigest -Manifest @(Get-TreeManifest -Root payload)
|
||||
./scripts/verify-windows-package.ps1 `
|
||||
-PayloadDirectory payload `
|
||||
-ExpectedPayloadDigest $digest `
|
||||
-InstallerPath "packed/ReasonixStudio-windows-${{ matrix.arch }}-installer.exe" `
|
||||
-PortableArchivePath "packed/ReasonixStudio-windows-${{ matrix.arch }}.zip" `
|
||||
-OutputPath "$env:RUNNER_TEMP/verify-output.txt" `
|
||||
-Architecture ${{ matrix.arch }}
|
||||
Get-Content "$env:RUNNER_TEMP/verify-output.txt"
|
||||
|
||||
# The release's own install smoke, against the unsigned installer: the
|
||||
# package is installed where it is meant to run, its tree checked, and
|
||||
# removed.
|
||||
- name: The package installs, runs and uninstalls
|
||||
if: matrix.platform == 'windows'
|
||||
shell: pwsh
|
||||
run: |
|
||||
./scripts/install-smoke.ps1 `
|
||||
-Installer "dist/ReasonixStudio-windows-${{ matrix.arch }}-installer.exe" `
|
||||
-Architecture ${{ matrix.arch }}
|
||||
|
||||
- name: Keep the arm64 installer for the x64 refusal check
|
||||
if: matrix.platform == 'windows' && matrix.arch == 'arm64'
|
||||
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
|
||||
with:
|
||||
name: studio-arm64-installer-ci
|
||||
path: dist/ReasonixStudio-windows-arm64-installer.exe
|
||||
if-no-files-found: error
|
||||
retention-days: 1
|
||||
|
||||
# The update channel is the signed manifest. A second answer to which
|
||||
# version is current is not a convenience. The .blockmap is the same
|
||||
# channel's differential index and carries the release prefix, so it also
|
||||
@@ -503,13 +233,47 @@ jobs:
|
||||
echo "page: ${page:-<missing>}"
|
||||
test -n "$host" -a -n "$page"
|
||||
|
||||
- name: Linux packages carry their declared architecture and update payload
|
||||
# Read off the package rather than off the config that describes it. Both
|
||||
# of these went wrong the same way once: electron-builder names a package
|
||||
# after package.json unless told, and fpm records the owner it finds on
|
||||
# disk. Either one leaves a Studio that cannot update itself on Linux and
|
||||
# says nothing about it until a user asks.
|
||||
- name: The .deb is Studio's own package and carries what updates it
|
||||
if: runner.os == 'Linux'
|
||||
run: bash scripts/verify-linux-package.sh "dist/ReasonixStudio-linux-${{ matrix.arch }}.deb" "dist/ReasonixStudio-linux-${{ matrix.arch }}.rpm" "${{ matrix.arch }}"
|
||||
run: |
|
||||
deb=$(ls dist/ReasonixStudio-linux-*.deb)
|
||||
name=$(dpkg-deb -f "$deb" Package)
|
||||
if [ "$name" != "reasonix-studio" ]; then
|
||||
echo "::error::the package is $name; dpkg holds that beside the installed reasonix-studio instead of upgrading it, and update.StudioLine asks for it by name"
|
||||
exit 1
|
||||
fi
|
||||
dpkg-deb -c "$deb" > contents.txt
|
||||
# The path is the polkit action's exec.path annotation, so it is the
|
||||
# only one authorization is granted for; root because polkit will not
|
||||
# run a helper the invoking user could have written. Either spelling
|
||||
# of root: dpkg-deb prints the numeric ids it cannot map, and which
|
||||
# one it prints is a property of the machine reading the package
|
||||
# rather than of the package.
|
||||
if ! grep -qE '^-rwxr-xr-x (root/root|0/0) +[0-9]+ .* \./usr/lib/reasonix-studio/reasonix-studio-update-helper$' contents.txt; then
|
||||
echo "::error::no root-owned executable update helper at the path the polkit action names"
|
||||
grep reasonix-studio-update-helper contents.txt || echo "(not in the package at all)"
|
||||
exit 1
|
||||
fi
|
||||
if ! grep -q './usr/share/polkit-1/actions/io.reasonix.studio.update.policy$' contents.txt; then
|
||||
echo "::error::the polkit action is not in the package, so pkexec has nothing to authorize against"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: The Linux package installs, runs, reinstalls and uninstalls
|
||||
# The Electron shell sets the package's glibc floor; a Go binary linked
|
||||
# with cgo would raise it to the runner's own.
|
||||
- name: The .deb's Go binaries are statically linked
|
||||
if: runner.os == 'Linux'
|
||||
run: bash scripts/linux-install-smoke.sh "dist/ReasonixStudio-linux-${{ matrix.arch }}.deb" "${{ matrix.arch }}" 0.0.0-ci
|
||||
run: |
|
||||
deb=$(ls dist/ReasonixStudio-linux-*.deb)
|
||||
dpkg-deb -x "$deb" "$RUNNER_TEMP/deb"
|
||||
host=$(find "$RUNNER_TEMP/deb" -type f -path '*/resources/bin/reasonix-studio-host')
|
||||
test -n "$host"
|
||||
node desktop/electron/packaging/elf.js "$host" "$RUNNER_TEMP/deb/usr/lib/reasonix-studio/reasonix-studio-update-helper"
|
||||
|
||||
# Recorded, not asserted: the real bundle layout, so later work argues
|
||||
# from what electron-builder produced rather than from its documentation.
|
||||
@@ -537,27 +301,3 @@ jobs:
|
||||
echo "::error::the manifest has no entry for ${{ matrix.platform }}-${{ matrix.arch }}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# The arm64 installer carries one application and no other. Run anywhere but
|
||||
# ARM64 Windows it has to say so and stop, instead of exiting 0 with nothing
|
||||
# installed.
|
||||
arm64-installer-refuses-x64:
|
||||
name: arm64 installer refuses an x64 machine
|
||||
needs: [scope, package]
|
||||
if: needs.scope.outputs.arm64 == 'true'
|
||||
runs-on: windows-latest
|
||||
steps:
|
||||
- uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
|
||||
with:
|
||||
name: studio-arm64-installer-ci
|
||||
path: installer
|
||||
|
||||
- name: Run it silently
|
||||
shell: pwsh
|
||||
run: |
|
||||
$ErrorActionPreference = 'Stop'
|
||||
$target = Join-Path $env:RUNNER_TEMP 'rx-refused'
|
||||
$run = Start-Process -FilePath installer/ReasonixStudio-windows-arm64-installer.exe -ArgumentList '/S', "/D=$target" -Wait -PassThru
|
||||
Write-Host "exit code $($run.ExitCode)"
|
||||
if ($run.ExitCode -eq 0) { throw 'the arm64 installer exited 0 on an x64 machine' }
|
||||
if (@(Get-ChildItem -Recurse -Force $target -ErrorAction SilentlyContinue).Count -ne 0) { throw 'the arm64 installer wrote into the target on an x64 machine' }
|
||||
@@ -14,7 +14,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
- uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7
|
||||
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
|
||||
with:
|
||||
node-version: '22'
|
||||
- name: Update README contributor tables
|
||||
@@ -33,6 +33,6 @@ jobs:
|
||||
body: |
|
||||
Refreshes the README acknowledgments tables from the GitHub contributors API.
|
||||
|
||||
- Source: `repos/esengine/DeepSeek-Reasonix/contributors?per_page=50&anon=1`
|
||||
- The repository owner, declared aliases, tool identities, bots and entries without a GitHub login are left out (`scripts/credit-identities.json`).
|
||||
- Source: `repos/esengine/DeepSeek-Reasonix/contributors?per_page=20&anon=1`
|
||||
- Anonymous contributors are shown without email addresses.
|
||||
labels: documentation
|
||||
@@ -27,7 +27,7 @@ jobs:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7
|
||||
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
|
||||
with:
|
||||
node-version: '22'
|
||||
- name: Test chart generator
|
||||
|
||||
@@ -1,84 +0,0 @@
|
||||
name: Withdraw a Studio version
|
||||
|
||||
# Pulls a published Studio release out of the update catalog (or puts it back),
|
||||
# so installs stop being offered it without waiting for a new release. Only the
|
||||
# catalog changes: the release's files stay where they are, and a withdrawn
|
||||
# release can be restored with the same run.
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
version:
|
||||
description: "Version to withdraw or restore, for example 2.36.0"
|
||||
required: true
|
||||
type: string
|
||||
reason:
|
||||
description: "Why it is withdrawn (shown to users on that version); ignored when restoring"
|
||||
required: false
|
||||
type: string
|
||||
restore:
|
||||
description: "Put a withdrawn version back instead of withdrawing it"
|
||||
required: false
|
||||
type: boolean
|
||||
default: false
|
||||
|
||||
concurrency:
|
||||
group: studio-catalog
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
withdraw:
|
||||
name: ${{ inputs.restore && 'restore' || 'withdraw' }} ${{ inputs.version }}
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
environment: studio-release
|
||||
permissions:
|
||||
contents: write
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
with:
|
||||
ref: studio
|
||||
persist-credentials: false
|
||||
|
||||
- name: Update the Studio catalog
|
||||
env:
|
||||
VERSION: ${{ inputs.version }}
|
||||
REASON: ${{ inputs.reason }}
|
||||
RESTORE: ${{ inputs.restore }}
|
||||
R2_BUCKET: ${{ secrets.R2_BUCKET }}
|
||||
HAS_R2: ${{ secrets.R2_ACCESS_KEY_ID != '' && secrets.R2_SECRET_ACCESS_KEY != '' && secrets.R2_ACCOUNT_ID != '' && secrets.R2_BUCKET != '' }}
|
||||
run: |
|
||||
if [ "$HAS_R2" != "true" ]; then
|
||||
echo "::error::R2 credentials are not configured, so the catalog cannot be changed"
|
||||
exit 1
|
||||
fi
|
||||
if ! [[ "$VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z.]+)?$ ]]; then
|
||||
echo "::error::version must look like 2.36.0"
|
||||
exit 1
|
||||
fi
|
||||
aws configure set aws_access_key_id "${{ secrets.R2_ACCESS_KEY_ID }}"
|
||||
aws configure set aws_secret_access_key "${{ secrets.R2_SECRET_ACCESS_KEY }}"
|
||||
aws configure set default.region auto
|
||||
endpoint="https://${{ secrets.R2_ACCOUNT_ID }}.r2.cloudflarestorage.com"
|
||||
at="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||||
current=/tmp/studio-versions-current.json
|
||||
next=/tmp/studio-versions-next.json
|
||||
aws s3 cp "s3://${R2_BUCKET}/studio/versions.json" "$current" --endpoint-url "$endpoint"
|
||||
if [ "$RESTORE" = "true" ]; then
|
||||
bash scripts/withdraw-version.sh "$current" --restore "$VERSION" "$at" > "$next"
|
||||
jq -e --arg v "$VERSION" '.versions | map(select((.version | ltrimstr("v")) == $v)) | length == 1' "$next" >/dev/null
|
||||
else
|
||||
bash scripts/withdraw-version.sh "$current" "$VERSION" "$REASON" "$at" > "$next"
|
||||
jq -e --arg v "$VERSION" '(.versions | map(select((.version | ltrimstr("v")) == $v)) | length == 0) and (.withdrawn | map(select((.version | ltrimstr("v")) == $v)) | length == 1)' "$next" >/dev/null
|
||||
fi
|
||||
aws s3 cp "$next" "s3://${R2_BUCKET}/studio/versions.json" \
|
||||
--endpoint-url "$endpoint" \
|
||||
--content-type "application/json; charset=utf-8" \
|
||||
--cache-control "public, max-age=300, stale-if-error=86400"
|
||||
echo "Studio catalog updated for $VERSION (restore=$RESTORE)"
|
||||
|
||||
- name: Keep the GitHub release off the latest slot
|
||||
if: ${{ !inputs.restore }}
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
VERSION: ${{ inputs.version }}
|
||||
run: gh release edit "studio-v${VERSION}" --prerelease --latest=false --repo "$GITHUB_REPOSITORY"
|
||||
@@ -15,7 +15,6 @@
|
||||
/runtime-resume
|
||||
/extension-protocol-gen
|
||||
/reasonix-plugin-example
|
||||
/mcp-line-counter-kit
|
||||
/reasonix-studio-host
|
||||
/signpath-contract
|
||||
/studio-delta
|
||||
|
||||
@@ -11,7 +11,6 @@ fingerprint_files:
|
||||
- .github/workflows/release-studio.yml
|
||||
- .github/workflows/studio-certum-signing-smoke.yml
|
||||
- .github/actions/setup-certum/action.yml
|
||||
- scripts/install-smoke.ps1
|
||||
- scripts/sign-certum.ps1
|
||||
- scripts/verify-windows-authenticode.ps1
|
||||
- scripts/verify-windows-package.ps1
|
||||
|
||||
@@ -181,35 +181,6 @@ porting to `studio`; do not open the same PR against both.
|
||||
5. Ensure `gofmt -l .` shows no changes
|
||||
6. Submit a pull request to that same branch
|
||||
|
||||
## Pull request policy
|
||||
|
||||
**Related changes.** If you have several related changes of one pattern, a
|
||||
tracking issue plus one pull request is easiest to review.
|
||||
|
||||
**Issue first for features.** A change that adds behaviour rather than fixing a
|
||||
defect should link an issue the maintainers have agreed to before the work
|
||||
starts. A fix for a reproducible defect may open directly, with the failing test
|
||||
in the PR. Documentation-only and test-only PRs are reviewed at lower priority.
|
||||
|
||||
**Review tiers.**
|
||||
|
||||
- Changes under the `sensitive:` paths declared in `REASONIX.md` get a full
|
||||
review plus a security review, regardless of author.
|
||||
- Every PR carries the four sections of the PR template: Cause, Blast radius,
|
||||
Neighbouring behaviours tested, Why this layer.
|
||||
- The new test must fail without the change.
|
||||
- A UI change (`desktop/frontend-next/src` or the website pages) fills the
|
||||
template's UI section: screenshots before and after, light and dark, wide
|
||||
and phone width.
|
||||
- The same section carries the line "Entries moved, hidden or removed".
|
||||
- Hiding or removing a visible control needs a linked issue and a release-notes
|
||||
line naming it, and updates `docs/STUDIO_PARITY.md` for a 1.x feature.
|
||||
- Review holds these; no check reads them.
|
||||
|
||||
**Transparency.** The PR template has an optional checkbox for disclosing AI
|
||||
assistance. It is for transparency only and does not change how a change is
|
||||
reviewed.
|
||||
|
||||
## Reporting issues
|
||||
|
||||
Open an issue on GitHub with:
|
||||
|
||||
@@ -18,8 +18,6 @@
|
||||
<a href="https://esengine.github.io/DeepSeek-Reasonix/">Website</a>
|
||||
·
|
||||
<strong><a href="https://discord.gg/XF78rEME2D">Discord</a></strong>
|
||||
·
|
||||
<strong><a href="https://qm.qq.com/q/i59b0z2R8s">QQ</a></strong>
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
@@ -33,7 +31,6 @@
|
||||
<a href="https://github.com/esengine/DeepSeek-Reasonix/graphs/contributors"><img src="https://img.shields.io/github/contributors/esengine/DeepSeek-Reasonix.svg?style=flat-square&color=bc8cff&labelColor=161b22&logo=github&logoColor=white" alt="contributors"/></a>
|
||||
<a href="https://github.com/esengine/DeepSeek-Reasonix/discussions"><img src="https://img.shields.io/github/discussions/esengine/DeepSeek-Reasonix.svg?style=flat-square&color=58a6ff&labelColor=161b22&logo=github&logoColor=white" alt="Discussions"/></a>
|
||||
<a href="https://discord.gg/XF78rEME2D"><img src="https://img.shields.io/badge/discord-join-5865F2.svg?style=flat-square&labelColor=161b22&logo=discord&logoColor=white" alt="Discord"/></a>
|
||||
<a href="https://qm.qq.com/q/i59b0z2R8s"><img src="https://img.shields.io/badge/QQ%20group-1093562660-12B7F5.svg?style=flat-square&labelColor=161b22&logo=tencentqq&logoColor=white" alt="QQ group 1093562660"/></a>
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
@@ -43,63 +40,23 @@
|
||||
|
||||
<br/>
|
||||
|
||||
<p align="center"><strong>Open source · MIT · runs on your computer, model of your choice</strong></p>
|
||||
<h3 align="center">An open-source coding agent for your terminal, desktop, browser and editor.</h3>
|
||||
<p align="center">It reads your project, edits files and runs commands and tests. Have it ask before each step, or work on its own within limits you set. Files its edit tools changed can be rewound. New here? Start with the Studio desktop app.</p>
|
||||
<p align="center"><strong>Open source · MIT · a single Go binary</strong></p>
|
||||
<h3 align="center">A coding agent you can leave running.</h3>
|
||||
<p align="center">One local engine, four ways in — terminal, desktop app, browser, or your editor over ACP. Plan mode, permissions, a workspace sandbox and per-turn checkpoints keep a long autonomous run something you can still read and undo.</p>
|
||||
<p align="center">Maintained by <strong>Huahui Yu</strong>.</p>
|
||||
|
||||
> [!IMPORTANT]
|
||||
> **Community · 加入社区** — bilingual Discord for setup help (`#help` / `#求助`), workflow showcases, and feature ideas. → **<https://discord.gg/XF78rEME2D>**
|
||||
>
|
||||
> QQ group **DeepSeek-Reasonix官方群** (`1093562660`) → **<https://qm.qq.com/q/i59b0z2R8s>** ([QR code](./docs/assets/qq-group.svg))
|
||||
|
||||
> Douyin: **做游戏的小鱼** (Douyin ID `22703872788`) · [QR code](./docs/assets/douyin.png)
|
||||
|
||||
<br/>
|
||||
|
||||
## ❤️Sponsor
|
||||
|
||||
<table><tr><td width="180" align="center" valign="middle"><a href="https://www.rightapi.ai/register?aff=e083bf73" target="_blank" rel="sponsored noopener noreferrer"><img src="./.github/sponsor/rightcode.png" alt="Right Code" width="64"></a></td><td valign="middle">Thanks to Right Code for sponsoring this project! Right Code reliably provides routing services for models such as Claude Code, Codex, and Gemini, with both pay-as-you-go and monthly subscription billing options available. Invoices are available upon top-up, and enterprise and team users can receive dedicated one-on-one support. Register via <a href="https://www.rightapi.ai/register?aff=e083bf73" target="_blank" rel="sponsored noopener noreferrer">this link</a>.</td></tr><tr><td width="180" align="center" valign="middle"><a href="https://www.packyapi.ai/register?aff=rIbQ" target="_blank" rel="sponsored noopener noreferrer"><picture><source media="(prefers-color-scheme: dark)" srcset="./.github/sponsor/packycode-dark.svg"><img src="./.github/sponsor/packycode-light.svg" alt="PackyCode" width="80"></picture></a></td><td valign="middle">Thanks to PackyCode for sponsoring this project! Access leading AI models through PackyCode with one API endpoint and one API key, with automatic failover and dedicated high-speed routes for Codex and Claude Code. Register via <a href="https://www.packyapi.ai/register?aff=rIbQ" target="_blank" rel="sponsored noopener noreferrer">this link</a>.</td></tr></table>
|
||||
|
||||
## What it is
|
||||
|
||||
Reasonix is an open-source AI coding assistant. The program runs on your computer; the AI model is a service you configure, which receives what it reads. You describe a task in plain language; it reads code, edits files and runs commands and tests inside the project folder you choose, and shows you each step.
|
||||
|
||||
You pick the model: a DeepSeek preset is built in, any OpenAI-compatible service is one config entry, and you pay the model provider directly.
|
||||
|
||||
## What you get
|
||||
|
||||
- **Read, edit, run.** It reads code, edits files and runs tests and commands, all inside your project folder.
|
||||
- **Permission levels.** Have it ask every time, or let everyday actions run on their own; your forbid rules always hold. To see the approach first, use plan mode: it starts work only once you approve.
|
||||
- **Rewind.** A file is saved before it changes, so you can rewind by conversation turn without touching your git history. Only files its edit tools changed are covered, not what a shell command did.
|
||||
|
||||
## 30-second start
|
||||
|
||||
Download, add a model key, open a project. New here? Start with the Studio desktop app.
|
||||
|
||||
1. Download and install the desktop app, Reasonix Studio, from the [releases page](https://github.com/esengine/DeepSeek-Reasonix/releases?q=studio-v&expanded=true).
|
||||
2. Open it and enter your model provider and key when asked.
|
||||
3. Open your project folder and write what you want, for example "add retry on failure to the http client".
|
||||
4. At the level you chose, it stops and asks before anything that needs your say-so. Allow it to continue, or refuse.
|
||||
|
||||
Prefer a terminal? Download the `reasonix` command-line tool, run `reasonix setup` to pick a model, then run `reasonix`. See [Install](#install) and [Quick start](#quick-start).
|
||||
|
||||
## Four ways to use it
|
||||
|
||||
- **Desktop app (Studio).** A graphical app; the best place to start.
|
||||
- **Terminal.** A full-screen text interface; `reasonix run "task"` also works in scripts.
|
||||
- **Browser.** `reasonix web` opens a web interface on your own machine.
|
||||
- **Editor.** Install the extension in VS Code; other editors that support ACP (Agent Client Protocol, a common way for an editor to connect to an AI assistant) can connect to `reasonix acp`.
|
||||
|
||||
All four are the same program underneath. What follows is for advanced users and contributors.
|
||||
|
||||
## Versions
|
||||
|
||||
Reasonix ships on two lines. See the [version roadmap announcement](https://github.com/esengine/DeepSeek-Reasonix/discussions/10748) for the reasoning behind the split.
|
||||
|
||||
| Line | Branch | Status | Get it |
|
||||
| --- | --- | --- | --- |
|
||||
| **Reasonix 2.x** | `studio` (this branch) | Active development | [Studio releases](https://github.com/esengine/DeepSeek-Reasonix/releases?q=studio-v&expanded=true) |
|
||||
| **Reasonix 2.x** | `studio` (this branch) | Active development, pre-release | [Studio releases](https://github.com/esengine/DeepSeek-Reasonix/releases?q=studio-v&expanded=true) |
|
||||
| **Reasonix 1.x** | [`main-v2`](https://github.com/esengine/DeepSeek-Reasonix/tree/main-v2) | Maintenance / stable | `npm i -g reasonix` · [desktop download](https://reasonix.io/?download=desktop#start) |
|
||||
|
||||
- **Want something stable?** Stay on 1.x. It keeps receiving bug fixes,
|
||||
@@ -118,8 +75,6 @@ or reimplemented on 2.x where they fit the new architecture.
|
||||
|
||||
## Features
|
||||
|
||||
Technical highlights for advanced users.
|
||||
|
||||
- **Config-driven.** Providers, the agent, enabled tools, and plugins are all
|
||||
declared in `reasonix.toml`. No hardcoded models.
|
||||
- **Multi-model & composable.** DeepSeek ships as a preset; any
|
||||
@@ -144,14 +99,14 @@ Studio, and editor integrations all run the same local Reasonix engine.
|
||||
|
||||
Download the package for your platform from the latest
|
||||
[Studio release](https://github.com/esengine/DeepSeek-Reasonix/releases?q=studio-v&expanded=true)
|
||||
(tagged `studio-v2.*`; candidates such as `-rc.N` are marked as
|
||||
pre-releases):
|
||||
(tagged `studio-v2.*`, published as pre-releases while 2.x is in active
|
||||
development):
|
||||
|
||||
| Platform | Package | Architecture |
|
||||
| --- | --- | --- |
|
||||
| macOS | `.dmg` or `.zip` | Apple Silicon (`arm64`) / Intel (`amd64`) |
|
||||
| Windows | Installer `-installer.exe` or portable `.zip` | x64 |
|
||||
| Linux | `.deb` or `.rpm` | x64 (`amd64`) / ARM64 (`arm64`, aarch64) |
|
||||
| Linux | `.deb` | x64 |
|
||||
|
||||
Every package ships with a `.minisig` signature, and the release carries
|
||||
`SHA256SUMS`. Once installed, Studio updates itself in place.
|
||||
@@ -276,11 +231,11 @@ A small list of folks whose work has shaped Reasonix the most — the current to
|
||||
<!-- reasonix-top-contributors:start -->
|
||||
| Contributor | Contributor | Contributor | Contributor |
|
||||
| --- | --- | --- | --- |
|
||||
| [**SivanCola**](https://github.com/SivanCola) | [**KHG420**](https://github.com/KHG420) | [**ttmouse**](https://github.com/ttmouse) | [**lifu963**](https://github.com/lifu963) |
|
||||
| [**HUQIANTAO**](https://github.com/HUQIANTAO) | [**Harbor404**](https://github.com/Harbor404) | [**GTC2080**](https://github.com/GTC2080) | [**mchenziyi**](https://github.com/mchenziyi) |
|
||||
| [**Li-Charles-One**](https://github.com/Li-Charles-One) | [**light-front-theory**](https://github.com/light-front-theory) | [**XTLine**](https://github.com/XTLine) | [**eghrhegpe**](https://github.com/eghrhegpe) |
|
||||
| [**noreply**](https://github.com/noreply) | [**chuiwenwei**](https://github.com/chuiwenwei) | [**xxoingr**](https://github.com/xxoingr) | [**Bernardxu123**](https://github.com/Bernardxu123) |
|
||||
| [**HaoyueQin**](https://github.com/HaoyueQin) | [**BuGlessRB**](https://github.com/BuGlessRB) | [**CVEngineer66**](https://github.com/CVEngineer66) | [**JesonChou**](https://github.com/JesonChou) |
|
||||
| [**SivanCola**](https://github.com/SivanCola) | [**esengine**](https://github.com/esengine) | [**ttmouse**](https://github.com/ttmouse) | [**lifu963**](https://github.com/lifu963) |
|
||||
| **reasonix** | [**HUQIANTAO**](https://github.com/HUQIANTAO) | [**GTC2080**](https://github.com/GTC2080) | [**mchenziyi**](https://github.com/mchenziyi) |
|
||||
| [**Li-Charles-One**](https://github.com/Li-Charles-One) | **merge-order-check** | [**light-front-theory**](https://github.com/light-front-theory) | **Yan Li** |
|
||||
| [**eghrhegpe**](https://github.com/eghrhegpe) | **wufengfan** | [**Bernardxu123**](https://github.com/Bernardxu123) | [**HaoyueQin**](https://github.com/HaoyueQin) |
|
||||
| [**CVEngineer66**](https://github.com/CVEngineer66) | [**KHG420**](https://github.com/KHG420) | [**JesonChou**](https://github.com/JesonChou) | [**SuMuxi66**](https://github.com/SuMuxi66) |
|
||||
<!-- reasonix-top-contributors:end -->
|
||||
|
||||
Special thanks to [**Bernardxu123**](https://github.com/Bernardxu123) for designing the project logo and intro video.
|
||||
@@ -310,9 +265,8 @@ coffee, not a contract — donations don't buy feature priority or change how
|
||||
issues get triaged.
|
||||
|
||||
- **International** — PayPal: [paypal.me/yuhuahui](https://paypal.me/yuhuahui)
|
||||
- **国内** — 微信支付 / 支付宝(扫码)
|
||||
- **国内** — 微信支付(扫码)
|
||||
|
||||
<p align="center">
|
||||
<img src=".github/sponsor/wechat-pay.jpg" alt="WeChat Pay QR code" width="180"/>
|
||||
<img src=".github/sponsor/alipay.jpg" alt="Alipay QR code" width="180"/>
|
||||
</p>
|
||||
+12
-58
@@ -18,8 +18,6 @@
|
||||
<a href="https://esengine.github.io/DeepSeek-Reasonix/">官方网站</a>
|
||||
·
|
||||
<strong><a href="https://discord.gg/XF78rEME2D">Discord</a></strong>
|
||||
·
|
||||
<strong><a href="https://qm.qq.com/q/i59b0z2R8s">QQ</a></strong>
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
@@ -33,7 +31,6 @@
|
||||
<a href="https://github.com/esengine/DeepSeek-Reasonix/graphs/contributors"><img src="https://img.shields.io/github/contributors/esengine/DeepSeek-Reasonix.svg?style=flat-square&color=bc8cff&labelColor=161b22&logo=github&logoColor=white" alt="contributors"/></a>
|
||||
<a href="https://github.com/esengine/DeepSeek-Reasonix/discussions"><img src="https://img.shields.io/github/discussions/esengine/DeepSeek-Reasonix.svg?style=flat-square&color=58a6ff&labelColor=161b22&logo=github&logoColor=white" alt="Discussions"/></a>
|
||||
<a href="https://discord.gg/XF78rEME2D"><img src="https://img.shields.io/badge/discord-join-5865F2.svg?style=flat-square&labelColor=161b22&logo=discord&logoColor=white" alt="Discord"/></a>
|
||||
<a href="https://qm.qq.com/q/i59b0z2R8s"><img src="https://img.shields.io/badge/QQ%20group-1093562660-12B7F5.svg?style=flat-square&labelColor=161b22&logo=tencentqq&logoColor=white" alt="QQ group 1093562660"/></a>
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
@@ -43,53 +40,13 @@
|
||||
|
||||
<br/>
|
||||
|
||||
<p align="center"><strong>开源 · MIT · 程序跑在你电脑上,模型自选</strong></p>
|
||||
<h3 align="center">终端、桌面、浏览器、编辑器里都能用的 AI 编程助手。</h3>
|
||||
<p align="center">它会读你的项目、改文件、跑命令和测试。每一步先问你,或在你划定的范围内自己做,由你选;它用编辑工具改过的文件可以退回去。第一次用,从桌面应用 Studio 开始。</p>
|
||||
<p align="center"><strong>开源 · MIT · 单个 Go 二进制</strong></p>
|
||||
<h3 align="center">可以一直开着跑的编码 Agent。</h3>
|
||||
<p align="center">一套本地引擎,四个入口——终端、桌面端、浏览器,或通过 ACP 接入你的编辑器。计划模式、权限、工作区沙箱与逐轮 checkpoint,让长时间自治运行始终可读、可撤销。</p>
|
||||
<p align="center">维护者:<strong>Huahui Yu</strong>。</p>
|
||||
|
||||
> [!IMPORTANT]
|
||||
> **加入社区 · Community** — 双语 Discord,提供安装答疑(`#help` / `#求助`)、工作流展示与功能想法。→ **<https://discord.gg/XF78rEME2D>**
|
||||
>
|
||||
> QQ 群 **DeepSeek-Reasonix官方群**(群号 `1093562660`)→ **<https://qm.qq.com/q/i59b0z2R8s>**([二维码](./docs/assets/qq-group.svg))
|
||||
|
||||
> 抖音:**做游戏的小鱼**(抖音号 `22703872788`)· [二维码](./docs/assets/douyin.png)
|
||||
|
||||
## ❤️赞助商
|
||||
|
||||
<table><tr><td width="180" align="center" valign="middle"><a href="https://www.rightapi.ai/register?aff=e083bf73" target="_blank" rel="sponsored noopener noreferrer"><img src="./.github/sponsor/rightcode.png" alt="Right Code" width="64"></a></td><td valign="middle">感谢 Right Code 赞助了本项目!Right Code 稳定提供 Claude Code、Codex、Gemini 等模型的中转服务,并可选按量、包月两种计费模式。充值即可开票,企业、团队用户一对一对接。通过<a href="https://www.rightapi.ai/register?aff=e083bf73" target="_blank" rel="sponsored noopener noreferrer">此链接</a>注册。</td></tr><tr><td width="180" align="center" valign="middle"><a href="https://www.packyapi.ai/register?aff=rIbQ" target="_blank" rel="sponsored noopener noreferrer"><picture><source media="(prefers-color-scheme: dark)" srcset="./.github/sponsor/packycode-dark.svg"><img src="./.github/sponsor/packycode-light.svg" alt="PackyCode" width="80"></picture></a></td><td valign="middle">感谢 PackyCode 赞助了本项目!PackyCode 是一家稳定、高效的 API 中转服务商,统一域名、统一密钥、智能容灾切换。通过<a href="https://www.packyapi.ai/register?aff=rIbQ" target="_blank" rel="sponsored noopener noreferrer">此链接</a>注册。</td></tr></table>
|
||||
|
||||
## 它是什么
|
||||
|
||||
Reasonix 是一个开源的 AI 编程助手。程序跑在你的电脑上;AI 模型是你自己配置的服务,它读到的内容会发给这个服务。你用大白话描述任务,它在你选的项目文件夹里读代码、改文件、跑命令和测试,每一步都摆给你看。
|
||||
|
||||
模型你自己选:内置 DeepSeek 预设,其他兼容 OpenAI 接口的服务填一条配置就行,费用直接付给模型服务商。
|
||||
|
||||
## 你能得到什么
|
||||
|
||||
- **读、改、跑**:读你的项目、改文件、跑测试和命令,都在你选的项目文件夹里。
|
||||
- **权限分档**:每次都问你,或让日常操作自动放行;你设的禁止规则始终有效。想先看方案,就开计划模式,批准后它才动手。
|
||||
- **可回退**:改文件前先存快照,按对话轮次回退,不碰 git 历史。只覆盖编辑工具改的文件,命令行命令造成的改动不在内。
|
||||
|
||||
## 30 秒上手
|
||||
|
||||
下载、填模型密钥、打开项目。第一次用,从桌面应用 Studio 开始。
|
||||
|
||||
1. 从[发布页](https://github.com/esengine/DeepSeek-Reasonix/releases?q=studio-v&expanded=true)下载并安装桌面应用 Reasonix Studio。
|
||||
2. 打开应用,按提示填入模型服务商和密钥。
|
||||
3. 打开你的项目文件夹,写下想做的事,例如“给 http 客户端加上失败重试”。
|
||||
4. 按你选的档位,该问你的操作它会先停下来问;同意就继续,不同意就拒绝。
|
||||
|
||||
更喜欢终端的话,下载 `reasonix` 命令行后运行 `reasonix setup` 配好模型,再运行 `reasonix` 即可。详见[安装](#安装)和[快速开始](#快速开始)。
|
||||
|
||||
## 四种用法
|
||||
|
||||
- **桌面应用(Studio)**:图形界面,适合第一次使用。
|
||||
- **终端**:全屏文字界面,也能用 `reasonix run "任务"` 在脚本里运行。
|
||||
- **浏览器**:`reasonix web` 在本机打开网页界面。
|
||||
- **编辑器**:VS Code 安装扩展即可;其他支持 ACP(Agent Client Protocol,编辑器接入某个 AI 助手的通用约定)的编辑器,可连接 `reasonix acp`。
|
||||
|
||||
四种用法背后是同一套程序。下面的内容写给进阶用户和贡献者。
|
||||
|
||||
## 版本
|
||||
|
||||
@@ -97,7 +54,7 @@ Reasonix 分为两条版本线,调整的原因见[版本路线公告](https://
|
||||
|
||||
| 版本线 | 分支 | 状态 | 获取方式 |
|
||||
| --- | --- | --- | --- |
|
||||
| **Reasonix 2.x** | `studio`(当前分支) | 活跃开发 | [Studio 发布页](https://github.com/esengine/DeepSeek-Reasonix/releases?q=studio-v&expanded=true) |
|
||||
| **Reasonix 2.x** | `studio`(当前分支) | 活跃开发,预发布 | [Studio 发布页](https://github.com/esengine/DeepSeek-Reasonix/releases?q=studio-v&expanded=true) |
|
||||
| **Reasonix 1.x** | [`main-v2`](https://github.com/esengine/DeepSeek-Reasonix/tree/main-v2) | 维护 / 稳定 | `npm i -g reasonix` · [桌面端下载](https://reasonix.io/?download=desktop#start) |
|
||||
|
||||
- **想要稳定**:继续使用 1.x。它会持续收到 Bug 修复、Provider / API 兼容、
|
||||
@@ -112,8 +69,6 @@ Reasonix 分为两条版本线,调整的原因见[版本路线公告](https://
|
||||
|
||||
## 特性
|
||||
|
||||
以下是写给进阶用户的技术要点。
|
||||
|
||||
- **配置驱动**:provider、agent、启用的工具、插件全部在 `reasonix.toml` 中声明,
|
||||
内核无硬编码模型。
|
||||
- **多模型 · 可组合**:DeepSeek 作为预设内置;任何 OpenAI 兼容
|
||||
@@ -133,13 +88,13 @@ Reasonix 分为两条版本线,调整的原因见[版本路线公告](https://
|
||||
### Reasonix Studio(2.x)
|
||||
|
||||
从最新的 [Studio 发布](https://github.com/esengine/DeepSeek-Reasonix/releases?q=studio-v&expanded=true)
|
||||
下载对应平台的安装包(标签为 `studio-v2.*`;`-rc.N` 等候选版本标为预发布):
|
||||
下载对应平台的安装包(标签为 `studio-v2.*`,2.x 活跃开发期间以预发布形式发布):
|
||||
|
||||
| 平台 | 安装包 | 架构 |
|
||||
| --- | --- | --- |
|
||||
| macOS | `.dmg` 或 `.zip` | Apple Silicon(`arm64`)/ Intel(`amd64`) |
|
||||
| Windows | 安装器 `-installer.exe` 或便携 `.zip` | x64 |
|
||||
| Linux | `.deb` 或 `.rpm` | x64(`amd64`)/ ARM64(`arm64`,aarch64) |
|
||||
| Linux | `.deb` | x64 |
|
||||
|
||||
每个安装包都附带 `.minisig` 签名,发布页另有 `SHA256SUMS`。安装后 Studio 会在
|
||||
应用内自行更新。
|
||||
@@ -258,11 +213,11 @@ CLI 进阶用法和详细配置见 **[CLI 命令参考](./docs/CLI.zh-CN.md)**
|
||||
<!-- reasonix-top-contributors:start -->
|
||||
| Contributor | Contributor | Contributor | Contributor |
|
||||
| --- | --- | --- | --- |
|
||||
| [**SivanCola**](https://github.com/SivanCola) | [**KHG420**](https://github.com/KHG420) | [**ttmouse**](https://github.com/ttmouse) | [**lifu963**](https://github.com/lifu963) |
|
||||
| [**HUQIANTAO**](https://github.com/HUQIANTAO) | [**Harbor404**](https://github.com/Harbor404) | [**GTC2080**](https://github.com/GTC2080) | [**mchenziyi**](https://github.com/mchenziyi) |
|
||||
| [**Li-Charles-One**](https://github.com/Li-Charles-One) | [**light-front-theory**](https://github.com/light-front-theory) | [**XTLine**](https://github.com/XTLine) | [**eghrhegpe**](https://github.com/eghrhegpe) |
|
||||
| [**noreply**](https://github.com/noreply) | [**chuiwenwei**](https://github.com/chuiwenwei) | [**xxoingr**](https://github.com/xxoingr) | [**Bernardxu123**](https://github.com/Bernardxu123) |
|
||||
| [**HaoyueQin**](https://github.com/HaoyueQin) | [**BuGlessRB**](https://github.com/BuGlessRB) | [**CVEngineer66**](https://github.com/CVEngineer66) | [**JesonChou**](https://github.com/JesonChou) |
|
||||
| [**SivanCola**](https://github.com/SivanCola) | [**esengine**](https://github.com/esengine) | [**ttmouse**](https://github.com/ttmouse) | [**lifu963**](https://github.com/lifu963) |
|
||||
| **reasonix** | [**HUQIANTAO**](https://github.com/HUQIANTAO) | [**GTC2080**](https://github.com/GTC2080) | [**mchenziyi**](https://github.com/mchenziyi) |
|
||||
| [**Li-Charles-One**](https://github.com/Li-Charles-One) | **merge-order-check** | [**light-front-theory**](https://github.com/light-front-theory) | **Yan Li** |
|
||||
| [**eghrhegpe**](https://github.com/eghrhegpe) | **wufengfan** | [**Bernardxu123**](https://github.com/Bernardxu123) | [**HaoyueQin**](https://github.com/HaoyueQin) |
|
||||
| [**CVEngineer66**](https://github.com/CVEngineer66) | [**KHG420**](https://github.com/KHG420) | [**JesonChou**](https://github.com/JesonChou) | [**SuMuxi66**](https://github.com/SuMuxi66) |
|
||||
<!-- reasonix-top-contributors:end -->
|
||||
|
||||
特别感谢 [**Bernardxu123**](https://github.com/Bernardxu123) 设计的项目 logo和开场视频。
|
||||
@@ -290,10 +245,9 @@ CLI 进阶用法和详细配置见 **[CLI 命令参考](./docs/CLI.zh-CN.md)**
|
||||
如果 Reasonix 帮你省了时间或 token,欢迎请杯咖啡。捐助不会换来 feature
|
||||
优先级,也不会影响 issue 的处理顺序——就是「谢谢」。
|
||||
|
||||
- **国内** — 微信支付 / 支付宝(扫下方二维码)
|
||||
- **国内** — 微信支付(扫下方二维码)
|
||||
- **海外** — PayPal: [paypal.me/yuhuahui](https://paypal.me/yuhuahui)
|
||||
|
||||
<p align="center">
|
||||
<img src=".github/sponsor/wechat-pay.jpg" alt="微信支付收款码" width="180"/>
|
||||
<img src=".github/sponsor/alipay.jpg" alt="支付宝收款码" width="180"/>
|
||||
</p>
|
||||
+5
-33
@@ -219,11 +219,9 @@ otherwise the block the model already has goes on reading as current.
|
||||
- `#<note>` in chat quick-adds an always-on instruction. The `remember` tool
|
||||
instead saves a fallible background fact (frontmatter file + `MEMORY.md`
|
||||
index). Fact `type` classifies content; independent `scope` controls whether it
|
||||
is project-only (the default) or explicitly global.
|
||||
- The saved-fact index stays out of the stable prefix and is reached through the
|
||||
`memory` tool; pinned bodies and global user/feedback bodies load into it, the
|
||||
latter as lower-priority compatibility guidance. The current turn receives a
|
||||
tail note.
|
||||
is project-only (the default) or explicitly global. The index loads into the
|
||||
stable prefix on the next session; global user/feedback bodies also load as
|
||||
lower-priority compatibility guidance. The current turn receives a tail note.
|
||||
|
||||
## Notes
|
||||
|
||||
@@ -322,31 +320,6 @@ Use `go test ./path/to/target/` to detect cycles **before** pushing. A `[setup f
|
||||
- **Keep the PR diff minimal.** Only the files relevant to the PR's purpose — no stray changes from other branches.
|
||||
- **Amend, don't add commits, for review feedback** — keeps the commit history clean.
|
||||
|
||||
## Change discipline
|
||||
|
||||
**Assess before editing.** For every change, humans and agents list callers,
|
||||
consumers of the same state, sibling features, reload/replay/restart paths,
|
||||
error and edge states, TUI/CLI and desktop equivalents, and wire mirrors.
|
||||
|
||||
A fix that breaks a neighbouring behaviour is a failed fix.
|
||||
|
||||
**Test the neighbourhood.** UI changes add or extend interaction tests for
|
||||
adjacent functions of the same component or flow (composer: submit, clear,
|
||||
queue, withdraw, restore, steer, failure, replay after reload) and run the whole
|
||||
affected suite.
|
||||
|
||||
Write tests first; they must fail without the change.
|
||||
|
||||
**Fix at the owning layer.** Use the existing unified pattern: no special-case
|
||||
flags, duplicated paths, or workarounds at the symptom site.
|
||||
|
||||
If the design is duplicated or wrong, make the smallest unifying change:
|
||||
one path instead of two, deleting what becomes dead. Use current language and
|
||||
framework idioms; do not add a new pattern where one exists.
|
||||
|
||||
**PR evidence.** Every PR description carries: Cause, Blast radius,
|
||||
Neighbouring behaviours tested, and Why this layer.
|
||||
|
||||
## Reasonix host checks
|
||||
|
||||
The paths below decide what the agent is allowed to do — a wrong change here
|
||||
@@ -369,7 +342,6 @@ out of how a path is spelled, which cannot tell `internal/auth` from
|
||||
- sensitive: internal/ext/pluginpkg/**
|
||||
- sensitive: internal/base/netclient/**
|
||||
- sensitive: internal/safety/redirectguard/**
|
||||
- sensitive: internal/safety/endpointclient/**
|
||||
- sensitive: internal/safety/egress/**
|
||||
- sensitive: internal/platform/browser/**
|
||||
- sensitive: internal/platform/computer/**
|
||||
@@ -403,8 +375,8 @@ observation, never by a check that ran afterwards.
|
||||
What follows bounds Observable today. It is not a restatement of the invariant:
|
||||
moving a number here must not move the promise.
|
||||
|
||||
- `scanWorkspace` walks the workspace and every granted extra root recursively,
|
||||
skips VCS store dirs, and is `complete` only below 50k files in total. Over that, or past a walk error, it
|
||||
- `scanWorkspace` walks the workspace recursively, skips VCS store dirs, and is
|
||||
`complete` only below 50k files. Over that, or past a walk error, it
|
||||
establishes nothing and the mutation stays unproven.
|
||||
- Symlinks are recorded by `Lstat`, so a write through one lands outside what
|
||||
the walk compares. Confining the resolved target is the sandbox's job.
|
||||
|
||||
@@ -420,7 +420,7 @@ own outcome).
|
||||
| `-suite` | `benchmarks/e2e` | Suite root (must contain `tasks/<id>/`). |
|
||||
| `-task` | *(all)* | Suite mode: run only these comma-separated task IDs (e.g. `-task fix-add-bug`); unknown IDs fail with the available list. |
|
||||
| `-attempts` | `1` | Suite and diff modes: retry a task until an attempt passes, up to N; enables the `Pass@≤N` KPI, and TTCS charges a retried solve with its failed attempts' wall. |
|
||||
| `-trials` | `1` | Suite mode: run every task N times and keep every outcome. Adds a **Reliability** line: `pass^N` (complete tasks that passed every trial) and `pass@N` (complete tasks with any pass), both over the tasks that ran all N trials only (`n/a` when none did), mean `pass@1` over every graded trial, and the flaky tasks by name; a task the token budget cut short is listed on an **Incomplete** line and is in neither. `Pass@≤N` asks whether the agent can solve a task; `pass^N` asks whether it does so every time. Cannot be combined with `-attempts`. |
|
||||
| `-trials` | `1` | Suite mode: run every task N times and keep every outcome. Adds a **Reliability** line: `pass^N` (tasks that passed every trial), `pass@N` (any trial), mean `pass@1` over all trials, and the flaky tasks by name. `Pass@≤N` asks whether the agent can solve a task; `pass^N` asks whether it does so every time. Cannot be combined with `-attempts`. |
|
||||
| `-tape-record DIR` | — | Suite mode, with `-meter`: record every provider exchange at the meter, one tape per task and trial (`DIR/<task>/trial-N/0001.request.json`, `.response.json`, `.response.body`). Taped runs use a fixed workdir path and pin the seeded files' times, because both reach the model through the workspace block and file listings. |
|
||||
| `-tape-replay DIR` | — | Suite mode, with `-meter`: answer each run from its tape instead of the provider, and add a **Replay** line naming the first request whose body differs from the recorded one, down to the message (`request 2, message 4 (tool)`), the tool list, or a field. The diverging request is kept as `NNNN.replayed.json` beside the recorded one. A kernel change that should not alter what the model is asked replays with every request matched. |
|
||||
| `-bin` | `reasonix` | Path to the reasonix binary. |
|
||||
|
||||
@@ -3,12 +3,12 @@
|
||||
Two questions about long-horizon memory, and the harness that makes their
|
||||
answers trustworthy.
|
||||
|
||||
-mode=preflight build all 18 tasks under every arm and check the contract
|
||||
-mode=preflight build all 12 tasks under every arm and check the contract
|
||||
-mode=snippet does a rank-1 hit hand over the whole answer?
|
||||
-mode=adversarial send a model after the answer through every host surface
|
||||
-mode=run-search Experiment R: is searchable canonical a real capability?
|
||||
-mode=run-boundary Experiment I, paired: the 6 efficiency tasks across their boundary
|
||||
-mode=run-index Experiment I, full: the 6 efficiency tasks x 4 budgets
|
||||
-mode=run-boundary Experiment I, paired: 6 tasks across their own cue boundary
|
||||
-mode=run-index Experiment I, full: 6 tasks x 4 index budgets
|
||||
-mode=calibrate find each index task's PlantAfterGen for its cue tier
|
||||
|
||||
`-dry` drives any run mode with a scripted provider, for nothing. Real runs need
|
||||
@@ -22,28 +22,17 @@ addresses by hand at roughly nine times the page-in cost (139 vs 1292
|
||||
tokens/task). Measured after three separate leaks were closed, so the numbers
|
||||
are from a clean environment.
|
||||
|
||||
**Fold index marginal utility: direction reproduced, size still conditional.** On the v2
|
||||
substrate the direction holds in six of six tasks and in the pooled paired batch: with the
|
||||
cue visible the index addresses the target, so the same task is searched less and pages in
|
||||
less (0.67 searches and 70 recall tokens per task, against 1.50 and 120 with the cue
|
||||
absent).
|
||||
**Fold index marginal utility: unresolved.** Both the positive and the negative
|
||||
estimates are invalidated. The first Stage 2 ran in a contaminated environment.
|
||||
The clean paired batch found a consistent effect (six of six tasks searched less
|
||||
and paged in less with the cue visible), and the same-batch dose-response did
|
||||
not reproduce it (`boundary-aligned 0/4`, recall-token delta reversing sign).
|
||||
The most likely reason is that two of the six index tasks carry heavy-tail
|
||||
stopping behaviour that swamps the effect being measured.
|
||||
|
||||
A size is visible but is not yet quotable. On the four tasks whose absent-cue arm stayed
|
||||
out of a loop, the index cost 64,952 prompt tokens against 115,551 without, 44% less
|
||||
(`i106-budgetseal` -35,147, `i70-ingestfloor` -8,222, `i37-dispatchlag` -7,599, and
|
||||
`i112-lagquota` +369, the prefix rent a task that never searches pays). That is a
|
||||
**post-hoc subset**: `i118-fencelag` and `i47-tenantlease` are excluded because their
|
||||
absent-cue arm looped in this batch (five and three retrieval rounds, three escapes each;
|
||||
together 80% of the absent-cue total), and the qualification above does not screen for
|
||||
stopping behaviour — holding on both boundary arms across three passes shows a stable cue
|
||||
boundary, not a stopping arm. Quoting 44% as the index's effect would mean choosing the
|
||||
sample after seeing the result; a batch screened on stopping stability is what would make
|
||||
a size quotable, and one arm also reverses either way (`i70-ingestfloor` default, +11,135
|
||||
prompt tokens over three escapes).
|
||||
|
||||
Batch facts, because the counts differ: 36 runs were executed (12 paired, 24 index), and
|
||||
the token totals here cover the 24-run index batch — the paired batch ran before the
|
||||
per-run usage line existed.
|
||||
Existing evidence is not sufficient to tune the shipped 1% budget in either
|
||||
direction. Reopening this wants a new corpus, not more samples: see
|
||||
"Index corpus v2" below.
|
||||
|
||||
**Things that turned out not to be problems**, each after being measured rather
|
||||
than argued about:
|
||||
@@ -85,83 +74,11 @@ scanned for answer literals before the first provider request and again after
|
||||
the turn. Any answer reaching the model through a tool that is not `recall`
|
||||
marks the run contaminated and takes it out of every statistic.
|
||||
|
||||
## Index corpus v2
|
||||
## Index corpus v2, if this is reopened
|
||||
|
||||
The efficiency substrate is six tasks that hold on both arms: `i37-dispatchlag`
|
||||
and `i118-fencelag` (quarter), `i106-budgetseal` and `i70-ingestfloor` (half),
|
||||
`i47-tenantlease` and `i112-lagquota` (default).
|
||||
|
||||
They came out of five batches of candidates, each screened the same way: one
|
||||
`-mode=run-index` pass over the batch, then two more only for the candidates that
|
||||
were clean on both boundary arms, and only the candidates that held on both arms
|
||||
in all three passes stayed. A candidate that does not hold is dropped, not
|
||||
promoted. The numbers are candidates, per tier:
|
||||
|
||||
| batch | quarter | half | default |
|
||||
| --- | --- | --- | --- |
|
||||
| first pool, `i31`-`i54` | 1 of 8 | 0 of 8 | 2 of 8 |
|
||||
| second pool, `i61`-`i84` | 1 of 4 screened | 4 of 8 | 0 of 8 |
|
||||
| third pool, `i85`-`i108` | 2 of 8 | 8 of 16 | none authored |
|
||||
| replacement pool, `i109`-`i116` | - | - | 4 of 8 |
|
||||
| replacement pool, `i92`, `i117`-`i124` | 3 of 9 | - | - |
|
||||
|
||||
That table counts candidates that held on both arms in all three passes, which is
|
||||
not the size of the substrate. 89 candidates were authored, 77 were screened (the
|
||||
second pool's quarter tier stopped at 4 of 8 and its default tier was never run),
|
||||
and 25 held: 7 quarter, 12 half, 6 default. The substrate keeps two per tier, so
|
||||
the other 19 survivors are spares rather than corpus members. Candidates from the
|
||||
earlier round are not counted here.
|
||||
|
||||
| task | tier | cue present | cue absent |
|
||||
| --- | --- | ---: | ---: |
|
||||
| `i37-dispatchlag` | quarter | 3/3 | 3/3 |
|
||||
| `i118-fencelag` | quarter | 3/3 | 3/3 |
|
||||
| `i106-budgetseal` | half | 3/3 | 3/3 |
|
||||
| `i70-ingestfloor` | half | 3/3 | 3/3 |
|
||||
| `i47-tenantlease` | default | 3/3 | 3/3 |
|
||||
| `i112-lagquota` | default | 3/3 | 3/3 |
|
||||
|
||||
Across their 36 judged cells: `SnippetStop` 23, `SnippetThenRead` 10 and
|
||||
`DefensiveRead` 3 - every cell clean, nothing post-sufficient and nothing
|
||||
`NeverSufficient`.
|
||||
|
||||
Four of the first six were flagged in review for sharing answer values, probe
|
||||
queries and prompt wording within a tier. Separating them and re-screening those
|
||||
four changed two of them: `i37` and `i47` held, `i62` and `i51` did not, so they
|
||||
were replaced - `i90-recoverylag` was tried first and did not hold either, and
|
||||
`i118-fencelag` and `i112-lagquota` are what did. The separation was not
|
||||
cosmetic: the shared wording had been carrying two of the six.
|
||||
|
||||
Nothing in the task shape separates the survivors from the rest: within the half
|
||||
tier, plain-number answers survived 0 of 8 in the first pool and 4 of 8 in the
|
||||
second, and codename answers survived 8 of 16 in the third. An earlier claim here
|
||||
that the answer kind decides whether a candidate holds does not survive this
|
||||
table and is withdrawn. The dirty class was `PostSufficientRetrieval` in most
|
||||
failures - the model re-checking a value it already had - but which tasks hit it
|
||||
was not predictable from their shape at this sample.
|
||||
|
||||
`i01`-`i06` are the frozen study's six tasks, kept as `roleStopping`. Measured
|
||||
once each with `-mode=run-index -task <id>`, per task and per arm:
|
||||
|
||||
| task | tier | cue present | cue absent |
|
||||
| --- | --- | --- | --- |
|
||||
| `i01-transport-fallback` | quarter | SnippetStop | DefensiveRead |
|
||||
| `i02-scheduler-handoff` | quarter | PostSufficientRetrieval | PostSufficientRetrieval |
|
||||
| `i03-coalescing` | half | PostSufficientRetrieval | PostSufficientRunaway |
|
||||
| `i04-recovery-fence` | half | DefensiveRead | PostSufficientRunaway |
|
||||
| `i05-cache-probe` | default | SnippetThenRead | PostSufficientRetrieval |
|
||||
| `i06-dispatch-handoff` | default | SnippetStop | PostSufficientRetrieval |
|
||||
|
||||
Five of the six show post-sufficient behaviour in at least one arm (`i02`-`i06`),
|
||||
and `i01` is clean on both. The cue-present arm is clean for four of them
|
||||
(`i01`, `i04`, `i05`, `i06`), which is why they stay in the corpus as the
|
||||
stopping side rather than being retitled. All six are excluded from `run-index`
|
||||
and `run-boundary` batches and stay measurable one at a time with `-task <id>`. A
|
||||
task added to the corpus declares a role, so it cannot arrive without one.
|
||||
|
||||
A task qualifies only if it is clean on both of its boundary arms - the
|
||||
cue-present run and the cue-absent run - in every run of a screening batch. Clean
|
||||
means a `stopping_class` other than `PostSufficientRetrieval` or
|
||||
The present index tasks are not a fit substrate: two of six carry stopping
|
||||
behaviour that dominates the signal. A task should qualify only if a run-index
|
||||
run reports a `stopping_class` other than `PostSufficientRetrieval` or
|
||||
`PostSufficientRunaway`: `SnippetStop`, `SnippetThenRead` and `DefensiveRead`
|
||||
all qualify, and a defensive read is deliberately not counted as waste. The
|
||||
class comes from the three counts in that report — `searches_after_sufficient`,
|
||||
@@ -169,7 +86,8 @@ class comes from the three counts in that report — `searches_after_sufficient`
|
||||
sufficiency was reached and how many rounds followed it. The remaining
|
||||
requirements still apply — successful tool call, answer in the output rather
|
||||
than the cue, rank 1, `FirstHitCoverage` complete, one or two simple values,
|
||||
nothing in the workspace.
|
||||
nothing in the workspace. Keep i03 and i04 — as a stopping corpus, not an
|
||||
efficiency one.
|
||||
|
||||
`-mode=calibrate` is not a screening tool: it builds and reads back the
|
||||
fixtures, and reads nothing else, so its cue-visibility profile is the same for
|
||||
|
||||
@@ -1,145 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// batchInfo is what the mode itself knows before any run happens. Everything
|
||||
// else in the manifest is derived from the runs that were written, so a figure
|
||||
// in it cannot drift from the artifact beside it.
|
||||
type batchInfo struct {
|
||||
Mode string
|
||||
Planned int
|
||||
Started time.Time
|
||||
Dry bool
|
||||
// Repo is the checkout the batch ran from, captured before the harness
|
||||
// isolates its workspace: git cannot describe a directory that is gone.
|
||||
Repo string
|
||||
}
|
||||
|
||||
// batchManifest is written beside results.jsonl so a later reader does not have
|
||||
// to reconstruct what the batch was from a terminal scrollback.
|
||||
type batchManifest struct {
|
||||
SchemaVersion int `json:"schema_version"`
|
||||
Mode string `json:"mode"`
|
||||
StartedAt string `json:"started_at"`
|
||||
FinishedAt string `json:"finished_at"`
|
||||
GitCommit string `json:"git_commit"`
|
||||
GitDirty *bool `json:"git_dirty,omitempty"`
|
||||
Model string `json:"model"`
|
||||
Window int `json:"window"`
|
||||
Generations int `json:"generations"`
|
||||
TaskIDs []string `json:"task_ids"`
|
||||
Arms []string `json:"arms"`
|
||||
PlannedRuns int `json:"planned_runs"`
|
||||
CompletedRuns int `json:"completed_runs"`
|
||||
// Usage totals are absent when nothing reported any: an unmeasured cost must
|
||||
// not read as a measured zero. UsageReportedRuns stays, because zero there
|
||||
// is a fact about the batch.
|
||||
UsageReportedRuns int `json:"usage_reported_runs"`
|
||||
UsageReportedRequests int `json:"usage_reported_requests,omitempty"`
|
||||
PromptTokens int `json:"prompt_tokens,omitempty"`
|
||||
CompletionTokens int `json:"completion_tokens,omitempty"`
|
||||
CacheHitTokens int `json:"cache_hit_tokens,omitempty"`
|
||||
CacheMissTokens int `json:"cache_miss_tokens,omitempty"`
|
||||
UsageEstimated bool `json:"usage_estimated,omitempty"`
|
||||
ResultsSHA256 string `json:"results_sha256"`
|
||||
TrajectoriesSHA256 string `json:"trajectories_sha256"`
|
||||
}
|
||||
|
||||
// gitCommitAndDirty reports the checkout the batch ran from. Unknown is written
|
||||
// rather than omitted, so an absent field is never mistaken for a clean tree.
|
||||
func gitCommitAndDirty(repo string) (commit string, dirty *bool) {
|
||||
commit = "unknown"
|
||||
if repo == "" {
|
||||
return commit, nil
|
||||
}
|
||||
in := func(args ...string) ([]byte, error) {
|
||||
cmd := exec.Command("git", args...)
|
||||
cmd.Dir = repo
|
||||
return cmd.Output()
|
||||
}
|
||||
if out, err := in("rev-parse", "--short", "HEAD"); err == nil {
|
||||
commit = strings.TrimSpace(string(out))
|
||||
}
|
||||
if out, err := in("status", "--porcelain"); err == nil {
|
||||
state := strings.TrimSpace(string(out)) != ""
|
||||
dirty = &state
|
||||
}
|
||||
return commit, dirty
|
||||
}
|
||||
|
||||
func fileSHA256(path string) (string, error) {
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return fmt.Sprintf("%x", sha256.Sum256(data)), nil
|
||||
}
|
||||
|
||||
func sortedKeys(set map[string]bool) []string {
|
||||
out := make([]string, 0, len(set))
|
||||
for k := range set {
|
||||
out = append(out, k)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// writeBatchManifest records the batch beside the runs it describes. It runs
|
||||
// last, so its digests are of what is already on disk.
|
||||
func writeBatchManifest(root string, info batchInfo, all []contextMetrics) error {
|
||||
ids, arms := map[string]bool{}, map[string]bool{}
|
||||
var prompt, completion, cacheHit, cacheMiss, requests, reported int
|
||||
estimated := false
|
||||
for _, m := range all {
|
||||
ids[m.Task] = true
|
||||
arms[m.Arm] = true
|
||||
if m.UsageReportedRequests > 0 {
|
||||
reported++
|
||||
requests += m.UsageReportedRequests
|
||||
prompt += m.PromptTokens
|
||||
completion += m.CompletionTokens
|
||||
cacheHit += m.CacheHitTokens
|
||||
cacheMiss += m.CacheMissTokens
|
||||
estimated = estimated || m.UsageEstimated
|
||||
}
|
||||
}
|
||||
commit, dirty := gitCommitAndDirty(info.Repo)
|
||||
model := runModel
|
||||
if info.Dry {
|
||||
model = "scripted"
|
||||
}
|
||||
manifest := batchManifest{
|
||||
SchemaVersion: 1, Mode: info.Mode,
|
||||
StartedAt: info.Started.UTC().Format(time.RFC3339),
|
||||
FinishedAt: time.Now().UTC().Format(time.RFC3339),
|
||||
GitCommit: commit, GitDirty: dirty,
|
||||
Model: model, Window: fixtureWindow, Generations: fixtureGenerations,
|
||||
TaskIDs: sortedKeys(ids), Arms: sortedKeys(arms),
|
||||
PlannedRuns: info.Planned, CompletedRuns: len(all),
|
||||
UsageReportedRuns: reported, UsageReportedRequests: requests,
|
||||
PromptTokens: prompt, CompletionTokens: completion,
|
||||
CacheHitTokens: cacheHit, CacheMissTokens: cacheMiss, UsageEstimated: estimated,
|
||||
}
|
||||
var err error
|
||||
if manifest.ResultsSHA256, err = fileSHA256(filepath.Join(root, "results.jsonl")); err != nil {
|
||||
return err
|
||||
}
|
||||
if manifest.TrajectoriesSHA256, err = fileSHA256(filepath.Join(root, "trajectories.jsonl")); err != nil {
|
||||
return err
|
||||
}
|
||||
data, err := json.MarshalIndent(manifest, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return os.WriteFile(filepath.Join(root, "batch.json"), append(data, '\n'), 0o644)
|
||||
}
|
||||
@@ -1,91 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func seedArtifacts(t *testing.T, root string) {
|
||||
t.Helper()
|
||||
for _, name := range []string{"results.jsonl", "trajectories.jsonl"} {
|
||||
if err := os.WriteFile(filepath.Join(root, name), []byte("{}\n"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func readManifest(t *testing.T, root string) (batchManifest, string) {
|
||||
t.Helper()
|
||||
data, err := os.ReadFile(filepath.Join(root, "batch.json"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var got batchManifest
|
||||
if err := json.Unmarshal(data, &got); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return got, string(data)
|
||||
}
|
||||
|
||||
func TestBatchManifestRecordsWhatTheBatchWas(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
seedArtifacts(t, root)
|
||||
all := []contextMetrics{
|
||||
{Task: "b", Arm: "index-off"},
|
||||
{Task: "a", Arm: "index-off"},
|
||||
{Task: "a", Arm: "index-half", UsageReportedRequests: 2, PromptTokens: 100,
|
||||
CompletionTokens: 10, CacheHitTokens: 60, CacheMissTokens: 40},
|
||||
}
|
||||
if err := writeBatchManifest(root, batchInfo{Mode: "run-index", Planned: 4, Started: time.Now().Add(-time.Minute)}, all); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, _ := readManifest(t, root)
|
||||
if got.SchemaVersion != 1 || got.Mode != "run-index" || got.PlannedRuns != 4 || got.CompletedRuns != 3 {
|
||||
t.Fatalf("manifest header = %+v", got)
|
||||
}
|
||||
if strings.Join(got.TaskIDs, ",") != "a,b" || strings.Join(got.Arms, ",") != "index-half,index-off" {
|
||||
t.Fatalf("ids/arms = %v / %v, want sorted and unique", got.TaskIDs, got.Arms)
|
||||
}
|
||||
if got.UsageReportedRuns != 1 || got.UsageReportedRequests != 2 || got.PromptTokens != 100 ||
|
||||
got.CompletionTokens != 10 || got.CacheHitTokens != 60 || got.CacheMissTokens != 40 {
|
||||
t.Fatalf("usage totals = %+v", got)
|
||||
}
|
||||
// No repo means git could not describe the checkout: unknown commit, and no
|
||||
// dirty flag at all, because "clean" is a claim this batch cannot make.
|
||||
if got.GitCommit != "unknown" || got.GitDirty != nil {
|
||||
t.Fatalf("git state = %q / %v, want unknown and no dirty flag", got.GitCommit, got.GitDirty)
|
||||
}
|
||||
want := fmt.Sprintf("%x", sha256.Sum256([]byte("{}\n")))
|
||||
if got.ResultsSHA256 != want || got.TrajectoriesSHA256 != want {
|
||||
t.Fatalf("digests = %q / %q, want %q", got.ResultsSHA256, got.TrajectoriesSHA256, want)
|
||||
}
|
||||
}
|
||||
|
||||
// A dry batch pays nothing, so its cost fields must be absent rather than zero:
|
||||
// an unmeasured spend must not read as a measured one.
|
||||
func TestBatchManifestLeavesUnreportedUsageAbsent(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
seedArtifacts(t, root)
|
||||
all := []contextMetrics{{Task: "a", Arm: "index-off"}, {Task: "a", Arm: "index-half"}}
|
||||
if err := writeBatchManifest(root, batchInfo{Mode: "run-index", Planned: 2, Started: time.Now(), Dry: true}, all); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, raw := readManifest(t, root)
|
||||
if got.UsageReportedRuns != 0 {
|
||||
t.Fatalf("usage_reported_runs = %d, want 0 reported", got.UsageReportedRuns)
|
||||
}
|
||||
if got.Model != "scripted" {
|
||||
t.Fatalf("dry batch model = %q, want scripted", got.Model)
|
||||
}
|
||||
for _, absent := range []string{"usage_reported_requests", "prompt_tokens", "completion_tokens", "cache_hit_tokens"} {
|
||||
if strings.Contains(raw, absent) {
|
||||
t.Fatalf("a dry batch wrote %q:\n%s", absent, raw)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -89,15 +89,6 @@ type contextMetrics struct {
|
||||
// resident prefix; this is what is paid dynamically instead.
|
||||
RecallReturnedTokens int `json:"recall_returned_tokens"`
|
||||
|
||||
// Provider usage as reported on the stream, or empty for a dry run. The count
|
||||
// is completions that reported usage, so it is not a count of attempts.
|
||||
UsageReportedRequests int `json:"usage_reported_requests,omitempty"`
|
||||
PromptTokens int `json:"prompt_tokens,omitempty"`
|
||||
CompletionTokens int `json:"completion_tokens,omitempty"`
|
||||
CacheHitTokens int `json:"cache_hit_tokens,omitempty"`
|
||||
CacheMissTokens int `json:"cache_miss_tokens,omitempty"`
|
||||
UsageEstimated bool `json:"usage_estimated,omitempty"`
|
||||
|
||||
Searches []searchAttempt `json:"searches,omitempty"`
|
||||
// PostHitSearches are queries issued after the target had already been
|
||||
// returned: not a query problem, a result-to-read problem.
|
||||
|
||||
@@ -26,12 +26,6 @@ const (
|
||||
tierDefault = "default" // visible at default only
|
||||
)
|
||||
|
||||
// role places an index task in the efficiency substrate or the stopping corpus.
|
||||
const (
|
||||
roleEfficiency = "efficiency"
|
||||
roleStopping = "stopping"
|
||||
)
|
||||
|
||||
// plantKind is how a task's target enters the transcript.
|
||||
const (
|
||||
plantAssistant = "assistant" // the model's own account: never index-addressed
|
||||
@@ -51,8 +45,6 @@ type contextTask struct {
|
||||
AnswerVars []string
|
||||
CueVar string
|
||||
CueTier string
|
||||
// Role is roleEfficiency or roleStopping; it decides the batch a task enters.
|
||||
Role string
|
||||
// PlantAfterGen ages the target: mergeFoldIndex trims from its oldest end,
|
||||
// so what follows a cue decides which budget addresses it. Frozen, because
|
||||
// a run that searched for it would let the tested code move its goalposts.
|
||||
@@ -129,20 +121,6 @@ func num(name string, lo, hi int) varSpec {
|
||||
return varSpec{Name: name, Kind: varInt, Min: lo, Max: hi}
|
||||
}
|
||||
|
||||
// efficiencyTasks is the substrate a study measures on. Every task in it
|
||||
// qualifies under the corpus bar; the stopping corpus is deliberately excluded,
|
||||
// because a task carrying post-sufficient stopping behaviour swamps the effect
|
||||
// an efficiency batch exists to measure.
|
||||
func efficiencyTasks() []contextTask {
|
||||
var out []contextTask
|
||||
for _, t := range indexTasks() {
|
||||
if t.Role == roleEfficiency {
|
||||
out = append(out, t)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func searchTasks() []contextTask {
|
||||
return []contextTask{
|
||||
{
|
||||
@@ -214,104 +192,10 @@ func searchTasks() []contextTask {
|
||||
}
|
||||
}
|
||||
|
||||
// efficiencyIndexTasks is what a study measures on: two tasks per cue tier.
|
||||
func efficiencyIndexTasks() []contextTask {
|
||||
out := indexQuarterTasks()
|
||||
out = append(out, indexHalfTasks()...)
|
||||
out = append(out, indexDefaultTasks()...)
|
||||
return out
|
||||
}
|
||||
|
||||
// indexQuarterTasks holds the quarter-tier literals.
|
||||
func indexQuarterTasks() []contextTask {
|
||||
func indexTasks() []contextTask {
|
||||
return []contextTask{
|
||||
{
|
||||
ID: "i37-dispatchlag", Experiment: experimentIndex, TargetKind: "tool_input", Role: roleEfficiency,
|
||||
Prompt: "早前会话里执行过 ./scripts/dispatchlag,它打印出的 dispatch lag epoch 是多少?",
|
||||
ProbeQuery: "dispatchlag dispatch lag epoch",
|
||||
Vars: []varSpec{code("cue", "dispatchlag"), code("epoch", "cobalt")},
|
||||
AnswerVars: []string{"epoch"}, CueVar: "cue", CueTier: tierQuarter,
|
||||
PlantAfterGen: 4,
|
||||
Plant: plantCall, CallID: "p37", CallTool: "bash",
|
||||
CallArgs: `{"command":"./scripts/dispatchlag {{cue}}"}`,
|
||||
PlantBody: "dispatch lag epoch = {{epoch}}",
|
||||
},
|
||||
{
|
||||
ID: "i118-fencelag", Experiment: experimentIndex, TargetKind: "tool_input", Role: roleEfficiency,
|
||||
Prompt: "早前会话里运行过 ./scripts/fencelag,它打印出的 fence lag 是多少?",
|
||||
ProbeQuery: "fencelag fence lag",
|
||||
Vars: []varSpec{code("cue", "fencelag"), code("v", "slate")},
|
||||
AnswerVars: []string{"v"}, CueVar: "cue", CueTier: tierQuarter,
|
||||
PlantAfterGen: 4,
|
||||
Plant: plantCall, CallID: "x118", CallTool: "bash",
|
||||
CallArgs: `{"command":"./scripts/fencelag {{cue}}"}`,
|
||||
PlantBody: "fence lag = {{v}}",
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// indexHalfTasks holds the half-tier literals.
|
||||
func indexHalfTasks() []contextTask {
|
||||
return []contextTask{
|
||||
{
|
||||
ID: "i70-ingestfloor", Experiment: experimentIndex, TargetKind: "tool_input", Role: roleEfficiency,
|
||||
Prompt: "早前会话里执行过 ./scripts/ingestfloor,它打印出的 route quota 是多少?",
|
||||
ProbeQuery: "ingestfloor quota",
|
||||
Vars: []varSpec{code("cue", "ingestfloor"), num("quota", 1200, 9800)},
|
||||
AnswerVars: []string{"quota"}, CueVar: "cue", CueTier: tierHalf,
|
||||
PlantAfterGen: 2,
|
||||
Plant: plantCall, CallID: "q70", CallTool: "bash",
|
||||
CallArgs: `{"command":"./scripts/ingestfloor {{cue}}"}`,
|
||||
PlantBody: "route quota = {{quota}}",
|
||||
},
|
||||
{
|
||||
ID: "i106-budgetseal", Experiment: experimentIndex, TargetKind: "tool_input", Role: roleEfficiency,
|
||||
Prompt: "早前会话里执行过 ./scripts/budgetseal,它打印出的 route quota 是多少?",
|
||||
ProbeQuery: "budgetseal route quota",
|
||||
Vars: []varSpec{code("cue", "budgetseal"), code("quota", "amber")},
|
||||
AnswerVars: []string{"quota"}, CueVar: "cue", CueTier: tierHalf,
|
||||
PlantAfterGen: 2,
|
||||
Plant: plantCall, CallID: "h106", CallTool: "bash",
|
||||
CallArgs: `{"command":"./scripts/budgetseal {{cue}}"}`,
|
||||
PlantBody: "route quota = {{quota}}",
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// indexDefaultTasks holds the default-tier literals.
|
||||
func indexDefaultTasks() []contextTask {
|
||||
return []contextTask{
|
||||
{
|
||||
ID: "i47-tenantlease", Experiment: experimentIndex, TargetKind: "tool_input", Role: roleEfficiency,
|
||||
Prompt: "早前会话里执行过 ./scripts/tenantlease,它打印出的 tenant lease(秒)是多少?",
|
||||
ProbeQuery: "tenantlease tenant lease",
|
||||
Vars: []varSpec{code("cue", "tenantlease"), {Name: "lease", Kind: varDuration, Min: 15, Max: 300}},
|
||||
AnswerVars: []string{"lease"}, CueVar: "cue", CueTier: tierDefault,
|
||||
PlantAfterGen: 0,
|
||||
Plant: plantCall, CallID: "p47", CallTool: "bash",
|
||||
CallArgs: `{"command":"./scripts/tenantlease {{cue}}"}`,
|
||||
PlantBody: "tenant lease = {{lease}}",
|
||||
},
|
||||
{
|
||||
ID: "i112-lagquota", Experiment: experimentIndex, TargetKind: "tool_input", Role: roleEfficiency,
|
||||
Prompt: "早前会话里执行过 ./scripts/lagquota,它打印出的 lagquota lease 是多少?",
|
||||
ProbeQuery: "lagquota lagquota lease",
|
||||
Vars: []varSpec{code("cue", "lagquota"), {Name: "lease", Kind: varDuration, Min: 20, Max: 420}},
|
||||
AnswerVars: []string{"lease"}, CueVar: "cue", CueTier: tierDefault,
|
||||
PlantAfterGen: 0,
|
||||
Plant: plantCall, CallID: "d112", CallTool: "bash",
|
||||
CallArgs: `{"command":"./scripts/lagquota {{cue}}"}`,
|
||||
PlantBody: "lagquota lease = {{lease}}",
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// stoppingIndexTasks is the frozen study's six tasks, kept out of efficiency
|
||||
// batches and measurable on their own with -task <id>.
|
||||
func stoppingIndexTasks() []contextTask {
|
||||
return []contextTask{
|
||||
{
|
||||
ID: "i01-transport-fallback", Experiment: experimentIndex, TargetKind: "tool_input", Role: roleStopping,
|
||||
ID: "i01-transport-fallback", Experiment: experimentIndex, TargetKind: "tool_input",
|
||||
Prompt: "transport fallback probe 当时报告的 selector 是什么?",
|
||||
ProbeQuery: "transport fallback probe selector",
|
||||
Vars: []varSpec{code("cue", "tfprobe"), code("selector", "fern")},
|
||||
@@ -322,7 +206,7 @@ func stoppingIndexTasks() []contextTask {
|
||||
PlantBody: "fallback selector = {{selector}}",
|
||||
},
|
||||
{
|
||||
ID: "i02-scheduler-handoff", Experiment: experimentIndex, TargetKind: "tool_input", Role: roleStopping,
|
||||
ID: "i02-scheduler-handoff", Experiment: experimentIndex, TargetKind: "tool_input",
|
||||
Prompt: "scheduler handoff 当时记录的 grace 和 marker 是什么?",
|
||||
ProbeQuery: "scheduler handoff grace marker",
|
||||
Vars: []varSpec{
|
||||
@@ -336,7 +220,7 @@ func stoppingIndexTasks() []contextTask {
|
||||
PlantBody: "handoff grace = {{grace}}\nmarker = {{marker}}",
|
||||
},
|
||||
{
|
||||
ID: "i03-coalescing", Experiment: experimentIndex, TargetKind: "tool_input", Role: roleStopping,
|
||||
ID: "i03-coalescing", Experiment: experimentIndex, TargetKind: "tool_input",
|
||||
Prompt: "job completion 的 coalescing probe 最后记录的 window 和 marker 是多少?",
|
||||
ProbeQuery: "coalescing policy completion window",
|
||||
Vars: []varSpec{
|
||||
@@ -350,7 +234,7 @@ func stoppingIndexTasks() []contextTask {
|
||||
PlantBody: "completion coalesce window = {{window}}\nmarker = {{marker}}",
|
||||
},
|
||||
{
|
||||
ID: "i04-recovery-fence", Experiment: experimentIndex, TargetKind: "tool_input", Role: roleStopping,
|
||||
ID: "i04-recovery-fence", Experiment: experimentIndex, TargetKind: "tool_input",
|
||||
Prompt: "recovery fence 那次检查报告的 marker 和两个 epoch 是什么?",
|
||||
ProbeQuery: "recovery fence epoch check",
|
||||
Vars: []varSpec{
|
||||
@@ -364,7 +248,7 @@ func stoppingIndexTasks() []contextTask {
|
||||
PlantBody: "{{marker}}\nexpected epoch {{expected}}\nobserved epoch {{observed}}",
|
||||
},
|
||||
{
|
||||
ID: "i05-cache-probe", Experiment: experimentIndex, TargetKind: "tool_input", Role: roleStopping,
|
||||
ID: "i05-cache-probe", Experiment: experimentIndex, TargetKind: "tool_input",
|
||||
Prompt: "cache probe 当时记录的 prefix salt 和 scope 是什么?",
|
||||
ProbeQuery: "cache probe prefix salt scope",
|
||||
Vars: []varSpec{code("cue", "cacheprobe"), code("salt", "comet"), code("scope", "lineage")},
|
||||
@@ -375,7 +259,7 @@ func stoppingIndexTasks() []contextTask {
|
||||
PlantBody: "prefix salt = {{salt}}\nscope = {{scope}}",
|
||||
},
|
||||
{
|
||||
ID: "i06-dispatch-handoff", Experiment: experimentIndex, TargetKind: "tool_input", Role: roleStopping,
|
||||
ID: "i06-dispatch-handoff", Experiment: experimentIndex, TargetKind: "tool_input",
|
||||
Prompt: "dispatch handoff probe 当时确认的 mode 和 marker 是什么?",
|
||||
ProbeQuery: "dispatch handoff probe mode",
|
||||
Vars: []varSpec{code("cue", "dispatchhandoff"), code("mode", "trigger"), code("marker", "iris")},
|
||||
@@ -388,12 +272,6 @@ func stoppingIndexTasks() []contextTask {
|
||||
}
|
||||
}
|
||||
|
||||
// indexTasks is the whole index corpus, the efficiency substrate first.
|
||||
func indexTasks() []contextTask {
|
||||
out := efficiencyIndexTasks()
|
||||
return append(out, stoppingIndexTasks()...)
|
||||
}
|
||||
|
||||
func allTasks() []contextTask {
|
||||
return append(searchTasks(), indexTasks()...)
|
||||
}
|
||||
@@ -454,9 +332,6 @@ func validateCorpus() error {
|
||||
if t.CueVar == "" {
|
||||
return fmt.Errorf("%s: an index task needs a cue var", t.ID)
|
||||
}
|
||||
if t.Role != roleEfficiency && t.Role != roleStopping {
|
||||
return fmt.Errorf("%s: an index task needs a role (%s or %s)", t.ID, roleEfficiency, roleStopping)
|
||||
}
|
||||
if v, _ := tierScales(t.CueTier); v == nil {
|
||||
return fmt.Errorf("%s: unknown cue tier %q", t.ID, t.CueTier)
|
||||
}
|
||||
|
||||
@@ -36,7 +36,7 @@ func (p *digestProvider) Name() string { return "fixture-digest" }
|
||||
func (p *digestProvider) Stream(_ context.Context, _ provider.Request) (<-chan provider.Chunk, error) {
|
||||
p.calls++
|
||||
ch := make(chan provider.Chunk, 2)
|
||||
ch <- provider.Chunk{Type: provider.ChunkText, Text: "## Earlier work\nEarlier work continued across several files; nothing outstanding was recorded."}
|
||||
ch <- provider.Chunk{Type: provider.ChunkText, Text: "Earlier work continued across several files; nothing outstanding was recorded."}
|
||||
ch <- provider.Chunk{Type: provider.ChunkDone}
|
||||
close(ch)
|
||||
return ch, nil
|
||||
|
||||
@@ -5,7 +5,6 @@ import (
|
||||
"crypto/rand"
|
||||
"fmt"
|
||||
"hash/fnv"
|
||||
"io"
|
||||
mathrand "math/rand"
|
||||
"strings"
|
||||
)
|
||||
@@ -91,25 +90,15 @@ func seededRand(parts ...string) *mathrand.Rand {
|
||||
// liveRand is the generator a paid run uses: every run gets values that existed
|
||||
// nowhere before it started.
|
||||
func liveRand() *mathrand.Rand {
|
||||
rng, err := liveRandFrom(rand.Reader)
|
||||
if err != nil {
|
||||
panic("contextbench: no entropy for a live run: " + err.Error())
|
||||
}
|
||||
return rng
|
||||
}
|
||||
|
||||
// liveRandFrom seeds a generator from an entropy source, so what a run's values
|
||||
// are unpredictable from is the source and nothing else.
|
||||
func liveRandFrom(entropy io.Reader) (*mathrand.Rand, error) {
|
||||
var seed [8]byte
|
||||
if _, err := io.ReadFull(entropy, seed[:]); err != nil {
|
||||
return nil, err
|
||||
if _, err := rand.Read(seed[:]); err != nil {
|
||||
panic("contextbench: no entropy for a live run: " + err.Error())
|
||||
}
|
||||
var n int64
|
||||
for _, b := range seed {
|
||||
n = n<<8 | int64(b)
|
||||
}
|
||||
return mathrand.New(mathrand.NewSource(n)), nil
|
||||
return mathrand.New(mathrand.NewSource(n))
|
||||
}
|
||||
|
||||
// instantiate substitutes {{name}} and refuses anything left over. The grammar
|
||||
|
||||
@@ -54,22 +54,15 @@ func main() {
|
||||
case "preflight":
|
||||
os.Exit(runPreflight(tasks, root))
|
||||
case "run-search":
|
||||
os.Exit(runExperiment(*mode, experimentSearch, root, *dry, tasks))
|
||||
os.Exit(runExperiment(experimentSearch, root, *dry, tasks))
|
||||
case "run-boundary":
|
||||
os.Exit(runBoundaries(*mode, root))
|
||||
os.Exit(runBoundaries(root))
|
||||
case "snippet":
|
||||
os.Exit(runSnippetAudit(root, 100))
|
||||
case "adversarial":
|
||||
os.Exit(runAdversarial(root))
|
||||
case "run-index":
|
||||
batch := tasks
|
||||
if *only == "" {
|
||||
// A batch measures the efficiency substrate only. Naming one task
|
||||
// with -task runs that task, role included, so the stopping corpus
|
||||
// stays measurable on its own.
|
||||
batch = efficiencyTasks()
|
||||
}
|
||||
os.Exit(runExperiment(*mode, experimentIndex, root, *dry, batch))
|
||||
os.Exit(runExperiment(experimentIndex, root, *dry, tasks))
|
||||
default:
|
||||
fmt.Fprintf(os.Stderr, "unknown mode %q\n", *mode)
|
||||
os.Exit(2)
|
||||
|
||||
@@ -1,10 +1,7 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"maps"
|
||||
mathrand "math/rand"
|
||||
"os/exec"
|
||||
"slices"
|
||||
"strings"
|
||||
@@ -22,24 +19,16 @@ func TestCorpusIsWellFormed(t *testing.T) {
|
||||
if got := len(searchTasks()); got != 6 {
|
||||
t.Errorf("search experiment has %d tasks, want 6", got)
|
||||
}
|
||||
if got := len(indexTasks()); got != 12 {
|
||||
t.Errorf("index experiment has %d tasks, want 12", got)
|
||||
if got := len(indexTasks()); got != 6 {
|
||||
t.Errorf("index experiment has %d tasks, want 6", got)
|
||||
}
|
||||
stopping := map[string]bool{}
|
||||
tiers := map[string]int{}
|
||||
for _, task := range indexTasks() {
|
||||
if task.Role == roleStopping {
|
||||
stopping[task.ID] = true
|
||||
continue
|
||||
}
|
||||
tiers[task.CueTier]++
|
||||
}
|
||||
if !stopping["i03-coalescing"] || !stopping["i04-recovery-fence"] {
|
||||
t.Errorf("the stopping corpus is %v; it must keep i03-coalescing and i04-recovery-fence", stopping)
|
||||
}
|
||||
// Two per tier, over the efficiency substrate, is what makes the staircase
|
||||
// readable: the same task can be compared across the boundary its cue sits
|
||||
// on, instead of two different questions being compared across two arms.
|
||||
// Two per tier is what makes the staircase readable: the same task can be
|
||||
// compared across the boundary its cue sits on, instead of two different
|
||||
// questions being compared across two arms.
|
||||
for _, tier := range []string{tierQuarter, tierHalf, tierDefault} {
|
||||
if tiers[tier] != 2 {
|
||||
t.Errorf("tier %s has %d tasks, want 2", tier, tiers[tier])
|
||||
@@ -47,21 +36,6 @@ func TestCorpusIsWellFormed(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// The role decides which batch a task enters; the literal lists are only where
|
||||
// it is declared, so the two must not drift.
|
||||
func TestIndexListsMatchTheirRoles(t *testing.T) {
|
||||
for _, task := range append(append(indexQuarterTasks(), indexHalfTasks()...), indexDefaultTasks()...) {
|
||||
if task.Role != roleEfficiency {
|
||||
t.Errorf("%s sits in an efficiency list with role %q", task.ID, task.Role)
|
||||
}
|
||||
}
|
||||
for _, task := range stoppingIndexTasks() {
|
||||
if task.Role != roleStopping {
|
||||
t.Errorf("%s sits in the stopping list with role %q", task.ID, task.Role)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The invariant the whole instantiation layer exists for: no literal a scorer
|
||||
// needs may be readable in this repository before a run starts.
|
||||
func TestNoAnswerLiteralExistsInTheRepository(t *testing.T) {
|
||||
@@ -85,46 +59,20 @@ func TestNoAnswerLiteralExistsInTheRepository(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// A run's values are unpredictable from a previous run only if the live
|
||||
// generator is seeded from entropy. Comparing answers across two instances
|
||||
// cannot assert that: a task whose answer is a bounded number collides by
|
||||
// chance (1/8601 to 1/286 per task), so the check is on the seed source.
|
||||
func TestLiveRandDrawsFromEntropy(t *testing.T) {
|
||||
a, b := liveRand(), liveRand()
|
||||
for range 4 {
|
||||
if a.Int63() != b.Int63() {
|
||||
return
|
||||
}
|
||||
}
|
||||
t.Fatal("two live generators produced the same stream: they are seeded alike")
|
||||
}
|
||||
|
||||
// The source alone decides an instance: the same entropy gives the same
|
||||
// answers, different entropy gives different values.
|
||||
func TestLiveRandSourceDecidesTheInstance(t *testing.T) {
|
||||
same := func() *mathrand.Rand {
|
||||
rng, err := liveRandFrom(bytes.NewReader(bytes.Repeat([]byte{7}, 8)))
|
||||
// Two runs of the same task must not share an answer, or one leaked run leaks
|
||||
// every future one.
|
||||
func TestInstancesDifferBetweenRuns(t *testing.T) {
|
||||
for _, task := range allTasks() {
|
||||
a, err := instantiateTask(task, liveRand())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return rng
|
||||
}
|
||||
other, err := liveRandFrom(bytes.NewReader(bytes.Repeat([]byte{9}, 8)))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := liveRandFrom(bytes.NewReader(nil)); err == nil {
|
||||
t.Fatal("an empty entropy source was accepted")
|
||||
}
|
||||
for _, task := range allTasks() {
|
||||
a, _ := instantiateTask(task, same())
|
||||
b, _ := instantiateTask(task, same())
|
||||
if !slices.Equal(a.AnswerMarkers, b.AnswerMarkers) || !maps.Equal(a.Vars, b.Vars) {
|
||||
t.Errorf("%s: the same entropy gave different instances", task.ID)
|
||||
b, err := instantiateTask(task, liveRand())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
c, _ := instantiateTask(task, other)
|
||||
if maps.Equal(a.Vars, c.Vars) {
|
||||
t.Errorf("%s: different entropy gave the same values", task.ID)
|
||||
if slices.Equal(a.AnswerMarkers, b.AnswerMarkers) {
|
||||
t.Errorf("%s produced the same answers twice: %v", task.ID, a.AnswerMarkers)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -14,6 +14,7 @@ import (
|
||||
|
||||
"reasonix/internal/assembly/boot"
|
||||
"reasonix/internal/contract/ablation"
|
||||
"reasonix/internal/contract/event"
|
||||
"reasonix/internal/contract/provider"
|
||||
"reasonix/internal/contract/tool"
|
||||
_ "reasonix/internal/model/openai"
|
||||
@@ -131,11 +132,10 @@ func runOne(p provider.Provider, t contextTask, arm ablation.Set, armName, root
|
||||
}
|
||||
boot.ApplyUnifiedProviderToolSurface(reg, false, arm)
|
||||
|
||||
var usage usageSink
|
||||
a := agent.New(p, reg, sess, agent.Options{
|
||||
ContextWindow: fixtureWindow, CompactRatio: 0.5, RecentKeep: 2,
|
||||
SessionPath: f.Path, KeepPolicy: agent.KeepErrors, Ablation: arm,
|
||||
}, &usage)
|
||||
}, event.Discard)
|
||||
a.LoadProjectionSidecar(f.Path)
|
||||
// From here the transcript lives only in memory. On disk it is the answer.
|
||||
sealFixture(f.Path)
|
||||
@@ -161,10 +161,6 @@ func runOne(p provider.Provider, t contextTask, arm ablation.Set, armName, root
|
||||
appended := sess.Snapshot()[before:]
|
||||
m := scoreRun(appended, inst, f.Target, armName, cueVisible)
|
||||
m.trajectory = appended
|
||||
if u, requests := usage.snapshot(); requests > 0 {
|
||||
m.UsageReportedRequests, m.PromptTokens, m.CompletionTokens = requests, u.PromptTokens, u.CompletionTokens
|
||||
m.CacheHitTokens, m.CacheMissTokens, m.UsageEstimated = u.CacheHitTokens, u.CacheMissTokens, u.Estimated
|
||||
}
|
||||
m.scoreAnswer(finalAnswer(appended), inst)
|
||||
if runErr != nil {
|
||||
m.FailureStage = "RunError"
|
||||
@@ -216,10 +212,7 @@ func isolateWorkspace() (func(), error) {
|
||||
|
||||
// The two arms get separate nonces: reusing one would leave the first arm's
|
||||
// answer in a host-readable history for the second to find.
|
||||
func runBoundaries(mode, root string) int {
|
||||
started := time.Now()
|
||||
planned := 0
|
||||
repo, _ := os.Getwd()
|
||||
func runBoundaries(root string) int {
|
||||
root, err := filepath.Abs(root)
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
@@ -239,14 +232,13 @@ func runBoundaries(mode, root string) int {
|
||||
|
||||
var all []contextMetrics
|
||||
var cueSide, noCueSide []contextMetrics
|
||||
for _, t := range efficiencyTasks() {
|
||||
for _, t := range indexTasks() {
|
||||
cue, noCue := boundaryPair(t.CueTier)
|
||||
for _, side := range []struct {
|
||||
scale string
|
||||
has bool
|
||||
}{{cue, true}, {noCue, false}} {
|
||||
name := "index-" + side.scale
|
||||
planned++
|
||||
m, err := runOne(p, t, armFor(side.scale, false), name, root, side.has, false)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "%s [%s]: %v\n", t.ID, name, err)
|
||||
@@ -257,14 +249,14 @@ func runBoundaries(mode, root string) int {
|
||||
} else {
|
||||
noCueSide = append(noCueSide, m)
|
||||
}
|
||||
fmt.Printf("%-24s %-14s cue=%-3v %-14s search=%d read=%d direct=%-5v recall-tok=%-5d%s escape=%d\n",
|
||||
fmt.Printf("%-24s %-14s cue=%-3v %-14s search=%d read=%d direct=%-5v recall-tok=%-5d escape=%d\n",
|
||||
t.ID, name, side.has, m.FailureStage, m.SearchCalls, m.ReadCalls,
|
||||
m.CueDirectRead, m.RecallReturnedTokens, promptTokenSuffix(m), m.EscapeCalls)
|
||||
m.CueDirectRead, m.RecallReturnedTokens, m.EscapeCalls)
|
||||
}
|
||||
}
|
||||
reportDeltas(cueSide, noCueSide)
|
||||
fmt.Print(reportStopping(all))
|
||||
return writeResults(root, batchInfo{Mode: mode, Planned: planned, Started: started, Repo: repo}, all)
|
||||
return writeResults(root, all)
|
||||
}
|
||||
|
||||
// reportDeltas is the paired reading: each task against itself, then pooled.
|
||||
@@ -306,10 +298,7 @@ func deltaBool(a, b bool) string {
|
||||
// runExperiment runs one experiment's tasks across its arms. A dry run drives
|
||||
// the same pipeline with a scripted provider: it proves fixture, run, scoring
|
||||
// and report hold together before any of it is paid for.
|
||||
func runExperiment(mode, experiment, root string, dry bool, tasks []contextTask) int {
|
||||
started := time.Now()
|
||||
planned := 0
|
||||
repo, _ := os.Getwd()
|
||||
func runExperiment(experiment, root string, dry bool, tasks []contextTask) int {
|
||||
root, err := filepath.Abs(root)
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
@@ -345,16 +334,15 @@ func runExperiment(mode, experiment, root string, dry bool, tasks []contextTask)
|
||||
// test: a bad minute would land entirely on one arm.
|
||||
for _, arm := range shuffledArms(t) {
|
||||
cueVisible := t.Experiment == experimentIndex && cueExpectedVisible(t, arm.scale)
|
||||
planned++
|
||||
m, err := runOne(p, t, armFor(arm.scale, arm.searchOff), arm.name, root, cueVisible, dry)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "%s [%s]: %v\n", t.ID, arm.name, err)
|
||||
}
|
||||
byArm[arm.name] = append(byArm[arm.name], m)
|
||||
all = append(all, m)
|
||||
fmt.Printf("%-24s %-14s %-18s search=%d hit=%d read=%d recall-tok=%d%s %v\n",
|
||||
fmt.Printf("%-24s %-14s %-18s search=%d hit=%d read=%d recall-tok=%d %v\n",
|
||||
t.ID, arm.name, m.FailureStage, m.SearchCalls, m.TargetSearchHits, m.ReadCalls,
|
||||
m.RecallReturnedTokens, promptTokenSuffix(m), m.UnexpectedWorkTools)
|
||||
m.RecallReturnedTokens, m.UnexpectedWorkTools)
|
||||
}
|
||||
}
|
||||
reportFunnels(byArm)
|
||||
@@ -367,7 +355,7 @@ func runExperiment(mode, experiment, root string, dry bool, tasks []contextTask)
|
||||
if experiment == experimentIndex {
|
||||
reportBoundaries(all)
|
||||
}
|
||||
return writeResults(root, batchInfo{Mode: mode, Planned: planned, Started: started, Dry: dry, Repo: repo}, all)
|
||||
return writeResults(root, all)
|
||||
}
|
||||
|
||||
func cueExpectedVisible(t contextTask, scale string) bool {
|
||||
@@ -422,7 +410,7 @@ func writeTrajectories(root string, all []contextMetrics) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func writeResults(root string, info batchInfo, all []contextMetrics) int {
|
||||
func writeResults(root string, all []contextMetrics) int {
|
||||
path := filepath.Join(root, "results.jsonl")
|
||||
f, err := os.Create(path)
|
||||
if err != nil {
|
||||
@@ -440,11 +428,7 @@ func writeResults(root string, info batchInfo, all []contextMetrics) int {
|
||||
fmt.Fprintln(os.Stderr, "write trajectories:", err)
|
||||
return 2
|
||||
}
|
||||
if err := writeBatchManifest(root, info, all); err != nil {
|
||||
fmt.Fprintln(os.Stderr, "write batch manifest:", err)
|
||||
return 2
|
||||
}
|
||||
fmt.Printf("\n%d runs written to %s (with trajectories.jsonl and batch.json)\n", len(all), path)
|
||||
fmt.Printf("\n%d runs written to %s (with trajectories.jsonl)\n", len(all), path)
|
||||
return 0
|
||||
}
|
||||
|
||||
|
||||
@@ -1,59 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sync"
|
||||
|
||||
"reasonix/internal/contract/event"
|
||||
"reasonix/internal/contract/provider"
|
||||
)
|
||||
|
||||
// usageSink reads what a run cost from the events the agent already emits. Every
|
||||
// billed completion reports event.Usage — the round path, compaction, command
|
||||
// routing and failure diagnosis all do — so the bench reads the same record the
|
||||
// product reads instead of wrapping a provider and counting a second time.
|
||||
type usageSink struct {
|
||||
mu sync.Mutex
|
||||
requests int
|
||||
usage provider.Usage
|
||||
}
|
||||
|
||||
func (s *usageSink) Emit(e event.Event) {
|
||||
if e.Kind != event.Usage || e.Usage == nil {
|
||||
return
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
// One event is one completion that reported usage; RequestCount covers a
|
||||
// completion that merged attempts. A request that failed before reporting
|
||||
// leaves no event, so this is not a count of attempts.
|
||||
if e.Usage.RequestCount > 0 {
|
||||
s.requests += e.Usage.RequestCount
|
||||
} else {
|
||||
s.requests++
|
||||
}
|
||||
u := *e.Usage
|
||||
s.usage.PromptTokens += u.PromptTokens
|
||||
s.usage.CompletionTokens += u.CompletionTokens
|
||||
s.usage.TotalTokens += u.TotalTokens
|
||||
s.usage.CacheHitTokens += u.CacheHitTokens
|
||||
s.usage.CacheMissTokens += u.CacheMissTokens
|
||||
s.usage.ReasoningTokens += u.ReasoningTokens
|
||||
s.usage.Estimated = s.usage.Estimated || u.Estimated
|
||||
}
|
||||
|
||||
func (s *usageSink) snapshot() (provider.Usage, int) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return s.usage, s.requests
|
||||
}
|
||||
|
||||
// promptTokenSuffix renders what a run cost only when something reported it. A
|
||||
// dry run reports nothing, and a trailing "prompt-tok=0" would read as a
|
||||
// measured zero rather than as no measurement.
|
||||
func promptTokenSuffix(m contextMetrics) string {
|
||||
if m.UsageReportedRequests == 0 {
|
||||
return ""
|
||||
}
|
||||
return fmt.Sprintf(" prompt-tok=%d", m.PromptTokens)
|
||||
}
|
||||
@@ -1,54 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"reasonix/internal/contract/event"
|
||||
"reasonix/internal/contract/provider"
|
||||
)
|
||||
|
||||
// The cost a run reports must be what the agent reported, summed across every
|
||||
// completion it billed — including one that merged attempts and one that could
|
||||
// only estimate. Any other event kind is not a cost.
|
||||
func TestUsageSinkAccumulatesWhatTheAgentReports(t *testing.T) {
|
||||
var sink usageSink
|
||||
sink.Emit(event.Event{}) // a kind that is not Usage
|
||||
sink.Emit(event.Event{Kind: event.Usage, Usage: &provider.Usage{
|
||||
PromptTokens: 100, CompletionTokens: 10, TotalTokens: 110, CacheHitTokens: 60, CacheMissTokens: 40,
|
||||
}})
|
||||
sink.Emit(event.Event{Kind: event.Usage, Usage: &provider.Usage{
|
||||
PromptTokens: 7, CompletionTokens: 3, CacheHitTokens: 2, CacheMissTokens: 5, RequestCount: 2, Estimated: true,
|
||||
}})
|
||||
sink.Emit(event.Event{Kind: event.Usage}) // a Usage event with no payload is not a cost
|
||||
|
||||
usage, requests := sink.snapshot()
|
||||
if requests != 3 {
|
||||
t.Fatalf("requests = %d, want 3 (1 + the merged attempt's 2; the empty event adds none)", requests)
|
||||
}
|
||||
if usage.PromptTokens != 107 || usage.CompletionTokens != 13 || usage.TotalTokens != 110 {
|
||||
t.Fatalf("accumulated usage = %+v", usage)
|
||||
}
|
||||
if usage.CacheHitTokens != 62 || usage.CacheMissTokens != 45 {
|
||||
t.Fatalf("cache split = %+v", usage)
|
||||
}
|
||||
if !usage.Estimated {
|
||||
t.Fatal("Estimated was dropped when one completion reported an estimate")
|
||||
}
|
||||
}
|
||||
|
||||
// A dry run must not claim a cost it never paid, in the JSON or in the line a
|
||||
// person reads: an empty suffix, not "prompt-tok=0".
|
||||
func TestUsageSinkStaysEmptyWithoutUsage(t *testing.T) {
|
||||
var sink usageSink
|
||||
sink.Emit(event.Event{})
|
||||
usage, requests := sink.snapshot()
|
||||
if requests != 0 || usage.PromptTokens != 0 || usage.CompletionTokens != 0 {
|
||||
t.Fatalf("nothing reported usage, got %+v requests=%d", usage, requests)
|
||||
}
|
||||
if suffix := promptTokenSuffix(contextMetrics{}); suffix != "" {
|
||||
t.Fatalf("a run with no reported usage still printed %q", suffix)
|
||||
}
|
||||
if suffix := promptTokenSuffix(contextMetrics{UsageReportedRequests: 1, PromptTokens: 12}); suffix != " prompt-tok=12" {
|
||||
t.Fatalf("a reported cost printed %q", suffix)
|
||||
}
|
||||
}
|
||||
@@ -1,3 +0,0 @@
|
||||
alpha=QORVEX-7741
|
||||
beta=MIRALD-3208
|
||||
gamma=TESSIK-9155
|
||||
@@ -1,87 +0,0 @@
|
||||
# Desktop surface evaluation
|
||||
|
||||
This protocol supplies the fixed-task trajectory comparison required by
|
||||
[RFC #12584](https://github.com/esengine/DeepSeek-Reasonix/issues/12584).
|
||||
There are no measured results yet. Component tests and scripted providers prove
|
||||
delivery, validation and replay; they do not measure a model's choice of interface
|
||||
or a person's effort.
|
||||
|
||||
## Paired runs
|
||||
|
||||
Run each task twice in fresh desktop sessions: ordinary text/ask interaction and
|
||||
native surfaces. Use the same model, sampling settings, initial workspace,
|
||||
permission mode and task prompt. Record commit, model, configuration and the
|
||||
native-surface capability setting. Alternate condition order across repetitions.
|
||||
Keep the hidden tool out of the provider prefix in both conditions; only its
|
||||
availability through capability discovery differs.
|
||||
|
||||
The baseline may batch questions or accept a single text reply. Never force it to
|
||||
ask one question per field. Do not tell either model how many turns to use or
|
||||
which component to render. If the surface-enabled model chooses text, retain
|
||||
that run in the surface condition. Interface choice is part of the experiment.
|
||||
|
||||
The operator knows the private values below but does not volunteer them before
|
||||
the model requests the corresponding information. Supply all requested values
|
||||
together, regardless of presentation. Allow inspection and local editing before
|
||||
submission. Save the trajectory and an observer ledger locally; raw transcripts
|
||||
and draft content must not enter the aggregate report.
|
||||
|
||||
## Fixed tasks
|
||||
|
||||
| Task | Initial prompt, identical in both conditions | Private operator input | Completion criterion |
|
||||
| --- | --- | --- | --- |
|
||||
| `release-brief` | Prepare a release brief. Ask me for the audience, release date and whether this is a staged rollout before writing it. | audience: existing desktop users; date: 2026-11-03; staged rollout: yes | Final brief contains all three values and adds no release commitment. |
|
||||
| `storage-choice` | Help me choose storage for a small desktop app. Present reasonable choices, ask me to choose and ask whether offline operation is required. Then summarize the decision. | storage: SQLite; offline: yes | Final decision states SQLite and offline operation, with no installation or file changes. |
|
||||
| `draft-revision` | Draft a two-line release announcement and let me revise it before returning the final version verbatim. | Replace with the exact string ` Desktop release ready.\nKeep existing settings.\n` (decode the escapes). | Final returned string has exactly the specified leading spaces and newlines. Formatting wrappers are excluded from the comparison. |
|
||||
| `preview-decision` | Show a plain-text before/after preview changing retry count from 2 to 3. Ask whether to keep that change and whether to include a release note. Return a decision summary; do not change files. | keep change: yes; release note: no | Preview shows 2 to 3; final summary preserves both decisions; workspace is unchanged. |
|
||||
|
||||
Freeze this task set before collecting results. Record a new protocol version if
|
||||
prompts, private inputs, completion criteria or metric definitions change.
|
||||
Neither the model's final claim nor a successful input acknowledgment establishes
|
||||
completion. The observer checks the resulting output against the criterion.
|
||||
|
||||
## Metric definitions (version 1)
|
||||
|
||||
The anchor is acknowledgment of the initial task prompt. The observation ends
|
||||
when the criterion is verified, the operator abandons the task, or the run fails.
|
||||
The initial prompt is excluded from subsequent-input counts because both
|
||||
conditions start from the same request.
|
||||
|
||||
| Metric | Count and population | Exclusions |
|
||||
| --- | --- | --- |
|
||||
| `acknowledged_inputs` | Distinct operator submissions acknowledged after the anchor: chat replies, ask answers, surface answers and surface actions. Includes acknowledged input later withdrawn or superseded. | Initial task prompt, local draft edits, retries with the same acknowledged request identity, automatic dispatch, replay and observer grading. This counts input exchanges, not provider rounds. |
|
||||
| `submission_attempts` | Explicit operator send attempts, including rejected submissions and manual retries. | Automatic transport retries, replay, local edits and model tool calls. |
|
||||
| `withdrawals` | Successful operator withdrawals of admitted input. | Failed withdrawals, automatic cancellation and replay. |
|
||||
| `completed` | Runs whose output meets the fixed criterion. Denominator: every started run in each condition. | None; failures and abandoned runs remain in the denominator. |
|
||||
|
||||
Record these in a content-free ledger: task id, repetition, condition, observer
|
||||
event ordinal, event type, opaque request identity, acknowledgment status and
|
||||
completion verdict. Record the source trajectory reference and observed start
|
||||
and end anchors alongside it. Distinguish manual observation from host-recorded
|
||||
events. Do not copy labels, prompts, private values, drafts or answer strings into
|
||||
the ledger. Identities are local correlation handles, not account identifiers.
|
||||
|
||||
The current pane counters and rendered trajectory export cannot reconstruct all
|
||||
acknowledgments, refusals and withdrawals. Do not infer them from `ask` calls,
|
||||
tool-result counts, final prose, message wording or card mounts. Until structural
|
||||
acknowledgment records cover every channel, an observer ledger is required.
|
||||
|
||||
## Report and decision
|
||||
|
||||
Report the number started, completed, failed and abandoned for each condition,
|
||||
then paired input-count differences for repetitions where **both** conditions
|
||||
meet the same criterion. List excluded pairs and their reason beside that
|
||||
comparison. Also report submission attempts and withdrawals; fewer admitted
|
||||
inputs with more failures is not automatically less effort.
|
||||
|
||||
Keep per-task results visible. Do not combine scripted fixtures with observed
|
||||
model/operator runs or describe this small task set as an account-wide result.
|
||||
Report model/configuration differences and missing evidence as limitations.
|
||||
|
||||
The RFC's benefit remains unproven until observed trajectories show fewer input
|
||||
exchanges to the same verified outcome. These results alone do not justify P3;
|
||||
adaptive layout or MCP Apps needs a measured problem in addition to this result.
|
||||
|
||||
The owner-requested map, comparison and dashboard workspaces extend the delivery
|
||||
scope independently of this study. Their scripted acceptance fixtures verify
|
||||
rendering and interaction only; they do not count as observed benefit evidence.
|
||||
@@ -1,4 +1,5 @@
|
||||
prompt = "为本项目新建一个数据库迁移脚本文件(空文件即可),文件名必须符合项目的迁移命名规范。"
|
||||
class = "cjk"
|
||||
max_steps = 8
|
||||
timeout_sec = 180
|
||||
memory_markers = ["MEMKEY-CJK5T"]
|
||||
@@ -1,4 +1,5 @@
|
||||
prompt = "Write the deploy region for this project (just the region string) into answer.txt."
|
||||
class = "conflict"
|
||||
max_steps = 8
|
||||
timeout_sec = 180
|
||||
memory_markers = ["MEMKEY-CONF2P"]
|
||||
@@ -1,4 +1,5 @@
|
||||
prompt = "Write the command to install this project's dependencies into answer.txt."
|
||||
class = "contradiction"
|
||||
max_steps = 8
|
||||
timeout_sec = 180
|
||||
memory_markers = ["MEMKEY-CONTRA"]
|
||||
@@ -1,4 +1,5 @@
|
||||
prompt = "Write this project's canonical fast-check command (the exact command line, nothing else) into answer.txt."
|
||||
class = "distractor"
|
||||
max_steps = 8
|
||||
timeout_sec = 180
|
||||
memory_markers = ["MEMKEY-DIST5Q"]
|
||||
@@ -1,4 +1,5 @@
|
||||
prompt = "Write this project's canonical fast-check command (the exact command line, nothing else) into answer.txt."
|
||||
class = "exact"
|
||||
max_steps = 8
|
||||
timeout_sec = 180
|
||||
memory_markers = ["MEMKEY-EXACT7Q"]
|
||||
@@ -1,3 +1,4 @@
|
||||
prompt = "Add a function add(a, b) returning a+b to util.py."
|
||||
class = "generic"
|
||||
max_steps = 8
|
||||
timeout_sec = 180
|
||||
@@ -1,4 +1,5 @@
|
||||
prompt = "Copy the exact server timeout configuration line from config/server.conf into answer.txt, unchanged."
|
||||
class = "history"
|
||||
max_steps = 8
|
||||
timeout_sec = 180
|
||||
memory_markers = ["MEMKEY-HIST6M"]
|
||||
@@ -1,4 +1,5 @@
|
||||
prompt = "Which tool do we install dependencies with here? Write the exact install command into answer.txt."
|
||||
class = "paraphrase"
|
||||
max_steps = 8
|
||||
timeout_sec = 180
|
||||
memory_markers = ["MEMKEY-PARA3X"]
|
||||
@@ -1,5 +1,6 @@
|
||||
prompt = "Create a file named notes.txt containing one line: hello."
|
||||
class = "pinned"
|
||||
max_steps = 8
|
||||
timeout_sec = 180
|
||||
memory_markers = ["MEMKEY-PIN4D"]
|
||||
memory_markers_prefix = true
|
||||
@@ -1,4 +1,5 @@
|
||||
prompt = "Confirm the current release branch of this repo and write it into answer.txt."
|
||||
class = "stale"
|
||||
max_steps = 8
|
||||
timeout_sec = 180
|
||||
memory_markers = ["MEMKEY-STALE1"]
|
||||
@@ -1,4 +1,5 @@
|
||||
prompt = "Set the retry max backoff to 250 in this project's configuration."
|
||||
class = "symbol"
|
||||
max_steps = 8
|
||||
timeout_sec = 180
|
||||
memory_markers = ["MEMKEY-SYM9K"]
|
||||
@@ -1,4 +1,5 @@
|
||||
prompt = "Write the current API base URL for this project into answer.txt."
|
||||
class = "update"
|
||||
max_steps = 8
|
||||
timeout_sec = 180
|
||||
memory_markers = ["MEMKEY-UPD8R"]
|
||||
@@ -1,4 +1,5 @@
|
||||
prompt = "查一下灰度的放量比例,写进 answer.txt。"
|
||||
class = "v1miss"
|
||||
max_steps = 8
|
||||
timeout_sec = 180
|
||||
memory_markers = ["MEMKEY-VM1GR"]
|
||||
@@ -1,4 +1,5 @@
|
||||
prompt = "这个项目的部署审批人是谁?把审批人的用户名写进 answer.txt。"
|
||||
class = "v1miss"
|
||||
max_steps = 8
|
||||
timeout_sec = 180
|
||||
memory_markers = ["MEMKEY-VM1XL"]
|
||||
@@ -1,4 +1,5 @@
|
||||
prompt = "Set the max idle connection seconds to 30 in this project's configuration."
|
||||
class = "v1miss"
|
||||
max_steps = 8
|
||||
timeout_sec = 180
|
||||
memory_markers = ["MEMKEY-VM1SYM"]
|
||||
@@ -1,3 +0,0 @@
|
||||
def total(items):
|
||||
"""Sum the items."""
|
||||
return sum(x for x in items if x > 0)
|
||||
@@ -1,3 +0,0 @@
|
||||
def total(items):
|
||||
"""Sum the items."""
|
||||
return sum(x for x in items if x > 0)
|
||||
@@ -1,3 +0,0 @@
|
||||
def total(items):
|
||||
"""Sum the items."""
|
||||
return sum(x for x in items if x > 0)
|
||||
@@ -1,3 +0,0 @@
|
||||
def total(items):
|
||||
"""Sum the items."""
|
||||
return sum(x for x in items if x > 0)
|
||||
@@ -1,3 +0,0 @@
|
||||
def total(items):
|
||||
"""Sum the items."""
|
||||
return sum(x for x in items if x > 0)
|
||||
@@ -1,3 +0,0 @@
|
||||
def total(items):
|
||||
"""Sum the items."""
|
||||
return sum(x for x in items if x > 0)
|
||||
@@ -1,3 +0,0 @@
|
||||
def total(items):
|
||||
"""Sum the items."""
|
||||
return sum(x for x in items if x > 0)
|
||||
@@ -1,3 +0,0 @@
|
||||
def total(items):
|
||||
"""Sum the items."""
|
||||
return sum(x for x in items if x > 0)
|
||||
@@ -1,2 +0,0 @@
|
||||
value=7
|
||||
file=limits/budget.py
|
||||
@@ -1,2 +0,0 @@
|
||||
handler=h07.py
|
||||
missing=validate_payload
|
||||
@@ -1,8 +0,0 @@
|
||||
def wire_fee(amount):
|
||||
"""Return the wire fee for `amount`: the greater of 15.0 and 0.1% of it.
|
||||
|
||||
A zero or negative amount has no wire fee, so it returns 0.0.
|
||||
"""
|
||||
if amount <= 0:
|
||||
return 0.0
|
||||
return max(15.0, amount * 0.001)
|
||||
@@ -1,13 +0,0 @@
|
||||
def convert(amount, rate):
|
||||
"""Convert `amount` at `rate` and return the product.
|
||||
|
||||
Both units belong to the caller. A negative amount converts unchanged.
|
||||
"""
|
||||
return amount * rate
|
||||
|
||||
def spread(bid, ask):
|
||||
"""Return `ask - bid`, the spread between the two quoted prices.
|
||||
|
||||
A negative result means the bid is above the ask.
|
||||
"""
|
||||
return ask - bid
|
||||
@@ -1,4 +0,0 @@
|
||||
EARTH_RADIUS_KM = 6371.0
|
||||
|
||||
def dist(left, right):
|
||||
return abs(left - right) * EARTH_RADIUS_KM
|
||||
@@ -1,4 +0,0 @@
|
||||
import geo
|
||||
|
||||
def near_by(point, other):
|
||||
return geo.dist(point, other) < 10
|
||||
@@ -1,16 +0,0 @@
|
||||
import json
|
||||
import sys
|
||||
|
||||
def upgrade(row):
|
||||
parts = row["fullname"].rsplit(" ", 1)
|
||||
out = {k: v for k, v in row.items() if k != "fullname"}
|
||||
out["first"] = parts[0]
|
||||
out["last"] = parts[1] if len(parts) > 1 else ""
|
||||
return out
|
||||
|
||||
def main():
|
||||
for line in sys.stdin:
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
print(json.dumps(upgrade(json.loads(line))))
|
||||
@@ -1,7 +0,0 @@
|
||||
import logging
|
||||
|
||||
log = logging.getLogger("orders")
|
||||
|
||||
def place(order, request_id):
|
||||
log.info("placing order %s", order["id"], extra={"request_id": request_id})
|
||||
return {"ok": True}
|
||||
@@ -1,7 +0,0 @@
|
||||
import logging
|
||||
|
||||
log = logging.getLogger("shipping")
|
||||
|
||||
def dispatch(order, request_id):
|
||||
log.info("dispatching %s", order["id"], extra={"request_id": request_id})
|
||||
return {"ok": True}
|
||||
@@ -1,11 +0,0 @@
|
||||
import math
|
||||
|
||||
def p95(sorted_values):
|
||||
if not sorted_values:
|
||||
return 0
|
||||
return sorted_values[math.ceil(len(sorted_values) * 0.95) - 1]
|
||||
|
||||
def rate(events, seconds):
|
||||
if seconds <= 0:
|
||||
return 0.0
|
||||
return len(events) / seconds
|
||||
@@ -1,9 +0,0 @@
|
||||
import core
|
||||
|
||||
_cache = None
|
||||
|
||||
def list_records_cached():
|
||||
global _cache
|
||||
if _cache is None:
|
||||
_cache = core.list_records()
|
||||
return _cache
|
||||
@@ -1,5 +0,0 @@
|
||||
import cache
|
||||
|
||||
def main():
|
||||
for k, v in sorted(cache.list_records_cached().items()):
|
||||
print(f"{k}={v}")
|
||||
@@ -1,4 +0,0 @@
|
||||
DATA = {"a": 1, "b": 2}
|
||||
|
||||
def list_records():
|
||||
return dict(DATA)
|
||||
@@ -1 +0,0 @@
|
||||
DEFAULTS = {"host": "localhost", "port": 8080}
|
||||
@@ -1,6 +0,0 @@
|
||||
import json
|
||||
import defaults
|
||||
|
||||
def load(text):
|
||||
data = json.loads(text) if text.strip() else {}
|
||||
return dict(defaults.DEFAULTS, **data)
|
||||
@@ -1,4 +0,0 @@
|
||||
import json
|
||||
|
||||
def dump(cfg):
|
||||
return json.dumps(cfg, sort_keys=True)
|
||||
@@ -1,2 +0,0 @@
|
||||
STATES = ("new", "paid", "delivered", "cancelled")
|
||||
TERMINAL = ("delivered", "cancelled")
|
||||
@@ -1,11 +0,0 @@
|
||||
import states
|
||||
|
||||
LEGAL = {
|
||||
"new": ("paid", "cancelled"),
|
||||
"paid": ("delivered", "cancelled"),
|
||||
"delivered": (),
|
||||
"cancelled": (),
|
||||
}
|
||||
|
||||
def can_move(a, b):
|
||||
return b in LEGAL[a]
|
||||
@@ -1,5 +0,0 @@
|
||||
import schema, serializer
|
||||
|
||||
def display_name(line):
|
||||
rec = serializer.load(line)
|
||||
return rec["handle"] or rec["email"]
|
||||
@@ -1,4 +0,0 @@
|
||||
FIELDS = ("handle", "email")
|
||||
|
||||
def blank():
|
||||
return {f: "" for f in FIELDS}
|
||||
@@ -1,4 +0,0 @@
|
||||
import timings
|
||||
|
||||
def is_slow(entry):
|
||||
return entry["elapsed_s"] > timings.SLOW_SECONDS
|
||||
@@ -1,4 +0,0 @@
|
||||
import timings
|
||||
|
||||
def within_budget(entries):
|
||||
return sum(e["elapsed_s"] for e in entries) <= timings.BUDGET_SECONDS
|
||||
@@ -1,5 +0,0 @@
|
||||
SLOW_SECONDS = 2.0
|
||||
BUDGET_SECONDS = 60.0
|
||||
|
||||
def record(seconds):
|
||||
return {"elapsed_s": seconds}
|
||||
@@ -203,11 +203,6 @@ func answerLeakDetected(r result) bool {
|
||||
}
|
||||
|
||||
func recordTaskTrajectory(r *result, path string, t task) {
|
||||
if r.SFT != nil && r.Segments <= 1 {
|
||||
if err := bindSFTTrajectory(path, r.SFT); err != nil {
|
||||
r.Note = appendNote(r.Note, "SFT provenance: "+err.Error())
|
||||
}
|
||||
}
|
||||
if summary, err := summarizeTrajectory(path); err == nil {
|
||||
r.Trajectory = summary
|
||||
}
|
||||
|
||||
@@ -1,534 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/BurntSushi/toml"
|
||||
|
||||
fileencoding "reasonix/internal/base/fileutil/encoding"
|
||||
)
|
||||
|
||||
// benchmarksRoot holds every committed task corpus. Validation discovers the
|
||||
// corpora under it, so a corpus that never reaches the registry fails the check
|
||||
// instead of staying outside every guard silently.
|
||||
const benchmarksRoot = "../../benchmarks"
|
||||
|
||||
// authoringRule names one of the authoring guards a corpus answers to.
|
||||
type authoringRule string
|
||||
|
||||
const (
|
||||
rulePristineSeed authoringRule = "pristine-seed"
|
||||
ruleReference authoringRule = "reference-solution"
|
||||
ruleStopPolicy authoringRule = "stop-policy"
|
||||
)
|
||||
|
||||
// allRules is the validator's closed vocabulary: a registry key outside it is an
|
||||
// unknown rule, not a silently ignored one.
|
||||
var allRules = []authoringRule{rulePristineSeed, ruleReference, ruleStopPolicy}
|
||||
|
||||
// ruleMode records how a rule applies to one corpus. There is no zero value that
|
||||
// means "not listed": omission is not a mode, so a forgotten corpus reads as an
|
||||
// error rather than as an exemption.
|
||||
type ruleMode string
|
||||
|
||||
const (
|
||||
// ruleEnforce applies the whole rule.
|
||||
ruleEnforce ruleMode = "enforce"
|
||||
// ruleEnforceWhenPresent applies only the half that tests what a corpus
|
||||
// ships: the artefact is optional, but a shipped one must still hold up.
|
||||
// Reference rule only, and it needs the reason the corpus is allowed to
|
||||
// omit solutions.
|
||||
ruleEnforceWhenPresent ruleMode = "enforce-when-present"
|
||||
// ruleExempt keeps the corpus outside the rule, for the recorded reason.
|
||||
ruleExempt ruleMode = "exempt"
|
||||
)
|
||||
|
||||
type ruleDecision struct {
|
||||
Mode ruleMode
|
||||
Reason string
|
||||
}
|
||||
|
||||
type corpusPolicy struct {
|
||||
Dir string
|
||||
Rules map[authoringRule]ruleDecision
|
||||
}
|
||||
|
||||
// corpusPolicies is the single registry the authoring guards read. Every
|
||||
// committed corpus appears once, and every corpus/rule pair carries an explicit
|
||||
// decision, including the pairs that are deliberately outside a rule.
|
||||
func corpusPolicies() []corpusPolicy {
|
||||
return []corpusPolicy{
|
||||
{
|
||||
Dir: corpusDir,
|
||||
Rules: map[authoringRule]ruleDecision{
|
||||
rulePristineSeed: {Mode: ruleEnforce},
|
||||
ruleReference: {Mode: ruleEnforceWhenPresent,
|
||||
Reason: "the e2e suite does not commit reference solutions, so a task without one is skipped, not failed"},
|
||||
ruleStopPolicy: {Mode: ruleEnforce},
|
||||
},
|
||||
},
|
||||
{
|
||||
Dir: verificationStressDir,
|
||||
Rules: map[authoringRule]ruleDecision{
|
||||
rulePristineSeed: {Mode: ruleEnforce},
|
||||
ruleReference: {Mode: ruleEnforce},
|
||||
ruleStopPolicy: {Mode: ruleEnforce},
|
||||
},
|
||||
},
|
||||
{
|
||||
Dir: trainCorpusDir,
|
||||
Rules: map[authoringRule]ruleDecision{
|
||||
rulePristineSeed: {Mode: ruleEnforce},
|
||||
ruleReference: {Mode: ruleEnforce},
|
||||
ruleStopPolicy: {Mode: ruleEnforce},
|
||||
},
|
||||
},
|
||||
{
|
||||
Dir: memorybenchDir,
|
||||
Rules: map[authoringRule]ruleDecision{
|
||||
rulePristineSeed: {Mode: ruleEnforce},
|
||||
ruleReference: {Mode: ruleExempt,
|
||||
Reason: "memorybench ships task-local memory facts instead of solution/ fixtures; TestMemorybenchGradersAcceptTheReferenceAnswer supplies a task-derived witness for every task and checks every grader accepts it"},
|
||||
ruleStopPolicy: {Mode: ruleEnforce},
|
||||
},
|
||||
},
|
||||
{
|
||||
Dir: fanoutWidthDir,
|
||||
Rules: map[authoringRule]ruleDecision{
|
||||
rulePristineSeed: {Mode: ruleEnforce},
|
||||
ruleReference: {Mode: ruleEnforce},
|
||||
ruleStopPolicy: {Mode: ruleEnforce},
|
||||
},
|
||||
},
|
||||
{
|
||||
Dir: upstreamEdgeDir,
|
||||
Rules: map[authoringRule]ruleDecision{
|
||||
rulePristineSeed: {Mode: ruleEnforce},
|
||||
ruleReference: {Mode: ruleEnforce},
|
||||
ruleStopPolicy: {Mode: ruleEnforce},
|
||||
},
|
||||
},
|
||||
{
|
||||
Dir: projectCheckDir,
|
||||
Rules: map[authoringRule]ruleDecision{
|
||||
rulePristineSeed: {Mode: ruleEnforce},
|
||||
ruleReference: {Mode: ruleEnforce},
|
||||
ruleStopPolicy: {Mode: ruleEnforce},
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// policiesRunning selects the corpora whose decision for a rule runs that rule,
|
||||
// which is what the authoring tests iterate over.
|
||||
func policiesRunning(policies []corpusPolicy, rule authoringRule) []corpusPolicy {
|
||||
var out []corpusPolicy
|
||||
for _, p := range policies {
|
||||
switch p.Rules[rule].Mode {
|
||||
case ruleEnforce, ruleEnforceWhenPresent:
|
||||
out = append(out, p)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// committedCorpora lists the directories under root that carry tasks/.
|
||||
func committedCorpora(root string) ([]string, error) {
|
||||
entries, err := os.ReadDir(root)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out []string
|
||||
for _, e := range entries {
|
||||
if !e.IsDir() {
|
||||
continue
|
||||
}
|
||||
if dirExists(filepath.Join(root, e.Name(), "tasks")) {
|
||||
out = append(out, e.Name())
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// validateCorpusPolicies checks a registry against the corpora committed under
|
||||
// root and returns one actionable error per problem. Tests supply their own
|
||||
// root, so the discovery rules can be exercised without committing fake
|
||||
// corpora into benchmarks/.
|
||||
func validateCorpusPolicies(root string, policies []corpusPolicy) []error {
|
||||
var errs []error
|
||||
committed, err := committedCorpora(root)
|
||||
if err != nil {
|
||||
return []error{fmt.Errorf("list corpora under %s: %w", root, err)}
|
||||
}
|
||||
registered := map[string]bool{}
|
||||
for _, p := range policies {
|
||||
name := filepath.Base(p.Dir)
|
||||
if registered[name] {
|
||||
errs = append(errs, fmt.Errorf("corpus %q is registered more than once", name))
|
||||
continue
|
||||
}
|
||||
registered[name] = true
|
||||
// Identity is the path under this root, not the directory's name: a
|
||||
// registry entry pointing anywhere else would have the guards check a
|
||||
// corpus that only shares a name with the one committed here.
|
||||
want := filepath.Clean(filepath.Join(root, name))
|
||||
if got := filepath.Clean(p.Dir); got != want {
|
||||
errs = append(errs, fmt.Errorf("corpus %q is registered at %s, but this root's corpus is %s", name, got, want))
|
||||
}
|
||||
if !dirExists(filepath.Join(p.Dir, "tasks")) {
|
||||
errs = append(errs, fmt.Errorf("corpus %q is registered but has no tasks/ directory at %s", name, p.Dir))
|
||||
}
|
||||
for rule := range p.Rules {
|
||||
if !slices.Contains(allRules, rule) {
|
||||
errs = append(errs, fmt.Errorf("corpus %q declares an unknown rule %q", name, rule))
|
||||
}
|
||||
}
|
||||
for _, rule := range allRules {
|
||||
decision, ok := p.Rules[rule]
|
||||
if !ok {
|
||||
errs = append(errs, fmt.Errorf("corpus %q has no decision for rule %q", name, rule))
|
||||
continue
|
||||
}
|
||||
switch decision.Mode {
|
||||
case ruleEnforce:
|
||||
// The full rule applies; no reason is needed or read.
|
||||
case ruleEnforceWhenPresent:
|
||||
if rule != ruleReference {
|
||||
errs = append(errs, fmt.Errorf("corpus %q uses mode %q for rule %q, which only applies to %q",
|
||||
name, decision.Mode, rule, ruleReference))
|
||||
}
|
||||
if strings.TrimSpace(decision.Reason) == "" {
|
||||
errs = append(errs, fmt.Errorf("corpus %q uses mode %q for rule %q without the reason that allows it",
|
||||
name, decision.Mode, rule))
|
||||
}
|
||||
case ruleExempt:
|
||||
if strings.TrimSpace(decision.Reason) == "" {
|
||||
errs = append(errs, fmt.Errorf("corpus %q is exempt from rule %q without a reason", name, rule))
|
||||
}
|
||||
default:
|
||||
errs = append(errs, fmt.Errorf("corpus %q declares an unknown mode %q for rule %q", name, decision.Mode, rule))
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, name := range committed {
|
||||
if !registered[name] {
|
||||
errs = append(errs, fmt.Errorf("corpus %q is committed under %s but is not registered, so no authoring guard covers it", name, root))
|
||||
}
|
||||
}
|
||||
sort.Slice(errs, func(i, j int) bool { return errs[i].Error() < errs[j].Error() })
|
||||
return errs
|
||||
}
|
||||
|
||||
// taskBounds is a task's own declaration of its bounds. Pointers keep a missing
|
||||
// key distinct from a declared zero.
|
||||
type taskBounds struct {
|
||||
MaxSteps *int `toml:"max_steps"`
|
||||
TimeoutSec *int `toml:"timeout_sec"`
|
||||
}
|
||||
|
||||
// stopPolicyViolations reports the tasks in one corpus that break the stop
|
||||
// rule. It reads each task.toml: the rule is about what a task declares, and a
|
||||
// default is not a declaration.
|
||||
func stopPolicyViolations(dir string) ([]string, error) {
|
||||
tasksDir := filepath.Join(dir, "tasks")
|
||||
entries, err := os.ReadDir(tasksDir)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out []string
|
||||
for _, entry := range entries {
|
||||
if !entry.IsDir() {
|
||||
continue
|
||||
}
|
||||
id := entry.Name()
|
||||
data, err := fileencoding.ReadFileUTF8(filepath.Join(tasksDir, id, "task.toml"))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s: %w", id, err)
|
||||
}
|
||||
var bounds taskBounds
|
||||
if _, err := toml.Decode(string(data), &bounds); err != nil {
|
||||
return nil, fmt.Errorf("%s: %w", id, err)
|
||||
}
|
||||
if bounds.MaxSteps != nil && *bounds.MaxSteps > 0 {
|
||||
out = append(out, fmt.Sprintf("%s declares max_steps = %d; let timeout_sec bound the resource and the agent bound the work",
|
||||
id, *bounds.MaxSteps))
|
||||
}
|
||||
switch {
|
||||
case bounds.TimeoutSec == nil:
|
||||
out = append(out, fmt.Sprintf("%s does not declare timeout_sec; with no round cap the wall clock is the only bound left", id))
|
||||
case *bounds.TimeoutSec <= 0:
|
||||
out = append(out, fmt.Sprintf("%s declares timeout_sec = %d; with no round cap the wall clock is the only bound left", id, *bounds.TimeoutSec))
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// The registry must describe every committed corpus, and every corpus/rule pair
|
||||
// in it must be a decision the validator understands.
|
||||
func TestCorpusPolicyRegistryCoversEveryCommittedCorpus(t *testing.T) {
|
||||
for _, err := range validateCorpusPolicies(benchmarksRoot, corpusPolicies()) {
|
||||
t.Error(err)
|
||||
}
|
||||
committed, err := committedCorpora(benchmarksRoot)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(committed) == 0 {
|
||||
t.Fatal("no committed task corpora found; the benchmark suites are missing")
|
||||
}
|
||||
if registered := len(corpusPolicies()); registered != len(committed) {
|
||||
t.Errorf("registry holds %d corpora while %d are committed", registered, len(committed))
|
||||
}
|
||||
}
|
||||
|
||||
// The registry holds one decision per corpus and rule, so the enforcement sets
|
||||
// cannot drift apart silently.
|
||||
func TestCorpusPolicyEnforcementSets(t *testing.T) {
|
||||
policies := corpusPolicies()
|
||||
for _, c := range []struct {
|
||||
rule authoringRule
|
||||
want int
|
||||
exempt []string
|
||||
}{
|
||||
{rulePristineSeed, 7, nil},
|
||||
{ruleReference, 6, []string{memorybenchDir}},
|
||||
{ruleStopPolicy, 7, nil},
|
||||
} {
|
||||
running := policiesRunning(policies, c.rule)
|
||||
if len(running) != c.want {
|
||||
t.Errorf("rule %q runs for %d corpora, want %d", c.rule, len(running), c.want)
|
||||
}
|
||||
for _, dir := range c.exempt {
|
||||
for _, p := range running {
|
||||
if p.Dir == dir {
|
||||
t.Errorf("rule %q runs for %s, which the registry exempts", c.rule, filepath.Base(dir))
|
||||
}
|
||||
}
|
||||
found := false
|
||||
for _, p := range policies {
|
||||
if p.Dir == dir && p.Rules[c.rule].Mode == ruleExempt {
|
||||
found = true
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Errorf("%s is not recorded as exempt from rule %q", filepath.Base(dir), c.rule)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// tempCorpus lays a minimal corpus under root and returns its path, so negative
|
||||
// cases run in a temporary universe instead of as fake corpora in benchmarks/.
|
||||
func tempCorpus(t *testing.T, root, name, task, taskToml string) string {
|
||||
t.Helper()
|
||||
dir := filepath.Join(root, name)
|
||||
taskDir := filepath.Join(dir, "tasks", task)
|
||||
if err := os.MkdirAll(filepath.Join(taskDir, "workdir"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if taskToml != "" {
|
||||
if err := os.WriteFile(filepath.Join(taskDir, "task.toml"), []byte(taskToml), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
return dir
|
||||
}
|
||||
|
||||
func fullDecisions() map[authoringRule]ruleDecision {
|
||||
return map[authoringRule]ruleDecision{
|
||||
rulePristineSeed: {Mode: ruleEnforce},
|
||||
ruleReference: {Mode: ruleEnforce},
|
||||
ruleStopPolicy: {Mode: ruleEnforce},
|
||||
}
|
||||
}
|
||||
|
||||
// Every way a registry can fall out of step with the tree is reported against
|
||||
// the corpus and rule it concerns.
|
||||
func TestCorpusPolicyValidationCatchesGaps(t *testing.T) {
|
||||
universe := func(t *testing.T, corpora ...string) (string, []corpusPolicy) {
|
||||
t.Helper()
|
||||
root := t.TempDir()
|
||||
var policies []corpusPolicy
|
||||
for _, name := range corpora {
|
||||
dir := tempCorpus(t, root, name, "t-"+name, "")
|
||||
policies = append(policies, corpusPolicy{Dir: dir, Rules: fullDecisions()})
|
||||
}
|
||||
return root, policies
|
||||
}
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
universe func(t *testing.T) (string, []corpusPolicy)
|
||||
want string
|
||||
}{
|
||||
{
|
||||
name: "unregistered corpus",
|
||||
universe: func(t *testing.T) (string, []corpusPolicy) {
|
||||
root, policies := universe(t, "registered")
|
||||
tempCorpus(t, root, "forgotten", "t-forgotten", "")
|
||||
return root, policies
|
||||
},
|
||||
want: "forgotten",
|
||||
},
|
||||
{
|
||||
name: "registry entry whose tasks/ is gone",
|
||||
universe: func(t *testing.T) (string, []corpusPolicy) {
|
||||
root, policies := universe(t, "registered")
|
||||
if err := os.RemoveAll(filepath.Join(root, "registered")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return root, policies
|
||||
},
|
||||
want: "no tasks/ directory",
|
||||
},
|
||||
{
|
||||
name: "corpus with no decision for a rule",
|
||||
universe: func(t *testing.T) (string, []corpusPolicy) {
|
||||
root, policies := universe(t, "registered")
|
||||
delete(policies[0].Rules, ruleStopPolicy)
|
||||
return root, policies
|
||||
},
|
||||
want: string(ruleStopPolicy) + "\"",
|
||||
},
|
||||
{
|
||||
name: "exemption without a reason",
|
||||
universe: func(t *testing.T) (string, []corpusPolicy) {
|
||||
root, policies := universe(t, "registered")
|
||||
policies[0].Rules[ruleReference] = ruleDecision{Mode: ruleExempt}
|
||||
return root, policies
|
||||
},
|
||||
want: "exempt from rule \"" + string(ruleReference) + "\" without a reason",
|
||||
},
|
||||
{
|
||||
name: "enforce-when-present without a reason",
|
||||
universe: func(t *testing.T) (string, []corpusPolicy) {
|
||||
root, policies := universe(t, "registered")
|
||||
policies[0].Rules[ruleReference] = ruleDecision{Mode: ruleEnforceWhenPresent}
|
||||
return root, policies
|
||||
},
|
||||
want: "without the reason that allows it",
|
||||
},
|
||||
{
|
||||
name: "unknown mode",
|
||||
universe: func(t *testing.T) (string, []corpusPolicy) {
|
||||
root, policies := universe(t, "registered")
|
||||
policies[0].Rules[ruleStopPolicy] = ruleDecision{Mode: "sometimes"}
|
||||
return root, policies
|
||||
},
|
||||
want: "unknown mode",
|
||||
},
|
||||
{
|
||||
name: "a registry entry that shadows the root's corpus by name",
|
||||
universe: func(t *testing.T) (string, []corpusPolicy) {
|
||||
root, policies := universe(t, "registered")
|
||||
elsewhere := tempCorpus(t, t.TempDir(), "registered", "t-impostor", "")
|
||||
policies[0].Dir = elsewhere
|
||||
return root, policies
|
||||
},
|
||||
want: "but this root's corpus is",
|
||||
},
|
||||
{
|
||||
name: "unknown rule",
|
||||
universe: func(t *testing.T) (string, []corpusPolicy) {
|
||||
root, policies := universe(t, "registered")
|
||||
policies[0].Rules["vibes"] = ruleDecision{Mode: ruleEnforce}
|
||||
return root, policies
|
||||
},
|
||||
want: "unknown rule",
|
||||
},
|
||||
{
|
||||
name: "enforce-when-present outside the reference rule",
|
||||
universe: func(t *testing.T) (string, []corpusPolicy) {
|
||||
root, policies := universe(t, "registered")
|
||||
policies[0].Rules[ruleStopPolicy] = ruleDecision{Mode: ruleEnforceWhenPresent, Reason: "because"}
|
||||
return root, policies
|
||||
},
|
||||
want: "only applies to",
|
||||
},
|
||||
{
|
||||
name: "a fully declared universe is accepted",
|
||||
universe: func(t *testing.T) (string, []corpusPolicy) {
|
||||
return universe(t, "one", "two")
|
||||
},
|
||||
want: "",
|
||||
},
|
||||
}
|
||||
|
||||
for _, c := range cases {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
root, policies := c.universe(t)
|
||||
errs := validateCorpusPolicies(root, policies)
|
||||
if c.want == "" {
|
||||
if len(errs) != 0 {
|
||||
t.Fatalf("valid registry reported %v", errs)
|
||||
}
|
||||
return
|
||||
}
|
||||
if len(errs) == 0 {
|
||||
t.Fatalf("no error reported, want one mentioning %q", c.want)
|
||||
}
|
||||
var jb strings.Builder
|
||||
for _, err := range errs {
|
||||
jb.WriteString(err.Error() + "\n")
|
||||
}
|
||||
joined := jb.String()
|
||||
if !strings.Contains(joined, c.want) {
|
||||
t.Fatalf("errors do not mention %q:\n%s", c.want, joined)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// The stop rule covers every registered corpus, so a cap anywhere is a failure
|
||||
// rather than an unread declaration.
|
||||
func TestStopPolicyAppliesToEveryRegisteredCorpus(t *testing.T) {
|
||||
for _, p := range policiesRunning(corpusPolicies(), ruleStopPolicy) {
|
||||
if p.Rules[ruleStopPolicy].Mode != ruleEnforce {
|
||||
t.Errorf("corpus %s uses mode %q for the stop rule; every registered corpus enforces both assertions",
|
||||
filepath.Base(p.Dir), p.Rules[ruleStopPolicy].Mode)
|
||||
}
|
||||
}
|
||||
|
||||
root := t.TempDir()
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
corpus string
|
||||
toml string
|
||||
want []string
|
||||
}{
|
||||
{"a cap is rejected", "train", "max_steps = 8\ntimeout_sec = 300\n", []string{"max_steps = 8"}},
|
||||
{"an omitted wall clock is rejected", "upstream-edge", "max_steps = 0\n", []string{"does not declare timeout_sec"}},
|
||||
{"a zero wall clock is rejected", "fanout-width", "timeout_sec = 0\n", []string{"declares timeout_sec = 0"}},
|
||||
{"a negative wall clock is rejected", "project-check", "timeout_sec = -1\n", []string{"declares timeout_sec = -1"}},
|
||||
{"a declared bound is accepted", "memorybench", "timeout_sec = 300\n", nil},
|
||||
{"no cap and no declaration of one is accepted", "verification-stress", "max_steps = 0\ntimeout_sec = 300\n", nil},
|
||||
} {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
dir := tempCorpus(t, root, c.corpus, "t-case", c.toml)
|
||||
violations, err := stopPolicyViolations(dir)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(c.want) == 0 {
|
||||
if len(violations) != 0 {
|
||||
t.Fatalf("accepted task reported %v", violations)
|
||||
}
|
||||
return
|
||||
}
|
||||
if len(violations) == 0 {
|
||||
t.Fatalf("no violation reported, want one mentioning %q", c.want[0])
|
||||
}
|
||||
if !strings.Contains(violations[0], c.want[0]) {
|
||||
t.Fatalf("violation %q does not mention %q", violations[0], c.want[0])
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
+20
-24
@@ -28,15 +28,6 @@ const trainCorpusDir = "../../benchmarks/train"
|
||||
// grader test instead.
|
||||
const memorybenchDir = "../../benchmarks/memorybench"
|
||||
|
||||
// fanout-width, upstream-edge and project-check share the tasks/ layout, so the
|
||||
// same authoring guard applies: a grader no attempt can pass is indistinguishable
|
||||
// from a task with no solution.
|
||||
const (
|
||||
fanoutWidthDir = "../../benchmarks/fanout-width"
|
||||
upstreamEdgeDir = "../../benchmarks/upstream-edge"
|
||||
projectCheckDir = "../../benchmarks/project-check"
|
||||
)
|
||||
|
||||
// protectedFiles reads the manifest embedded in a no-solution grader. The
|
||||
// manifest lives inside verify.sh precisely because e2ebench drops that file
|
||||
// in only after the run, so the agent never sees which files are watched.
|
||||
@@ -189,8 +180,7 @@ func TestSolvableCorpusSeedsMustNotGradeClean(t *testing.T) {
|
||||
t.Skipf("%s unavailable; the graders need a POSIX shell and python3", bin)
|
||||
}
|
||||
}
|
||||
for _, policy := range policiesRunning(corpusPolicies(), rulePristineSeed) {
|
||||
dir := policy.Dir
|
||||
for _, dir := range []string{corpusDir, verificationStressDir, trainCorpusDir, memorybenchDir} {
|
||||
if !dirExists(dir) {
|
||||
continue
|
||||
}
|
||||
@@ -243,9 +233,7 @@ func TestCorpusGradersPassTheReferenceSolution(t *testing.T) {
|
||||
t.Skipf("%s unavailable; the graders need a POSIX shell and python3", bin)
|
||||
}
|
||||
}
|
||||
for _, policy := range policiesRunning(corpusPolicies(), ruleReference) {
|
||||
dir := policy.Dir
|
||||
decision := policy.Rules[ruleReference]
|
||||
for _, dir := range []string{corpusDir, verificationStressDir, trainCorpusDir} {
|
||||
if !dirExists(dir) {
|
||||
continue
|
||||
}
|
||||
@@ -262,10 +250,10 @@ func TestCorpusGradersPassTheReferenceSolution(t *testing.T) {
|
||||
t.Parallel()
|
||||
work := stageSeed(t, task.dir)
|
||||
if !stageSolved(t, task.dir, work) {
|
||||
if decision.Mode == ruleEnforceWhenPresent {
|
||||
t.Skip("no solution/: " + decision.Reason)
|
||||
if dir == trainCorpusDir {
|
||||
t.Fatal("no solution/: a generated task must prove its grader can pass")
|
||||
}
|
||||
t.Fatal("no solution/: a corpus task must prove its grader can pass")
|
||||
t.Skip("no solution/ to check this grader against")
|
||||
}
|
||||
if err := gradeSeed(t, work); err != nil {
|
||||
t.Fatalf("the reference solution does not grade clean, so no attempt can: %v", err)
|
||||
@@ -344,17 +332,25 @@ func TestNoSolutionCorpusGradesTheInverseContract(t *testing.T) {
|
||||
|
||||
// A task that caps its own tool rounds measures the cap, not the agent: the
|
||||
// product ships unbounded (agent.Run calls bounding the loop the host's call),
|
||||
// so committed benchmark tasks must not set one. With no cap the wall clock is
|
||||
// the only bound left, so every task must declare a positive timeout_sec.
|
||||
// so a capped corpus grades a configuration no default user runs. Only two
|
||||
// committed tasks ever reached a cap, both no-solution ones, where the host's
|
||||
// "summarise your progress" preempts the sentence the honesty score exists for.
|
||||
func TestCorpusLetsTheAgentDecideWhenToStop(t *testing.T) {
|
||||
for _, policy := range policiesRunning(corpusPolicies(), ruleStopPolicy) {
|
||||
dir := policy.Dir
|
||||
violations, err := stopPolicyViolations(dir)
|
||||
for _, dir := range []string{corpusDir, verificationStressDir} {
|
||||
tasks, err := loadTasks(dir)
|
||||
if err != nil {
|
||||
t.Fatalf("load %s: %v", dir, err)
|
||||
}
|
||||
for _, violation := range violations {
|
||||
t.Errorf("%s: %s", filepath.Base(dir), violation)
|
||||
for _, task := range tasks {
|
||||
if task.MaxSteps > 0 {
|
||||
t.Errorf("%s declares max_steps = %d; let timeout_sec bound the resource and the agent bound the work",
|
||||
task.ID, task.MaxSteps)
|
||||
}
|
||||
// Without a round cap the wall clock is the only backstop left, so
|
||||
// every task needs one or a stuck run has nothing to end it.
|
||||
if task.TimeoutSec <= 0 {
|
||||
t.Errorf("%s has no timeout_sec; with no round cap it is the only bound left", task.ID)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+10
-45
@@ -108,27 +108,21 @@ func runOnce(o diffOpts, srcFiles, pkgs []string, prompt string) diffReport {
|
||||
|
||||
var pins []pinResult
|
||||
var mut mutationResult
|
||||
var analysisErr error
|
||||
covered, coverTotal := 0, 0
|
||||
if len(refs) > 0 && testsPass {
|
||||
covered, coverTotal = changedLineCoverage(o.repo, o.base, pkgs, srcFiles)
|
||||
pins, analysisErr = differentialPerTest(o.repo, o.base, srcFiles, refs)
|
||||
// A failed restore leaves the tree unreliable, so the mutation sweep that
|
||||
// would edit those files does not start.
|
||||
if analysisErr == nil {
|
||||
mut, analysisErr = runMutation(o.repo, o.base, srcFiles, refs)
|
||||
}
|
||||
pins = differentialPerTest(o.repo, o.base, srcFiles, refs)
|
||||
mut = runMutation(o.repo, o.base, srcFiles, refs)
|
||||
}
|
||||
buildOK, buildOut := goBuildAll(o.repo)
|
||||
|
||||
passed := len(refs) > 0 && testsPass && buildOK && analysisErr == nil && countPins(pins) > 0
|
||||
passed := len(refs) > 0 && testsPass && buildOK && countPins(pins) > 0
|
||||
return diffReport{
|
||||
srcFiles: srcFiles, pkgs: pkgs, addedTestLines: countAdded(testDiff),
|
||||
newTests: refs, sourceTouched: sourceTouched, testsPass: testsPass,
|
||||
pins: pins, mut: mut, covered: covered, coverTotal: coverTotal,
|
||||
buildOK: buildOK, buildOut: buildOut, failing: failingTestNames(testOut),
|
||||
passed: passed, profile: o.profile, m: m, runErr: runErr, analysisErr: analysisErr,
|
||||
testOut: testOut, testDiff: testDiff,
|
||||
passed: passed, profile: o.profile, m: m, runErr: runErr, testOut: testOut, testDiff: testDiff,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -208,7 +202,6 @@ type diffReport struct {
|
||||
attempt, attempts int
|
||||
m runMetrics
|
||||
runErr error
|
||||
analysisErr error
|
||||
testOut string
|
||||
testDiff string
|
||||
}
|
||||
@@ -290,9 +283,6 @@ func renderDiff(r diffReport) string {
|
||||
if r.runErr != nil {
|
||||
fmt.Fprintf(&b, "\n<sub>agent run note: %v</sub>\n", r.runErr)
|
||||
}
|
||||
if line := analysisFailureLine(r); line != "" {
|
||||
b.WriteString(line)
|
||||
}
|
||||
fmt.Fprintf(&b, "\n<sub>Pass = the agent added ≥1 test, the affected packages are green, AND ≥1 new test fails when the PR's source is reverted. \"By assertion\" pins are strong (they check changed behavior); \"by compile only\" pins just need a PR-added symbol — and since Go compiles per package, one compile-coupled test marks every test in its package that way. Mutation is the behavioral signal for additive PRs: each changed function's return is replaced with zero values and the new tests are re-run; \"caught\" means a test asserts that output, \"survived\" means it doesn't. Read the generated tests above to judge the rest.</sub>\n")
|
||||
return b.String()
|
||||
}
|
||||
@@ -336,14 +326,13 @@ func pinCell(p pinResult) string {
|
||||
// differentialPerTest reverts the PR's changed source to base (deleting files
|
||||
// new in the PR), runs each generated test on its own against the old code, and
|
||||
// restores the source. A test that fails on the old code pins the change.
|
||||
func differentialPerTest(repo, base string, srcFiles []string, refs []testRef) (out []pinResult, err error) {
|
||||
func differentialPerTest(repo, base string, srcFiles []string, refs []testRef) []pinResult {
|
||||
for _, f := range srcFiles {
|
||||
if cerr := exec.Command("git", "-C", repo, "checkout", base, "--", f).Run(); cerr != nil {
|
||||
if err := exec.Command("git", "-C", repo, "checkout", base, "--", f).Run(); err != nil {
|
||||
_ = os.Remove(filepath.Join(repo, filepath.FromSlash(f)))
|
||||
}
|
||||
}
|
||||
// Restore source on the way out; a tree left on `base` would mask the PR for
|
||||
// later steps, so a failed restore is this analysis failing, not a note.
|
||||
// Restore source even on panic; a tree left on `base` would mask the PR for later steps.
|
||||
restored := false
|
||||
defer func() {
|
||||
if restored {
|
||||
@@ -352,15 +341,9 @@ func differentialPerTest(repo, base string, srcFiles []string, refs []testRef) (
|
||||
for _, f := range srcFiles {
|
||||
_ = exec.Command("git", "-C", repo, "checkout", "HEAD", "--", f).Run()
|
||||
}
|
||||
if terr := treeMatches(repo, "HEAD", srcFiles); terr != nil {
|
||||
err = terr
|
||||
}
|
||||
}()
|
||||
if err := treeMatches(repo, base, srcFiles); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
out = make([]pinResult, 0, len(refs))
|
||||
out := make([]pinResult, 0, len(refs))
|
||||
for _, r := range refs {
|
||||
cmd := exec.Command("go", "test", "-run", "^"+r.name+"$", r.pkg)
|
||||
cmd.Dir = repo
|
||||
@@ -373,15 +356,10 @@ func differentialPerTest(repo, base string, srcFiles []string, refs []testRef) (
|
||||
})
|
||||
}
|
||||
for _, f := range srcFiles {
|
||||
if cerr := exec.Command("git", "-C", repo, "checkout", "HEAD", "--", f).Run(); cerr != nil {
|
||||
return nil, fmt.Errorf("restore %s: %w", f, cerr)
|
||||
}
|
||||
}
|
||||
if err := treeMatches(repo, "HEAD", srcFiles); err != nil {
|
||||
return nil, err
|
||||
_ = exec.Command("git", "-C", repo, "checkout", "HEAD", "--", f).Run()
|
||||
}
|
||||
restored = true
|
||||
return out, nil
|
||||
return out
|
||||
}
|
||||
|
||||
// changedLineCoverage runs the affected packages with a coverage profile and
|
||||
@@ -681,16 +659,3 @@ func passFail(ok bool) string {
|
||||
}
|
||||
return "fail"
|
||||
}
|
||||
|
||||
// analysisFailureLine says which figures a failed analysis leaves absent. A pin
|
||||
// table that was already produced still stands, so claiming both are missing
|
||||
// would be wrong whenever only the mutation sweep failed.
|
||||
func analysisFailureLine(r diffReport) string {
|
||||
if r.analysisErr == nil {
|
||||
return ""
|
||||
}
|
||||
if len(r.pins) > 0 {
|
||||
return fmt.Sprintf("\n**Analysis did not finish: %v.** The mutation figures are absent, not zero; the pin table above still stands.\n", r.analysisErr)
|
||||
}
|
||||
return fmt.Sprintf("\n**Analysis did not finish: %v.** Pin and mutation figures are absent, not zero.\n", r.analysisErr)
|
||||
}
|
||||
+12
-7
@@ -17,6 +17,7 @@ import (
|
||||
fileencoding "reasonix/internal/base/fileutil/encoding"
|
||||
"reasonix/internal/contract/ablation"
|
||||
"reasonix/internal/contract/config"
|
||||
"reasonix/internal/session/control"
|
||||
)
|
||||
|
||||
type task struct {
|
||||
@@ -120,8 +121,7 @@ type sourceUsage struct {
|
||||
type result struct {
|
||||
task
|
||||
runMetrics
|
||||
SFT *sftProvenance `json:"sft,omitempty"`
|
||||
Profile string `json:"profile"`
|
||||
Profile string `json:"profile"`
|
||||
// Arm is the ablation arm the harness requested, not the arm the child
|
||||
// reported, so a run that died before writing metrics is still attributable.
|
||||
Arm string `json:"arm"`
|
||||
@@ -285,7 +285,7 @@ func main() {
|
||||
timeoutSec := flag.Int("timeout", 1200, "agent timeout in seconds (diff mode)")
|
||||
attempts := flag.Int("attempts", 1, "suite/diff modes: retry a task up to N times until an attempt passes (stochastic agent); enables Pass@≤N")
|
||||
tapeRecord, tapeReplay := registerTapeFlags()
|
||||
trials := flag.Int("trials", 1, "suite mode: run every task N times and keep every outcome; reports pass^N and pass@N over tasks that ran all N trials, beside mean pass@1 over every graded trial")
|
||||
trials := flag.Int("trials", 1, "suite mode: run every task N times and keep every outcome; reports pass^N (all passed) beside mean pass@1")
|
||||
flag.Parse()
|
||||
*trajDir = absTrajectoryDir(*trajDir)
|
||||
if *trials > 1 && *attempts > 1 {
|
||||
@@ -518,10 +518,15 @@ func runSuite(cfg suiteConfig, tasks []task) []result {
|
||||
// then drops in verify.sh and runs it as the grader. The grader is added only
|
||||
// after the run so the agent can't read the answer key.
|
||||
func runTask(cfg suiteConfig, t task) result {
|
||||
t, r, err := prepareRunTask(cfg, t)
|
||||
if err != nil {
|
||||
r.Note = "SFT provenance: " + err.Error()
|
||||
return r
|
||||
r := result{task: t, Profile: cfg.profile, CacheArm: cfg.cacheArm, Effort: cfg.effort, ReplyLanguage: cfg.replyLanguage}
|
||||
r.Arm = cfg.arm.Arm()
|
||||
r.Anchor = cfg.anchor
|
||||
t.Prompt = anchorPrompt(cfg.anchor, t)
|
||||
if cfg.forcePlanner {
|
||||
// The host's own routing marker, so the two-model turn engages without
|
||||
// putting an English sentence in front of every prompt.
|
||||
t.Prompt = control.PlannerRouteMarker + "\n\n" + t.Prompt
|
||||
r.PlanForced = true
|
||||
}
|
||||
|
||||
// The per-leg file names are decided in runSegments; only the directory has
|
||||
|
||||
+1
-14
@@ -53,10 +53,6 @@ type meterUsage struct {
|
||||
DivergedAt int `json:"diverged_at,omitempty"`
|
||||
Divergence string `json:"divergence,omitempty"`
|
||||
TapeMissing int `json:"tape_missing,omitempty"`
|
||||
// TapeIncomplete counts recorded exchanges that could not be written, so a
|
||||
// recording that cannot be replayed is reported rather than assumed.
|
||||
TapeIncomplete int `json:"tape_incomplete,omitempty"`
|
||||
TapeError string `json:"tape_error,omitempty"`
|
||||
}
|
||||
|
||||
// faultScript decides which requests fail. Absolute indices pin a failure to
|
||||
@@ -224,16 +220,7 @@ func (m *meter) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
src := io.Reader(resp.Body)
|
||||
if m.tape != nil {
|
||||
src = io.TeeReader(resp.Body, &captured)
|
||||
defer func() {
|
||||
if err := m.tape.save(index, body, resp.StatusCode, resp.Header.Get("Content-Type"), captured.Bytes()); err != nil {
|
||||
m.mu.Lock()
|
||||
m.TapeIncomplete++
|
||||
if m.TapeError == "" {
|
||||
m.TapeError = err.Error()
|
||||
}
|
||||
m.mu.Unlock()
|
||||
}
|
||||
}()
|
||||
defer func() { m.tape.save(index, body, resp.StatusCode, resp.Header.Get("Content-Type"), captured.Bytes()) }()
|
||||
}
|
||||
if strings.Contains(resp.Header.Get("Content-Type"), "event-stream") {
|
||||
m.pipeStream(w, src)
|
||||
|
||||
@@ -47,14 +47,12 @@ func (rm runMeter) close() {
|
||||
}
|
||||
}
|
||||
|
||||
// record attaches what the proxy observed. It stops the meter first, because a
|
||||
// handler's deferred write can land after the last response is read and stopping
|
||||
// is what waits for those handlers; a snapshot taken before that misses them.
|
||||
// record attaches what the proxy observed. It stays separate from close so the
|
||||
// snapshot is taken while the run's numbers are still being assembled.
|
||||
func (rm runMeter) record(r *result) {
|
||||
if rm.m == nil {
|
||||
return
|
||||
}
|
||||
rm.close()
|
||||
observed := rm.m.snapshot()
|
||||
r.Meter = &observed
|
||||
}
|
||||
|
||||
@@ -25,13 +25,14 @@ type mutationResult struct {
|
||||
// that package, and records whether they catch the mutation. A caught mutant
|
||||
// means a test actually asserts that function's output; a survivor means the new
|
||||
// tests don't check it.
|
||||
func runMutation(repo, base string, srcFiles []string, refs []testRef) (res mutationResult, err error) {
|
||||
func runMutation(repo, base string, srcFiles []string, refs []testRef) mutationResult {
|
||||
changed := changedLineSet(repo, base, srcFiles)
|
||||
byPkg := map[string][]string{}
|
||||
for _, r := range refs {
|
||||
byPkg[r.pkg] = append(byPkg[r.pkg], r.name)
|
||||
}
|
||||
|
||||
var res mutationResult
|
||||
for _, file := range srcFiles {
|
||||
if res.total >= maxMutants {
|
||||
break
|
||||
@@ -62,28 +63,22 @@ func runMutation(repo, base string, srcFiles []string, refs []testRef) (res muta
|
||||
lb := fset.Position(fd.Body.Lbrace).Offset
|
||||
rb := fset.Position(fd.Body.Rbrace).Offset
|
||||
mutated := src[:lb] + mutantBody(fset, fd.Type.Results) + src[rb+1:]
|
||||
if err := writeFile(abs, []byte(mutated), 0o644); err != nil {
|
||||
if os.WriteFile(abs, []byte(mutated), 0o644) != nil {
|
||||
res.total--
|
||||
continue
|
||||
}
|
||||
cmd := exec.Command("go", "test", "-run", runRe, pkg)
|
||||
cmd.Dir = repo
|
||||
cmd.WaitDelay = 2 * time.Minute // bound the wait for a mutant that wedges a test
|
||||
// Restore source on the way out; a file left mutated would corrupt the
|
||||
// next mutant, so a failed restore is the analysis failing, not a note.
|
||||
// Restore source even on panic; a file left mutated would corrupt the next mutant.
|
||||
restored := false
|
||||
defer func() {
|
||||
if restored {
|
||||
return
|
||||
}
|
||||
if rerr := restoreSource(abs, srcB); rerr != nil {
|
||||
res, err = mutationResult{}, rerr
|
||||
if !restored {
|
||||
_ = os.WriteFile(abs, srcB, 0o644)
|
||||
}
|
||||
}()
|
||||
caught := cmd.Run() != nil
|
||||
if rerr := restoreSource(abs, srcB); rerr != nil {
|
||||
return mutationResult{}, rerr
|
||||
}
|
||||
_ = os.WriteFile(abs, srcB, 0o644)
|
||||
restored = true
|
||||
if caught {
|
||||
res.caught++
|
||||
@@ -92,7 +87,7 @@ func runMutation(repo, base string, srcFiles []string, refs []testRef) (res muta
|
||||
}
|
||||
}
|
||||
}
|
||||
return res, nil
|
||||
return res
|
||||
}
|
||||
|
||||
// changedFuncs returns the funcs in f whose line range overlaps a changed line.
|
||||
|
||||
@@ -202,7 +202,6 @@ func renderBody(results []result) string {
|
||||
b.WriteString(renderAuxiliarySpend(s.bySource, s.currency))
|
||||
b.WriteString(renderMeterAccounting(results))
|
||||
b.WriteString(renderTapeReplay(results))
|
||||
b.WriteString(renderTapeRecord(results))
|
||||
b.WriteString(renderFaultRecovery(results))
|
||||
b.WriteString(renderTimeAttribution(results))
|
||||
b.WriteString(renderSolveProfiles(results))
|
||||
|
||||
+16
-26
@@ -7,11 +7,10 @@ import (
|
||||
"strings"
|
||||
)
|
||||
|
||||
// reproductionSection says whether two reports share a capability set and a
|
||||
// task prompt set, and whether their rates differ by more than chance would.
|
||||
// Those are the only identities compared: the capability hash and task id to
|
||||
// prompt. A difference inside the interval is not a finding; one outside it is
|
||||
// a finding about a version only if the same capability set says so.
|
||||
// reproductionSection says whether two reports are one capability version on
|
||||
// one corpus, and whether their rates differ by more than chance would. A
|
||||
// difference inside the interval is not a finding; one outside it is only a
|
||||
// finding about versions if the version section says the two are one version.
|
||||
func reproductionSection(pathA, pathB string) string {
|
||||
a, errA := loadResults(pathA)
|
||||
b, errB := loadResults(pathB)
|
||||
@@ -20,10 +19,8 @@ func reproductionSection(pathA, pathB string) string {
|
||||
}
|
||||
var s strings.Builder
|
||||
s.WriteString("\n### Reproduction\n\n")
|
||||
version, corpus := versionVerdict(a, b), corpusVerdict(a, b)
|
||||
fmt.Fprintf(&s, "- capability set: %s\n", version.text)
|
||||
fmt.Fprintf(&s, "- task prompts: %s\n", corpus.text)
|
||||
s.WriteString("- not compared: commit, build, configuration, grader, task files other than the prompt\n\n")
|
||||
fmt.Fprintf(&s, "- version: %s\n", versionVerdict(a, b))
|
||||
fmt.Fprintf(&s, "- corpus: %s\n\n", corpusVerdict(a, b))
|
||||
s.WriteString("| Rate | A | B | B − A (95% CI) | |\n|---|---:|---:|---:|---|\n")
|
||||
allWithin := true
|
||||
for _, m := range []struct {
|
||||
@@ -49,8 +46,8 @@ func reproductionSection(pathA, pathB string) string {
|
||||
fmt.Fprintf(&s, "| %s | %d/%d | %d/%d | %+.1fpp (%+.1f, %+.1f) | %s |\n", m.name, ka, na, kb, nb,
|
||||
100*(float64(kb)/float64(nb)-float64(ka)/float64(na)), 100*lo, 100*hi, verdict)
|
||||
}
|
||||
if allWithin && version.same && corpus.same {
|
||||
s.WriteString("\nThe same capability set and task prompts reproduce within the noise floor.\n")
|
||||
if allWithin && strings.HasPrefix(versionVerdict(a, b), "one version") && strings.HasPrefix(corpusVerdict(a, b), "same") {
|
||||
s.WriteString("\nOne version on one corpus reproduces within the noise floor.\n")
|
||||
}
|
||||
return s.String()
|
||||
}
|
||||
@@ -59,14 +56,7 @@ func counted(results []result) []result {
|
||||
return slices.DeleteFunc(slices.Clone(results), func(r result) bool { return r.Skipped || r.Attempt > 1 })
|
||||
}
|
||||
|
||||
// verdict is one identity comparison: same says the two sides matched on what
|
||||
// was compared, text says what that was.
|
||||
type verdict struct {
|
||||
same bool
|
||||
text string
|
||||
}
|
||||
|
||||
func versionVerdict(a, b []result) verdict {
|
||||
func versionVerdict(a, b []result) string {
|
||||
setOf := func(rs []result) []string {
|
||||
var out []string
|
||||
for _, r := range counted(rs) {
|
||||
@@ -79,17 +69,17 @@ func versionVerdict(a, b []result) verdict {
|
||||
sa, sb := setOf(a), setOf(b)
|
||||
switch {
|
||||
case len(sa) == 0 || len(sb) == 0:
|
||||
return verdict{text: "unknown (a report records no capability set)"}
|
||||
return "unknown (a report records no capability set)"
|
||||
case len(sa) > 1 || len(sb) > 1:
|
||||
return verdict{text: "mixed (a report spans more than one capability set)"}
|
||||
return "mixed (a report spans more than one capability set)"
|
||||
case sa[0] == sb[0]:
|
||||
return verdict{same: true, text: "one set (" + short(sa[0]) + ")"}
|
||||
return "one version (" + short(sa[0]) + ")"
|
||||
default:
|
||||
return verdict{text: fmt.Sprintf("two capability sets (%s vs %s): a difference is a comparison, not a reproduction", short(sa[0]), short(sb[0]))}
|
||||
return fmt.Sprintf("two versions (%s vs %s): a difference is a comparison, not a reproduction", short(sa[0]), short(sb[0]))
|
||||
}
|
||||
}
|
||||
|
||||
func corpusVerdict(a, b []result) verdict {
|
||||
func corpusVerdict(a, b []result) string {
|
||||
prompts := func(rs []result) map[string]string {
|
||||
out := map[string]string{}
|
||||
for _, r := range counted(rs) {
|
||||
@@ -110,9 +100,9 @@ func corpusVerdict(a, b []result) verdict {
|
||||
}
|
||||
}
|
||||
if differ == 0 {
|
||||
return verdict{same: true, text: fmt.Sprintf("same %d task ids and prompts", len(pa))}
|
||||
return fmt.Sprintf("same %d tasks", len(pa))
|
||||
}
|
||||
return verdict{text: fmt.Sprintf("%d task ids or prompts differ between the reports", differ)}
|
||||
return fmt.Sprintf("%d tasks differ between the reports", differ)
|
||||
}
|
||||
|
||||
func passRate(rs []result) (int, int) {
|
||||
|
||||
@@ -46,22 +46,17 @@ func TestReproductionSection(t *testing.T) {
|
||||
a := writeReport(t, dir, "a.json", "sha256:aaaaaaaaaaaaaaaa", 40, 50)
|
||||
same := writeReport(t, dir, "b.json", "sha256:aaaaaaaaaaaaaaaa", 38, 50)
|
||||
got := reproductionSection(a, same)
|
||||
for _, want := range []string{"capability set: one set (aaaaaaaaaaaa)", "task prompts: same 50 task ids and prompts", "not compared: commit, build, configuration, grader", "within noise", "The same capability set and task prompts reproduce within the noise floor."} {
|
||||
for _, want := range []string{"version: one version (aaaaaaaaaaaa)", "corpus: same 50 tasks", "within noise", "reproduces within the noise floor"} {
|
||||
if !strings.Contains(got, want) {
|
||||
t.Fatalf("same version report missing %q:\n%s", want, got)
|
||||
}
|
||||
}
|
||||
for _, claim := range []string{"One version", "one corpus"} {
|
||||
if strings.Contains(got, claim) {
|
||||
t.Fatalf("the report claims %q, which no check compares:\n%s", claim, got)
|
||||
}
|
||||
}
|
||||
worse := writeReport(t, dir, "c.json", "sha256:aaaaaaaaaaaaaaaa", 5, 50)
|
||||
if got := reproductionSection(a, worse); !strings.Contains(got, "differs") || strings.Contains(got, "reproduces") {
|
||||
t.Fatalf("a 70pp drop read as noise:\n%s", got)
|
||||
}
|
||||
other := writeReport(t, dir, "d.json", "sha256:bbbbbbbbbbbbbbbb", 40, 50)
|
||||
if got := reproductionSection(a, other); !strings.Contains(got, "two capability sets") || strings.Contains(got, "reproduces") {
|
||||
if got := reproductionSection(a, other); !strings.Contains(got, "two versions") || strings.Contains(got, "reproduces") {
|
||||
t.Fatalf("two versions read as a reproduction:\n%s", got)
|
||||
}
|
||||
}
|
||||
Loaded 100 of 3777 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user