mirror of
https://github.com/Twigpine/zero.git
synced 2026-11-08 17:56:14 -10:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
c5fe4e951c | ||
|
|
0310f8bcf2 | ||
|
|
af875df587 | ||
|
|
dbd9443014 | ||
|
|
0fae754460 | ||
|
|
2cc6f43bf7 |
No files matched your search
@@ -1,5 +1,11 @@
|
||||
blank_issues_enabled: false
|
||||
contact_links:
|
||||
- name: Questions and setup help
|
||||
url: https://github.com/Gitlawb/zero/discussions/categories/q-a
|
||||
about: Ask in Discussions Q&A. Include zero --version, your OS, provider/model, and zero doctor output for setup problems.
|
||||
- name: Feature ideas
|
||||
url: https://github.com/Gitlawb/zero/discussions/categories/ideas
|
||||
about: Propose and discuss ideas in Discussions before opening a PR (Zero is in controlled development, see CONTRIBUTING.md).
|
||||
- name: Security vulnerability
|
||||
url: https://github.com/Gitlawb/zero/security/advisories/new
|
||||
about: Report security issues privately via GitHub Security Advisories (see SECURITY.md). Do not file a public issue.
|
||||
@@ -1,12 +1,22 @@
|
||||
name: Feature request
|
||||
description: Propose a new capability or change
|
||||
name: Feature request (previously discussed)
|
||||
description: File only after your idea gained traction in Discussions Ideas or a maintainer asked for a tracking issue
|
||||
labels: ["enhancement"]
|
||||
body:
|
||||
- type: markdown
|
||||
attributes:
|
||||
value: |
|
||||
Please do not begin implementation until the issue has been reviewed and marked with the
|
||||
New feature ideas start in [Discussions Ideas](https://github.com/Gitlawb/zero/discussions/categories/ideas),
|
||||
not here. This form is for the step after that: an idea that gained traction in Discussions,
|
||||
or one a maintainer asked to have tracked as an issue. Please link the discussion below.
|
||||
Do not begin implementation until the issue has been reviewed and marked with the
|
||||
`issue-approved` label (see CONTRIBUTING.md).
|
||||
- type: input
|
||||
id: discussion
|
||||
attributes:
|
||||
label: Ideas discussion
|
||||
description: Link to the Discussions Ideas thread (or note which maintainer asked for this issue).
|
||||
validations:
|
||||
required: true
|
||||
- type: textarea
|
||||
id: problem
|
||||
attributes:
|
||||
|
||||
@@ -55,3 +55,6 @@ docs/superpowers/
|
||||
zero-termux-arm64
|
||||
zero-linux-sandbox
|
||||
zero-seccomp
|
||||
|
||||
# Generated benchmark reports are evidence for one configuration, never tree state
|
||||
internal/perfbench/reports/*.json
|
||||
+26
-5
@@ -8,6 +8,13 @@ stabilizes. Community participation is still welcome, especially through bug
|
||||
reports, feature requests, reproduction details, logs, and discussion on existing
|
||||
issues.
|
||||
|
||||
Questions and setup help belong in
|
||||
[GitHub Discussions](https://github.com/Gitlawb/zero/discussions) (see
|
||||
[SUPPORT.md](SUPPORT.md)), not in the issue tracker. Feature ideas start in the
|
||||
[Ideas](https://github.com/Gitlawb/zero/discussions/categories/ideas) category;
|
||||
an idea that gains traction there is the path to an approved issue and,
|
||||
eventually, an accepted PR.
|
||||
|
||||
Please read this policy before opening a pull request. Pull requests that do not
|
||||
follow it may be closed without review.
|
||||
|
||||
@@ -33,7 +40,11 @@ unless the issue has been explicitly approved for outside contribution.
|
||||
There are three primary ways to contribute:
|
||||
|
||||
1. Open a bug report with clear reproduction steps.
|
||||
2. Open a feature request describing the problem, use case, and expected behavior.
|
||||
2. Propose a feature in
|
||||
[Discussions Ideas](https://github.com/Gitlawb/zero/discussions/categories/ideas),
|
||||
describing the problem, use case, and expected behavior. A feature-request
|
||||
issue is filed only after the idea gains traction there or a maintainer asks
|
||||
for one.
|
||||
3. Add useful context to an existing issue, such as logs, screenshots, examples,
|
||||
affected versions, or reproduction cases.
|
||||
|
||||
@@ -46,7 +57,8 @@ All community pull requests require an approved parent issue.
|
||||
|
||||
Before starting implementation work:
|
||||
|
||||
1. Open an issue using the appropriate issue template.
|
||||
1. For a bug, open an issue using the bug-report template. For a feature, start
|
||||
in Discussions Ideas; the feature-request issue comes after the discussion.
|
||||
2. Describe the bug, feature request, or functional change clearly.
|
||||
3. Wait for the core team to review the issue.
|
||||
4. Do not open a pull request unless the issue has the `issue-approved` label.
|
||||
@@ -133,7 +145,12 @@ Clear bug reports are the most useful way to help the project during this phase.
|
||||
|
||||
## Submitting a Feature Request
|
||||
|
||||
When filing a feature request, please include:
|
||||
Feature requests start as a thread in
|
||||
[Discussions Ideas](https://github.com/Gitlawb/zero/discussions/categories/ideas),
|
||||
not as an issue. The feature-request issue form exists for the step AFTER that:
|
||||
when an idea has gained traction in Discussions or a maintainer has asked for an
|
||||
issue to track it. Whether in the discussion or the eventual issue, please
|
||||
include:
|
||||
|
||||
- The problem you are trying to solve.
|
||||
- The behavior or functionality you are requesting.
|
||||
@@ -156,5 +173,9 @@ contribution phase.
|
||||
|
||||
## Getting Help
|
||||
|
||||
If you are unsure whether a change is appropriate, open an issue first and ask.
|
||||
Please keep discussion focused, specific, and respectful of maintainer time.
|
||||
General questions, setup help, and "is this a bug or my config" uncertainty all
|
||||
go to
|
||||
[Discussions Q&A](https://github.com/Gitlawb/zero/discussions/categories/q-a).
|
||||
If you are unsure whether a proposed code change is appropriate, raise it in
|
||||
Discussions Ideas or on the relevant issue first and ask. Please keep discussion
|
||||
focused, specific, and respectful of maintainer time.
|
||||
@@ -402,6 +402,24 @@ go run ./cmd/zero-release build --goos windows --goarch amd64 --output dist/zero
|
||||
- [Performance](docs/PERFORMANCE.md)
|
||||
- [Agent evals](docs/AGENT_EVALS.md)
|
||||
|
||||
## Community
|
||||
|
||||
Questions, setup help, ideas, and sharing all live in
|
||||
[GitHub Discussions](https://github.com/Gitlawb/zero/discussions):
|
||||
|
||||
| Category | Use it for |
|
||||
|---|---|
|
||||
| [Q&A](https://github.com/Gitlawb/zero/discussions/categories/q-a) | Setup help, provider/model configuration, "how do I" questions |
|
||||
| [Ideas](https://github.com/Gitlawb/zero/discussions/categories/ideas) | Feature proposals and design discussion before any PR |
|
||||
| [Show and tell](https://github.com/Gitlawb/zero/discussions/categories/show-and-tell) | Your skills, plugins, MCP setups, themes, and workflows |
|
||||
| [Announcements](https://github.com/Gitlawb/zero/discussions/categories/announcements) | Releases and project news from the maintainers |
|
||||
|
||||
For a good Q&A answer fast, include `zero --version`, your OS and install
|
||||
method, the provider/model in use, and `zero doctor` output. See
|
||||
[SUPPORT.md](SUPPORT.md). Bugs belong in
|
||||
[issues](https://github.com/Gitlawb/zero/issues/new/choose); security reports
|
||||
follow [SECURITY.md](SECURITY.md), never a public thread.
|
||||
|
||||
## Contributing
|
||||
|
||||
Contributions are welcome. Read [CONTRIBUTING.md](CONTRIBUTING.md), run the
|
||||
|
||||
+36
@@ -0,0 +1,36 @@
|
||||
# Getting help with Zero
|
||||
|
||||
## Questions, setup help, configuration
|
||||
|
||||
Ask in [GitHub Discussions](https://github.com/Gitlawb/zero/discussions). Use
|
||||
[Q&A](https://github.com/Gitlawb/zero/discussions/categories/q-a) so answers can
|
||||
be marked and found by the next person.
|
||||
|
||||
The fastest way to a useful answer is to include:
|
||||
|
||||
- `zero --version` output
|
||||
- your OS (Windows, macOS, Linux) and how you installed Zero (npm, install
|
||||
script, source)
|
||||
- the provider and model you are using (`zero providers list`)
|
||||
- `zero doctor` output for setup problems
|
||||
- the exact command you ran and the exact error text
|
||||
|
||||
## Bugs
|
||||
|
||||
If Zero does something wrong (crashes, wrong output, broken behavior), file a
|
||||
[bug report](https://github.com/Gitlawb/zero/issues/new/choose). If you are not
|
||||
sure whether it is a bug or a setup problem, start in Discussions and we will
|
||||
promote it to an issue if it is one.
|
||||
|
||||
## Feature ideas
|
||||
|
||||
Start a thread in
|
||||
[Ideas](https://github.com/Gitlawb/zero/discussions/categories/ideas). Zero is
|
||||
in controlled development (see CONTRIBUTING.md), so discussing an idea first is
|
||||
also the path to getting a PR accepted.
|
||||
|
||||
## Security issues
|
||||
|
||||
Never file a public issue or discussion. Report privately via
|
||||
[GitHub Security Advisories](https://github.com/Gitlawb/zero/security/advisories/new)
|
||||
(see SECURITY.md).
|
||||
@@ -4,6 +4,8 @@ import (
|
||||
"bytes"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/Gitlawb/zero/internal/perfbench"
|
||||
)
|
||||
|
||||
func TestParseArgsUsesCliAndEnvOverrides(t *testing.T) {
|
||||
@@ -92,3 +94,18 @@ func TestHelpText(t *testing.T) {
|
||||
func emptyEnv(string) string {
|
||||
return ""
|
||||
}
|
||||
|
||||
func TestTurnExitCodeFailsWhenEveryTaskErrored(t *testing.T) {
|
||||
var stderr bytes.Buffer
|
||||
allErrored := perfbench.TurnBenchResult{TasksAttempted: 3, TasksErrored: 3}
|
||||
if code := turnExitCode(allErrored, &stderr); code != 1 {
|
||||
t.Fatalf("exit=%d, want 1 when every task errored", code)
|
||||
}
|
||||
if !strings.Contains(stderr.String(), "no accepted benchmark sample") {
|
||||
t.Fatalf("stderr missing explanation: %q", stderr.String())
|
||||
}
|
||||
partial := perfbench.TurnBenchResult{TasksAttempted: 3, TasksErrored: 2}
|
||||
if code := turnExitCode(partial, &stderr); code != 0 {
|
||||
t.Fatalf("exit=%d, want 0 for partial errors (summary surfaces them)", code)
|
||||
}
|
||||
}
|
||||
@@ -86,9 +86,23 @@ func runTurnCommand(args []string, getenv func(string) string, stdout io.Writer,
|
||||
_, _ = fmt.Fprintln(stderr, "[zero] Turn benchmark failed: "+err.Error())
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
return turnExitCode(result, stderr)
|
||||
}
|
||||
_, _ = fmt.Fprintln(stdout, perfbench.FormatTurnBenchSummary(result))
|
||||
return turnExitCode(result, stderr)
|
||||
}
|
||||
|
||||
// turnExitCode fails the command when every attempted task errored (no
|
||||
// iteration produced an accepted benchmark sample): such a report measures
|
||||
// nothing, and exiting 0 would let a broken configuration (missing binary, bad
|
||||
// path, failing harness step) pass as a clean baseline. Partial errors keep
|
||||
// exit 0 — the summary surfaces them loudly and the surviving samples are
|
||||
// still valid measurements.
|
||||
func turnExitCode(result perfbench.TurnBenchResult, stderr io.Writer) int {
|
||||
if result.TasksAttempted > 0 && result.TasksErrored == result.TasksAttempted {
|
||||
_, _ = fmt.Fprintln(stderr, "[zero] Turn benchmark failed: every task errored with no accepted benchmark sample (see warnings); the report contains no valid measurements")
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
|
||||
+55
-1
@@ -185,6 +185,30 @@ func Run(ctx context.Context, prompt string, provider Provider, options Options)
|
||||
|
||||
guards := newGuardState()
|
||||
compactor := newCompactionState(options)
|
||||
// The execution-profile posture controller. nil Options.Profile (every
|
||||
// existing caller) makes every observe/decide call a no-op, keeping the
|
||||
// loop byte-identical for unprofiled runs.
|
||||
posture := newProfileController(options.Profile)
|
||||
// applyPosture applies at most one posture escalation when an armed trigger
|
||||
// has fired: raise the hoisted turn ceiling, restore effort and the
|
||||
// completion gate, stamp the counter. Shared by the end-of-turn tail and
|
||||
// the completion-gate uncertain path (which continues the loop before the
|
||||
// tail runs). No-op forever after the first escalation and for unprofiled
|
||||
// runs.
|
||||
applyPosture := func() {
|
||||
if target, fired := posture.maybeEscalate(); fired {
|
||||
if target.MaxTurns > maxTurns {
|
||||
maxTurns = target.MaxTurns
|
||||
}
|
||||
if target.ReasoningEffort != "" {
|
||||
options.ReasoningEffort = target.ReasoningEffort
|
||||
}
|
||||
if target.RestoreCompletionGate {
|
||||
options.RequireCompletionSignal = true
|
||||
}
|
||||
options.Trace.Counter(trace.CounterPostureEscalations, 1)
|
||||
}
|
||||
}
|
||||
|
||||
// Background post-edit diagnostics: files changed by mutating tools are
|
||||
// checked off the tool-call critical path and any errors are appended as a
|
||||
@@ -554,6 +578,11 @@ func Run(ctx context.Context, prompt string, provider Provider, options Options)
|
||||
result.Messages = copyMessages(messages)
|
||||
return result, nil
|
||||
case CompletionUncertain:
|
||||
// Observe the uncertainty signal and apply any escalation NOW:
|
||||
// this branch continues the turn loop before the end-of-turn
|
||||
// act point, so deferring would skip escalation entirely.
|
||||
posture.observeUncertain()
|
||||
applyPosture()
|
||||
switch evaluation.Action {
|
||||
case completionActionContinue:
|
||||
options.Trace.Counter(trace.CounterCompletionNudges, 1)
|
||||
@@ -692,6 +721,7 @@ func Run(ctx context.Context, prompt string, provider Provider, options Options)
|
||||
// would misdirect the model toward JSON shape or blocked behavior.
|
||||
retriableFailure := isRetriableToolError(toolResult)
|
||||
outcome := guards.observeToolResult(call.Name, retriableFailure, toolResult.Output)
|
||||
posture.observeToolOutcome(outcome, toolResult)
|
||||
if outcome.Stop {
|
||||
// The assistant message advertised EVERY collected tool call, but
|
||||
// the guard halts mid-turn so the calls after this one never run.
|
||||
@@ -724,7 +754,9 @@ func Run(ctx context.Context, prompt string, provider Provider, options Options)
|
||||
// the assistant's tool_results stay contiguous (a user message between
|
||||
// tool_results breaks strict provider replay). nil SelfCorrect is a no-op.
|
||||
if options.SelfCorrect != nil && len(changedFilesThisBatch) > 0 {
|
||||
if feedback, _ := options.SelfCorrect.AfterEdit(ctx, dedupeStrings(changedFilesThisBatch)); feedback != "" {
|
||||
feedback, selfCorrectOutcome := options.SelfCorrect.AfterEdit(ctx, dedupeStrings(changedFilesThisBatch))
|
||||
posture.observeSelfCorrect(selfCorrectOutcome)
|
||||
if feedback != "" {
|
||||
messages = append(messages, zeroruntime.Message{
|
||||
Role: zeroruntime.MessageRoleUser,
|
||||
Content: feedback,
|
||||
@@ -817,6 +849,13 @@ func Run(ctx context.Context, prompt string, provider Provider, options Options)
|
||||
Content: reminder,
|
||||
})
|
||||
}
|
||||
|
||||
// One-shot posture escalation: when an armed profile trigger fired this
|
||||
// turn, restore the stricter knob values the profile displaced. Mutates
|
||||
// only per-turn-read policy (the hoisted turn ceiling, reasoning effort,
|
||||
// completion gate); never messages, model, or session. No-op forever
|
||||
// after the first escalation and for every unprofiled run.
|
||||
applyPosture()
|
||||
}
|
||||
|
||||
if ctx.Err() != nil {
|
||||
@@ -1338,10 +1377,25 @@ func executeToolCall(ctx context.Context, registry *tools.Registry, call ToolCal
|
||||
result = registry.RebudgetAfterHook(call.Name, args, result)
|
||||
}
|
||||
}
|
||||
// Stamp the sandbox risk classification for this EXECUTED call so run-policy
|
||||
// observers can see the risk of an allowed mutation. Prefer the preflight
|
||||
// decision's classification when the sandbox evaluated the call; otherwise
|
||||
// run the same pure classifier the permission path uses. Denied/canceled
|
||||
// results returned earlier keep the zero value.
|
||||
executedRisk := sandbox.Risk{}
|
||||
if preflightDecision != nil {
|
||||
executedRisk = preflightDecision.Risk
|
||||
} else if toolFound {
|
||||
// Unknown-tool results fall through here with a nil tool; they carry a
|
||||
// denial and keep the zero risk value.
|
||||
executedRisk = sandbox.Classify(sandboxRequest(call.Name, tool, args, permissionGranted, permissionMode, options))
|
||||
}
|
||||
|
||||
// Secret scrubbing happens at the registry boundary (the single point both
|
||||
// the agent loop and the MCP server pass through), so result.Output is
|
||||
// already redacted here and result.Redacted reflects whether it changed.
|
||||
return ToolResult{
|
||||
Risk: executedRisk,
|
||||
ToolCallID: call.ID,
|
||||
Name: call.Name,
|
||||
Status: result.Status,
|
||||
|
||||
@@ -0,0 +1,118 @@
|
||||
package agent
|
||||
|
||||
import (
|
||||
"github.com/Gitlawb/zero/internal/sandbox"
|
||||
)
|
||||
|
||||
// profileController observes per-turn run signals and decides at most one
|
||||
// posture escalation per run for the armed execution profile. A nil policy (or
|
||||
// a policy without Escalate) makes every method a no-op, so runs without a
|
||||
// profile are byte-identical to before — the same opt-in convention as Trace
|
||||
// and SelfCorrect.
|
||||
//
|
||||
// The controller only ever CHANGES loop policy knobs (turn ceiling, reasoning
|
||||
// effort, completion gate) through the values the loop applies at its act
|
||||
// point; it never injects messages, never touches the model or session, and
|
||||
// never interacts with the model-initiated escalate_model path.
|
||||
type profileController struct {
|
||||
policy *PostureEscalation
|
||||
|
||||
escalated bool
|
||||
uncertainSeen int
|
||||
|
||||
failureTripped bool
|
||||
riskTripped bool
|
||||
scTripped bool
|
||||
uncertainTrip bool
|
||||
}
|
||||
|
||||
// newProfileController is nil-safe: a nil ProfilePolicy (the default for every
|
||||
// existing caller) or a nil Escalate yields a controller that observes and
|
||||
// decides nothing.
|
||||
func newProfileController(policy *ProfilePolicy) *profileController {
|
||||
if policy == nil {
|
||||
return &profileController{}
|
||||
}
|
||||
return &profileController{policy: policy.Escalate}
|
||||
}
|
||||
|
||||
// observeToolOutcome watches two signals from an executed tool call: the
|
||||
// repeated-failure guard's streak (reusing the guard's own counting, not a
|
||||
// second failure model) and the sandbox risk classification of an executed
|
||||
// result.
|
||||
func (c *profileController) observeToolOutcome(outcome toolFailureOutcome, result ToolResult) {
|
||||
if c.policy == nil || c.escalated {
|
||||
return
|
||||
}
|
||||
if c.policy.OnToolFailureStreak > 0 && outcome.Count >= c.policy.OnToolFailureStreak {
|
||||
c.failureTripped = true
|
||||
}
|
||||
if threshold := riskRank(c.policy.OnRiskyMutation); threshold > 0 &&
|
||||
result.DenialReason == DenialNone &&
|
||||
riskRank(result.Risk.Level) >= threshold {
|
||||
// The call EXECUTED (was not denied): partial failures count too, since
|
||||
// the mutation ran. An unrecognized threshold ranks 0 and disables the
|
||||
// signal instead of matching everything.
|
||||
c.riskTripped = true
|
||||
}
|
||||
}
|
||||
|
||||
// observeUncertain counts uncertain completion evaluations (continue nudges and
|
||||
// semantic checks). Headless-only by construction: the completion gate never
|
||||
// runs interactively.
|
||||
func (c *profileController) observeUncertain() {
|
||||
if c.policy == nil || c.escalated || c.policy.OnCompletionUncertain <= 0 {
|
||||
return
|
||||
}
|
||||
c.uncertainSeen++
|
||||
if c.uncertainSeen >= c.policy.OnCompletionUncertain {
|
||||
c.uncertainTrip = true
|
||||
}
|
||||
}
|
||||
|
||||
// observeSelfCorrect watches the post-edit verification outcome. Any failing
|
||||
// outcome (correcting, reported, aborted) is a signal; passed and disabled are
|
||||
// not.
|
||||
func (c *profileController) observeSelfCorrect(outcome Outcome) {
|
||||
if c.policy == nil || c.escalated || !c.policy.OnSelfCorrectFailure {
|
||||
return
|
||||
}
|
||||
switch outcome {
|
||||
case OutcomeCorrecting, OutcomeReported, OutcomeAborted:
|
||||
c.scTripped = true
|
||||
}
|
||||
}
|
||||
|
||||
// maybeEscalate reports the escalation target exactly once, the first time any
|
||||
// armed trigger has fired. Subsequent calls return false for the rest of the
|
||||
// run: escalation is one-shot and never de-escalates, so no cooldown state is
|
||||
// needed.
|
||||
func (c *profileController) maybeEscalate() (PostureEscalation, bool) {
|
||||
if c.policy == nil || c.escalated {
|
||||
return PostureEscalation{}, false
|
||||
}
|
||||
if !(c.failureTripped || c.riskTripped || c.scTripped || c.uncertainTrip) {
|
||||
return PostureEscalation{}, false
|
||||
}
|
||||
c.escalated = true
|
||||
return *c.policy, true
|
||||
}
|
||||
|
||||
// riskRank orders sandbox risk levels for threshold comparison. Unknown levels
|
||||
// rank 0: an unrecognized RESULT level can never meet a valid threshold, and an
|
||||
// unrecognized THRESHOLD is rejected before comparison (rank 0 disables the
|
||||
// signal) so a typo in a profile can never make every result match.
|
||||
func riskRank(level sandbox.RiskLevel) int {
|
||||
switch level {
|
||||
case sandbox.RiskLow:
|
||||
return 1
|
||||
case sandbox.RiskMedium:
|
||||
return 2
|
||||
case sandbox.RiskHigh:
|
||||
return 3
|
||||
case sandbox.RiskCritical:
|
||||
return 4
|
||||
default:
|
||||
return 0
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,300 @@
|
||||
package agent
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/Gitlawb/zero/internal/sandbox"
|
||||
"github.com/Gitlawb/zero/internal/tools"
|
||||
"github.com/Gitlawb/zero/internal/trace"
|
||||
"github.com/Gitlawb/zero/internal/zeroruntime"
|
||||
)
|
||||
|
||||
func TestProfileControllerNilPolicyIsNoOp(t *testing.T) {
|
||||
for _, controller := range []*profileController{
|
||||
newProfileController(nil),
|
||||
newProfileController(&ProfilePolicy{Name: "balanced"}), // Escalate nil
|
||||
} {
|
||||
controller.observeToolOutcome(toolFailureOutcome{Count: 99}, ToolResult{Status: tools.StatusOK, Risk: sandbox.Risk{Level: sandbox.RiskCritical}})
|
||||
controller.observeUncertain()
|
||||
controller.observeSelfCorrect(OutcomeAborted)
|
||||
if _, fired := controller.maybeEscalate(); fired {
|
||||
t.Fatal("nil policy must never escalate")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestProfileControllerEscalatesOnFailureStreak(t *testing.T) {
|
||||
controller := newProfileController(&ProfilePolicy{Escalate: &PostureEscalation{OnToolFailureStreak: 2, MaxTurns: 80}})
|
||||
controller.observeToolOutcome(toolFailureOutcome{Count: 1}, ToolResult{Status: tools.StatusError})
|
||||
if _, fired := controller.maybeEscalate(); fired {
|
||||
t.Fatal("streak below threshold must not escalate")
|
||||
}
|
||||
controller.observeToolOutcome(toolFailureOutcome{Count: 2}, ToolResult{Status: tools.StatusError})
|
||||
target, fired := controller.maybeEscalate()
|
||||
if !fired || target.MaxTurns != 80 {
|
||||
t.Fatalf("expected escalation with the policy target, got fired=%t target=%+v", fired, target)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProfileControllerEscalatesOnRiskyMutation(t *testing.T) {
|
||||
controller := newProfileController(&ProfilePolicy{Escalate: &PostureEscalation{OnRiskyMutation: sandbox.RiskHigh}})
|
||||
// Medium risk on an executed result: below threshold.
|
||||
controller.observeToolOutcome(toolFailureOutcome{}, ToolResult{Status: tools.StatusOK, Risk: sandbox.Risk{Level: sandbox.RiskMedium}})
|
||||
if _, fired := controller.maybeEscalate(); fired {
|
||||
t.Fatal("medium risk must not trip a high threshold")
|
||||
}
|
||||
// Critical risk on a DENIED result: never counts (the mutation never ran).
|
||||
controller.observeToolOutcome(toolFailureOutcome{}, ToolResult{Status: tools.StatusError, DenialReason: DenialSandboxBlock, Risk: sandbox.Risk{Level: sandbox.RiskCritical}})
|
||||
if _, fired := controller.maybeEscalate(); fired {
|
||||
t.Fatal("a denied result must not trip the risky-mutation signal")
|
||||
}
|
||||
// Critical risk on an executed PARTIAL FAILURE (error status, no denial):
|
||||
// the mutation ran, so it counts.
|
||||
controller.observeToolOutcome(toolFailureOutcome{}, ToolResult{Status: tools.StatusError, Risk: sandbox.Risk{Level: sandbox.RiskCritical}})
|
||||
if _, fired := controller.maybeEscalate(); !fired {
|
||||
t.Fatal("an executed partial failure at critical risk must trip a high threshold")
|
||||
}
|
||||
}
|
||||
|
||||
func TestProfileControllerIgnoresInvalidRiskThreshold(t *testing.T) {
|
||||
controller := newProfileController(&ProfilePolicy{Escalate: &PostureEscalation{OnRiskyMutation: sandbox.RiskLevel("hgih")}})
|
||||
controller.observeToolOutcome(toolFailureOutcome{}, ToolResult{Status: tools.StatusOK, Risk: sandbox.Risk{Level: sandbox.RiskCritical}})
|
||||
if _, fired := controller.maybeEscalate(); fired {
|
||||
t.Fatal("an unrecognized threshold must disable the signal, not match every result")
|
||||
}
|
||||
}
|
||||
|
||||
func TestProfileControllerEscalatesOnSelfCorrectFailure(t *testing.T) {
|
||||
controller := newProfileController(&ProfilePolicy{Escalate: &PostureEscalation{OnSelfCorrectFailure: true}})
|
||||
controller.observeSelfCorrect(OutcomePassed)
|
||||
controller.observeSelfCorrect(OutcomeDisabled)
|
||||
if _, fired := controller.maybeEscalate(); fired {
|
||||
t.Fatal("passing/disabled outcomes must not escalate")
|
||||
}
|
||||
controller.observeSelfCorrect(OutcomeReported)
|
||||
if _, fired := controller.maybeEscalate(); !fired {
|
||||
t.Fatal("a failing verification outcome must escalate")
|
||||
}
|
||||
}
|
||||
|
||||
func TestProfileControllerEscalatesOnUncertainCompletion(t *testing.T) {
|
||||
controller := newProfileController(&ProfilePolicy{Escalate: &PostureEscalation{OnCompletionUncertain: 2}})
|
||||
controller.observeUncertain()
|
||||
if _, fired := controller.maybeEscalate(); fired {
|
||||
t.Fatal("first uncertain evaluation must not trip a threshold of 2")
|
||||
}
|
||||
controller.observeUncertain()
|
||||
if _, fired := controller.maybeEscalate(); !fired {
|
||||
t.Fatal("second uncertain evaluation must escalate")
|
||||
}
|
||||
}
|
||||
|
||||
func TestProfileControllerEscalatesAtMostOnce(t *testing.T) {
|
||||
controller := newProfileController(&ProfilePolicy{Escalate: &PostureEscalation{OnToolFailureStreak: 1}})
|
||||
controller.observeToolOutcome(toolFailureOutcome{Count: 1}, ToolResult{Status: tools.StatusError})
|
||||
if _, fired := controller.maybeEscalate(); !fired {
|
||||
t.Fatal("expected first escalation")
|
||||
}
|
||||
controller.observeToolOutcome(toolFailureOutcome{Count: 5}, ToolResult{Status: tools.StatusOK, Risk: sandbox.Risk{Level: sandbox.RiskCritical}})
|
||||
controller.observeUncertain()
|
||||
if _, fired := controller.maybeEscalate(); fired {
|
||||
t.Fatal("escalation must be one-shot per run")
|
||||
}
|
||||
}
|
||||
|
||||
// TestPostureEscalationOnUncertainCompletion proves the uncertain-path act
|
||||
// point: the completion gate's continue branch escapes the end-of-turn tail,
|
||||
// so escalation must apply before the continue. A cue turn under the gate
|
||||
// counts as uncertain, escalates, and the very next request carries the target
|
||||
// effort.
|
||||
func TestPostureEscalationOnUncertainCompletion(t *testing.T) {
|
||||
provider := &mockProvider{turns: [][]zeroruntime.StreamEvent{
|
||||
{
|
||||
{Type: zeroruntime.StreamEventText, Content: "Let me read the file:"},
|
||||
{Type: zeroruntime.StreamEventDone},
|
||||
},
|
||||
{
|
||||
{Type: zeroruntime.StreamEventText, Content: "Done. All set."},
|
||||
{Type: zeroruntime.StreamEventDone},
|
||||
},
|
||||
}}
|
||||
|
||||
recorder := trace.NewRecorder("posture-session", "run-2", "fast")
|
||||
result, err := Run(context.Background(), "go", provider, Options{
|
||||
Registry: tools.NewRegistry(),
|
||||
MaxTurns: 10,
|
||||
RequireCompletionSignal: true,
|
||||
Trace: recorder,
|
||||
Profile: &ProfilePolicy{
|
||||
Name: "fast",
|
||||
Escalate: &PostureEscalation{
|
||||
ReasoningEffort: "high",
|
||||
OnCompletionUncertain: 1,
|
||||
},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if result.FinalAnswer != "Done. All set." {
|
||||
t.Fatalf("expected the completed answer, got %q", result.FinalAnswer)
|
||||
}
|
||||
if len(provider.requests) != 2 {
|
||||
t.Fatalf("expected 2 requests, got %d", len(provider.requests))
|
||||
}
|
||||
if provider.requests[1].ReasoningEffort != "high" {
|
||||
t.Fatalf("expected the post-escalation request to carry the target effort, got %q", provider.requests[1].ReasoningEffort)
|
||||
}
|
||||
tr := recorder.Finish()
|
||||
if got := tr.Counter(trace.CounterPostureEscalations); got != 1 {
|
||||
t.Fatalf("posture_escalations = %d, want 1", got)
|
||||
}
|
||||
}
|
||||
|
||||
// failingProfileTool always errors with the same signature so the repeated-
|
||||
// failure guard builds a streak.
|
||||
type failingProfileTool struct{}
|
||||
|
||||
func (failingProfileTool) Name() string { return "flaky_probe" }
|
||||
func (failingProfileTool) Description() string { return "always fails identically (test)" }
|
||||
func (failingProfileTool) Parameters() tools.Schema {
|
||||
return tools.Schema{Type: "object", AdditionalProperties: false}
|
||||
}
|
||||
func (failingProfileTool) Safety() tools.Safety {
|
||||
return tools.Safety{SideEffect: tools.SideEffectRead, Permission: tools.PermissionAllow}
|
||||
}
|
||||
func (failingProfileTool) Run(context.Context, map[string]any) tools.Result {
|
||||
return tools.Result{Status: tools.StatusError, Output: "probe exploded: same signature"}
|
||||
}
|
||||
|
||||
// TestPostureEscalationRaisesTurnCeilingMidRun proves the end-to-end act path:
|
||||
// a Fast-style run with a 2-turn ceiling hits a failure streak, escalates, and
|
||||
// finishes on a turn the original ceiling would have cut off, with the effort
|
||||
// target applied to the post-escalation request and the counter emitted.
|
||||
func TestPostureEscalationRaisesTurnCeilingMidRun(t *testing.T) {
|
||||
registry := tools.NewRegistry()
|
||||
registry.Register(failingProfileTool{})
|
||||
|
||||
toolTurn := []zeroruntime.StreamEvent{
|
||||
{Type: zeroruntime.StreamEventToolCallStart, ToolCallID: "c1", ToolName: "flaky_probe"},
|
||||
{Type: zeroruntime.StreamEventToolCallEnd, ToolCallID: "c1"},
|
||||
{Type: zeroruntime.StreamEventDone},
|
||||
}
|
||||
provider := &mockProvider{turns: [][]zeroruntime.StreamEvent{
|
||||
toolTurn,
|
||||
toolTurn,
|
||||
{
|
||||
{Type: zeroruntime.StreamEventText, Content: "recovered after escalation"},
|
||||
{Type: zeroruntime.StreamEventDone},
|
||||
},
|
||||
}}
|
||||
|
||||
recorder := trace.NewRecorder("posture-session", "run-1", "fast")
|
||||
result, err := Run(context.Background(), "go", provider, Options{
|
||||
Registry: registry,
|
||||
MaxTurns: 2,
|
||||
Trace: recorder,
|
||||
Profile: &ProfilePolicy{
|
||||
Name: "fast",
|
||||
Escalate: &PostureEscalation{
|
||||
MaxTurns: 4,
|
||||
ReasoningEffort: "high",
|
||||
OnToolFailureStreak: 2,
|
||||
},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if result.FinalAnswer != "recovered after escalation" {
|
||||
t.Fatalf("expected the run to continue past the original 2-turn ceiling, got %q (turns=%d)", result.FinalAnswer, result.Turns)
|
||||
}
|
||||
if len(provider.requests) != 3 {
|
||||
t.Fatalf("expected 3 requests (ceiling raised from 2 to 4), got %d", len(provider.requests))
|
||||
}
|
||||
if provider.requests[2].ReasoningEffort != "high" {
|
||||
t.Fatalf("expected the post-escalation request to carry the target effort, got %q", provider.requests[2].ReasoningEffort)
|
||||
}
|
||||
tr := recorder.Finish()
|
||||
if got := tr.Counter(trace.CounterPostureEscalations); got != 1 {
|
||||
t.Fatalf("posture_escalations = %d, want 1", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestPostureEscalationAbsentWithoutProfile pins the no-regression invariant at
|
||||
// the loop level: the identical failing script without a profile ends at the
|
||||
// original ceiling with the max-turns answer.
|
||||
func TestPostureEscalationAbsentWithoutProfile(t *testing.T) {
|
||||
registry := tools.NewRegistry()
|
||||
registry.Register(failingProfileTool{})
|
||||
|
||||
toolTurn := []zeroruntime.StreamEvent{
|
||||
{Type: zeroruntime.StreamEventToolCallStart, ToolCallID: "c1", ToolName: "flaky_probe"},
|
||||
{Type: zeroruntime.StreamEventToolCallEnd, ToolCallID: "c1"},
|
||||
{Type: zeroruntime.StreamEventDone},
|
||||
}
|
||||
provider := &mockProvider{turns: [][]zeroruntime.StreamEvent{toolTurn, toolTurn}}
|
||||
|
||||
result, err := Run(context.Background(), "go", provider, Options{
|
||||
Registry: registry,
|
||||
MaxTurns: 2,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(provider.requests) > 3 {
|
||||
t.Fatalf("unprofiled run must not extend the ceiling, got %d requests", len(provider.requests))
|
||||
}
|
||||
if !strings.Contains(result.FinalAnswer, "maximum number of turns") {
|
||||
t.Fatalf("unprofiled run must end with the max-turns answer, got %q", result.FinalAnswer)
|
||||
}
|
||||
}
|
||||
|
||||
// TestExecuteToolCallClassifiesRiskWithoutSandbox verifies the executed-risk
|
||||
// stamp on the unsandboxed path: a read-only tool classifies low.
|
||||
func TestExecuteToolCallClassifiesRiskWithoutSandbox(t *testing.T) {
|
||||
registry := tools.NewRegistry()
|
||||
registry.Register(failingProfileTool{})
|
||||
|
||||
result, abortErr := executeToolCall(
|
||||
context.Background(),
|
||||
registry,
|
||||
ToolCall{ID: "c1", Name: "flaky_probe"},
|
||||
PermissionModeAuto,
|
||||
Options{},
|
||||
)
|
||||
if abortErr != nil {
|
||||
t.Fatal(abortErr)
|
||||
}
|
||||
if result.Risk.Level == "" {
|
||||
t.Fatal("executed result must carry a risk classification")
|
||||
}
|
||||
if riskRank(result.Risk.Level) > riskRank(sandbox.RiskMedium) {
|
||||
t.Fatalf("read-only tool classified %q, want at most medium", result.Risk.Level)
|
||||
}
|
||||
}
|
||||
|
||||
// TestUnknownToolResultCarriesZeroRisk verifies a call that never executed a
|
||||
// tool (unknown name, nil tool) keeps the zero risk value instead of panicking
|
||||
// or inventing a classification.
|
||||
func TestUnknownToolResultCarriesZeroRisk(t *testing.T) {
|
||||
registry := tools.NewRegistry()
|
||||
result, abortErr := executeToolCall(
|
||||
context.Background(),
|
||||
registry,
|
||||
ToolCall{ID: "c1", Name: "no_such_tool"},
|
||||
PermissionModeAuto,
|
||||
Options{},
|
||||
)
|
||||
if abortErr != nil {
|
||||
t.Fatal(abortErr)
|
||||
}
|
||||
if result.Status == tools.StatusOK {
|
||||
t.Fatal("expected an error result for an unknown tool")
|
||||
}
|
||||
if result.Risk.Level != "" {
|
||||
t.Fatalf("not-executed result must keep the zero risk value, got %q", result.Risk.Level)
|
||||
}
|
||||
}
|
||||
@@ -75,6 +75,12 @@ type ToolResult struct {
|
||||
// DenialReason categorizes why a tool call was blocked (empty when it ran).
|
||||
// It lets a surface distinguish the cause precisely instead of parsing Output.
|
||||
DenialReason DenialCategory
|
||||
// Risk is the sandbox risk classification of this call, stamped for EXECUTED
|
||||
// results so run-policy observers (the execution-profile controller) can see
|
||||
// the risk level of an allowed mutation. It mirrors the classification the
|
||||
// permission path already computes; denied or canceled results keep the zero
|
||||
// value. Pure observation: nothing about permissions or sandboxing changes.
|
||||
Risk sandbox.Risk
|
||||
// LoadedTools carries the deferred-tool names a tool_search call asked the
|
||||
// loop to expose next turn (lifted from Meta["load_tools"]). nil for every
|
||||
// ordinary tool result; only tool_search populates it.
|
||||
@@ -98,6 +104,48 @@ const (
|
||||
DenialHookBlocked DenialCategory = "hook_blocked" // vetoed by a beforeTool hook
|
||||
)
|
||||
|
||||
// ProfilePolicy is the loop-facing slice of a selected execution profile.
|
||||
// The surface (exec flag, TUI command) resolves a named profile into this
|
||||
// policy; the loop only ever sees the policy, never the catalog.
|
||||
type ProfilePolicy struct {
|
||||
// Name labels trace counters and diagnostics (e.g. "fast"). The trace
|
||||
// recorder's profile label is set by the caller at recorder construction.
|
||||
Name string
|
||||
// Escalate, when non-nil, arms one-shot in-run posture escalation.
|
||||
Escalate *PostureEscalation
|
||||
}
|
||||
|
||||
// PostureEscalation describes a one-shot escalation to stricter knob values,
|
||||
// applied mid-run when an armed trigger fires. Targets are the values the
|
||||
// selected profile DISPLACED at run start (i.e. "restore the balanced
|
||||
// posture"), so escalation can never introduce a value that was not already
|
||||
// valid for this run and model. Zero-valued targets leave that knob untouched.
|
||||
type PostureEscalation struct {
|
||||
// MaxTurns raises the turn ceiling to this value when greater than the
|
||||
// ceiling in effect. 0 leaves the ceiling untouched.
|
||||
MaxTurns int
|
||||
// ReasoningEffort replaces the run's effort when non-empty.
|
||||
ReasoningEffort string
|
||||
// RestoreCompletionGate re-enables RequireCompletionSignal (headless
|
||||
// completion semantics) when true.
|
||||
RestoreCompletionGate bool
|
||||
|
||||
// Triggers. A zero value disables that signal entirely.
|
||||
// OnToolFailureStreak fires when the repeated-failure guard observes a
|
||||
// same-tool retriable-failure streak of at least this length.
|
||||
OnToolFailureStreak int
|
||||
// OnCompletionUncertain fires on the Nth uncertain completion evaluation
|
||||
// (continue nudge or semantic check). Headless only: the completion gate
|
||||
// never runs interactively.
|
||||
OnCompletionUncertain int
|
||||
// OnSelfCorrectFailure fires when a post-edit verification cycle reports a
|
||||
// failing outcome (correcting, reported, or aborted).
|
||||
OnSelfCorrectFailure bool
|
||||
// OnRiskyMutation fires when an EXECUTED tool result carries a sandbox risk
|
||||
// level at or above this threshold. Empty disables the signal.
|
||||
OnRiskyMutation sandbox.RiskLevel
|
||||
}
|
||||
|
||||
type PermissionRequest struct {
|
||||
ToolCallID string `json:"toolCallId"`
|
||||
ToolName string `json:"name"`
|
||||
@@ -298,6 +346,12 @@ type Options struct {
|
||||
// contract as ModelSwitcher: a returned error records a note and the run
|
||||
// continues on the current model and session.
|
||||
ModelSessionSwitcher func(ctx context.Context, modelID string) (zeroruntime.TurnSessionProvider, error)
|
||||
// Profile, when set, arms the execution-profile posture controller for this
|
||||
// run (auto-escalation to stricter knob values on failure/uncertainty/risky
|
||||
// mutation signals). nil — the default everywhere today — leaves the loop
|
||||
// byte-identical: no observation, no escalation, no counters. Same opt-in
|
||||
// convention as Trace and SelfCorrect.
|
||||
Profile *ProfilePolicy
|
||||
// Trace, when set, records per-turn timing for the run: the loop stamps
|
||||
// spans (prompt build, generation, tool execution, permission wait,
|
||||
// compaction, provider connect) and counters (model requests, tool calls,
|
||||
|
||||
@@ -133,7 +133,7 @@
|
||||
"verificationCommand": [
|
||||
"bash",
|
||||
"-c",
|
||||
"go test ./... && ! grep -RIn 'MaxRetries =' ."
|
||||
"go test ./... && ! grep --exclude=.zero-answer.txt -RIn 'MaxRetries =' ."
|
||||
],
|
||||
"oracleTest": "package edit\n\nvar _ = RetryLimit\n"
|
||||
},
|
||||
@@ -146,7 +146,7 @@
|
||||
"verificationCommand": [
|
||||
"bash",
|
||||
"-c",
|
||||
"go test ./... && ! grep -R receieves . && grep -R receives ."
|
||||
"go test ./... && ! grep --exclude=.zero-answer.txt -R receieves . && grep --exclude=.zero-answer.txt -R receives ."
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -400,7 +400,7 @@
|
||||
"verificationCommand": [
|
||||
"bash",
|
||||
"-c",
|
||||
"go test ./... && ! grep -R 'package refactor' . && grep -R 'package zeroapp' ."
|
||||
"go test ./... && ! grep --exclude=.zero-answer.txt -R 'package refactor' . && grep --exclude=.zero-answer.txt -R 'package zeroapp' ."
|
||||
],
|
||||
"oracleTest": "package zeroapp\n\nvar _ = Config{}\n"
|
||||
},
|
||||
@@ -426,7 +426,7 @@
|
||||
"verificationCommand": [
|
||||
"bash",
|
||||
"-c",
|
||||
"go test ./... && ! grep -R 'func Wrapper' ."
|
||||
"go test ./... && ! grep --exclude=.zero-answer.txt -R 'func Wrapper' ."
|
||||
],
|
||||
"oracleTest": "package refactor\n\nimport \"testing\"\n\n// TestGreetWrapped proves greetWrapped still exists and behaves as the inlined\n// Wrapper did: deleting both Wrapper and greetWrapped (or stubbing greetWrapped\n// to return \"\") fails to compile or run, so the oracle can't be passed by\n// removing the target along with its caller. Paired with the ! grep 'func\n// Wrapper' verificationCommand that proves Wrapper itself is gone.\nfunc TestGreetWrapped(t *testing.T) {\n\tif got := greetWrapped(\"x\"); got != \"hello, x\" {\n\t\tt.Fatalf(\"greetWrapped(x) = %q, want %q\", got, \"hello, x\")\n\t}\n}\n"
|
||||
},
|
||||
|
||||
@@ -40,7 +40,19 @@ import (
|
||||
// prevents a v2→v2 cross-version comparison from misreading the jump as a model
|
||||
// improvement. New counts: correctness 34 (edit 10 + fix 8 + nav 10 + refactor
|
||||
// 6), build 0, latency 14 (longproc 4 + longctx 4 + parallel 6).
|
||||
const TurnSchemaVersion = 3
|
||||
//
|
||||
// v4 adds tasksErrored: the number of tasks whose every iteration produced no
|
||||
// accepted benchmark sample. That covers pre-run failures (missing binary,
|
||||
// spawn error, process crash) and harness errors after a successful agent run
|
||||
// (e.g. oracle stamping failed) alike — in every case the iteration yields no
|
||||
// usable measurement. Errored tasks were previously visible only in warnings,
|
||||
// so a run where nothing was actually measured still printed a clean-looking
|
||||
// 0% pass summary and exited 0. tasksErrored makes that state first-class:
|
||||
// the summary surfaces it, and the turn command exits nonzero when every
|
||||
// attempted task errored. Accounting: an errored oracle task stays in its
|
||||
// tier denominator as a failure; an errored latency-only task counts in
|
||||
// latencyOnlyTasks and, as always, in no pass rate.
|
||||
const TurnSchemaVersion = 4
|
||||
|
||||
// TurnRunner runs one benchmark task and reports its outcome plus the captured
|
||||
// per-turn trace. A non-nil Err means the run failed to execute (process crash);
|
||||
@@ -144,6 +156,7 @@ type TurnBenchResult struct {
|
||||
Commit string `json:"commit,omitempty"`
|
||||
Date string `json:"date"`
|
||||
TasksAttempted int `json:"tasksAttempted"`
|
||||
TasksErrored int `json:"tasksErrored"`
|
||||
TasksVerified int `json:"tasksVerified"`
|
||||
TasksPassed int `json:"tasksPassed"`
|
||||
LatencyOnlyTasks int `json:"latencyOnlyTasks"`
|
||||
@@ -249,6 +262,7 @@ func RunTurnBench(ctx context.Context, set TaskSet, cfg TurnBenchConfig) (TurnBe
|
||||
// runner is cold-start, so a flaky pass on one iteration and a fail on
|
||||
// another is a real regression signal, not noise to average away.
|
||||
passedForTask := true
|
||||
erroredIterations := 0
|
||||
for iter := 0; iter < iterations; iter++ {
|
||||
outcome := cfg.Runner(ctx, task, rc)
|
||||
if outcome.TraceIssue != "" {
|
||||
@@ -266,6 +280,7 @@ func RunTurnBench(ctx context.Context, set TaskSet, cfg TurnBenchConfig) (TurnBe
|
||||
Message: fmt.Sprintf("task %s: %v", task.ID, outcome.Err),
|
||||
})
|
||||
passedForTask = false
|
||||
erroredIterations++
|
||||
continue
|
||||
}
|
||||
if !outcome.Passed {
|
||||
@@ -306,6 +321,14 @@ func RunTurnBench(ctx context.Context, set TaskSet, cfg TurnBenchConfig) (TurnBe
|
||||
// counters. A latency-only task (no verificationCommand) is never counted
|
||||
// in any pass rate even when the runner reports Passed — an exit-0
|
||||
// read-only run proves the turn ran, not that the answer was right.
|
||||
if erroredIterations == iterations {
|
||||
// Every iteration produced no accepted sample (spawn failure or a
|
||||
// harness error after the run). The task still counts in its tier
|
||||
// below — an oracle task as a failure, a latency-only task in no
|
||||
// pass rate as always — but the errored state is first-class so a
|
||||
// broken run can never print a clean-looking summary.
|
||||
result.TasksErrored++
|
||||
}
|
||||
hasOracle := len(task.VerificationCommand) > 0
|
||||
switch {
|
||||
case !hasOracle:
|
||||
@@ -456,6 +479,20 @@ func FormatTurnBenchSummary(result TurnBenchResult) string {
|
||||
result.BuildPassedTasks, result.BuildCheckedTasks, result.BuildPassRate*100,
|
||||
result.LatencyOnlyTasks, result.Iterations),
|
||||
}
|
||||
if result.TasksErrored > 0 {
|
||||
lines = append(lines, fmt.Sprintf("ERRORED: %d task(s) produced no accepted benchmark sample (spawn/crash or harness error) — errored oracle tasks count as failures in their tier's pass rate; latency-only tasks stay out of pass rates as always", result.TasksErrored))
|
||||
shown := 0
|
||||
for _, warning := range result.Warnings {
|
||||
if warning.Metric != "run" {
|
||||
continue
|
||||
}
|
||||
lines = append(lines, " "+warning.Message)
|
||||
shown++
|
||||
if shown == 3 {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
if result.Mode != "" {
|
||||
lines = append(lines, "mode: "+result.Mode)
|
||||
}
|
||||
@@ -747,7 +784,15 @@ func ResolveBinary(explicit string) (string, error) {
|
||||
if _, err := os.Stat(v); err != nil {
|
||||
return "", fmt.Errorf("trace binary not found: %w", err)
|
||||
}
|
||||
return v, nil
|
||||
// Pin the explicit path to an absolute one: the turn runner sets each
|
||||
// child's cmd.Dir to a per-task fixture copy, so a relative path
|
||||
// (./zero, the Makefile default) that stats fine here would fail to
|
||||
// spawn from inside every fixture dir, silently erroring all tasks.
|
||||
absolute, err := filepath.Abs(v)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("resolve trace binary path: %w", err)
|
||||
}
|
||||
return absolute, nil
|
||||
}
|
||||
if path, err := exec.LookPath("zero"); err == nil {
|
||||
return path, nil
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
@@ -530,14 +531,15 @@ func TestRunTurnBenchBuildOnlyMechanism(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestTurnSchemaVersion3 pins the schema bump. v3 records the tier
|
||||
// reclassification (refactor structural-positive and nav answer-oracles moved
|
||||
// into correctnessPassRate), so a v2->v3 cross-version comparison cannot
|
||||
// silently misread the jump as a model improvement — exactly the misread the
|
||||
// tier system exists to prevent.
|
||||
func TestTurnSchemaVersion3(t *testing.T) {
|
||||
if TurnSchemaVersion != 3 {
|
||||
t.Fatalf("TurnSchemaVersion = %d, want 3", TurnSchemaVersion)
|
||||
// TestTurnSchemaVersion pins the schema bump so a shape change is a conscious
|
||||
// decision. v3 recorded the tier reclassification (refactor structural-positive
|
||||
// and nav answer-oracles moved into correctnessPassRate). v4 adds tasksErrored:
|
||||
// tasks whose every iteration died before the agent produced a run are now
|
||||
// first-class in the report instead of visible only in warnings, so a
|
||||
// spawn-broken run cannot print a clean-looking summary or exit 0.
|
||||
func TestTurnSchemaVersion(t *testing.T) {
|
||||
if TurnSchemaVersion != 4 {
|
||||
t.Fatalf("TurnSchemaVersion = %d, want 4", TurnSchemaVersion)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1067,3 +1069,95 @@ printf '%s\n' '{"type":"run_end","exitCode":0}'
|
||||
`)
|
||||
assertVerifyFailed(t, "nav-08 fmt-only-no-testing", outcome)
|
||||
}
|
||||
|
||||
func TestResolveBinaryAbsolutizesExplicitPath(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
name := "zero-probe.exe"
|
||||
if err := os.WriteFile(filepath.Join(dir, name), []byte("x"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cwd, err := os.Getwd()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.Chdir(dir); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = os.Chdir(cwd) })
|
||||
|
||||
resolved, err := ResolveBinary("./" + name)
|
||||
if err != nil {
|
||||
t.Fatalf("ResolveBinary: %v", err)
|
||||
}
|
||||
if !filepath.IsAbs(resolved) {
|
||||
t.Fatalf("ResolveBinary returned relative path %q; the turn runner sets cmd.Dir per task, so a relative binary fails to spawn from every fixture copy", resolved)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunTurnBenchCountsErroredTasks(t *testing.T) {
|
||||
set := TaskSet{
|
||||
ID: "errored-suite",
|
||||
Tasks: []BenchTask{
|
||||
{ID: "t1", Class: "nav", Prompt: "p1", VerificationCommand: []string{"true"}},
|
||||
{ID: "t2", Class: "longproc", Prompt: "p2"},
|
||||
},
|
||||
}
|
||||
cfg := TurnBenchConfig{
|
||||
Model: "fake-model",
|
||||
Iterations: 1,
|
||||
Runner: func(context.Context, BenchTask, RunContext) TurnTaskOutcome {
|
||||
return TurnTaskOutcome{Err: errors.New("fork/exec ./zero: file does not exist")}
|
||||
},
|
||||
Now: func() time.Time { return time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC) },
|
||||
}
|
||||
result, err := RunTurnBench(context.Background(), set, cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("RunTurnBench: %v", err)
|
||||
}
|
||||
if result.TasksErrored != 2 {
|
||||
t.Fatalf("TasksErrored=%d, want 2 (every iteration of both tasks errored)", result.TasksErrored)
|
||||
}
|
||||
if len(result.Warnings) != 2 {
|
||||
t.Fatalf("warnings=%d, want 2 run warnings", len(result.Warnings))
|
||||
}
|
||||
summary := FormatTurnBenchSummary(result)
|
||||
if !strings.Contains(summary, "ERRORED: 2 task(s)") {
|
||||
t.Fatalf("summary does not surface the errored tasks:\n%s", summary)
|
||||
}
|
||||
if !strings.Contains(summary, "fork/exec ./zero") {
|
||||
t.Fatalf("summary does not echo the underlying run error:\n%s", summary)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunTurnBenchPartialErrorStillCounts(t *testing.T) {
|
||||
set := TaskSet{
|
||||
ID: "partial-suite",
|
||||
Tasks: []BenchTask{
|
||||
{ID: "ok", Class: "nav", Prompt: "p", VerificationCommand: []string{"true"}},
|
||||
{ID: "dead", Class: "nav", Prompt: "p", VerificationCommand: []string{"true"}},
|
||||
},
|
||||
}
|
||||
canned := map[string]*trace.TurnTrace{"ok": cannedTrace(100, 10, 1000)}
|
||||
inner := fakeTurnRunner(canned)
|
||||
cfg := TurnBenchConfig{
|
||||
Model: "fake-model",
|
||||
Iterations: 1,
|
||||
Runner: func(ctx context.Context, task BenchTask, rc RunContext) TurnTaskOutcome {
|
||||
if task.ID == "dead" {
|
||||
return TurnTaskOutcome{Err: errors.New("spawn failed")}
|
||||
}
|
||||
return inner(ctx, task, rc)
|
||||
},
|
||||
Now: func() time.Time { return time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC) },
|
||||
}
|
||||
result, err := RunTurnBench(context.Background(), set, cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("RunTurnBench: %v", err)
|
||||
}
|
||||
if result.TasksErrored != 1 {
|
||||
t.Fatalf("TasksErrored=%d, want 1", result.TasksErrored)
|
||||
}
|
||||
if result.TasksAttempted != 2 {
|
||||
t.Fatalf("TasksAttempted=%d, want 2", result.TasksAttempted)
|
||||
}
|
||||
}
|
||||
@@ -57,6 +57,11 @@ const (
|
||||
CounterPrewarmAttempts = "prewarm_attempts"
|
||||
CounterPrefixStable = "prefix_stable"
|
||||
CounterPrefixDrift = "prefix_drift"
|
||||
// CounterPostureEscalations counts one-shot in-run posture escalations by
|
||||
// the execution-profile controller. Deliberately distinct from
|
||||
// model_switches: a posture escalation changes loop policy knobs, never the
|
||||
// model or session.
|
||||
CounterPostureEscalations = "posture_escalations"
|
||||
)
|
||||
|
||||
// Span is one named wall interval attributed to part of a run. Each stamp is
|
||||
@@ -289,6 +294,7 @@ func OptionalEventKeys() []string {
|
||||
"counter:" + CounterPrewarmAttempts,
|
||||
"counter:" + CounterPrefixStable,
|
||||
"counter:" + CounterPrefixDrift,
|
||||
"counter:" + CounterPostureEscalations,
|
||||
"event:prefix_hash",
|
||||
}
|
||||
}
|
||||
@@ -577,3 +577,12 @@ func TestCounterPrecisionRoundTrip(t *testing.T) {
|
||||
t.Fatalf("counter round-trip = %d, want %d (float64 precision loss)", got, big)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOptionalEventKeysIncludePostureEscalations(t *testing.T) {
|
||||
for _, key := range OptionalEventKeys() {
|
||||
if key == "counter:"+CounterPostureEscalations {
|
||||
return
|
||||
}
|
||||
}
|
||||
t.Fatalf("posture_escalations missing from OptionalEventKeys: %v", OptionalEventKeys())
|
||||
}
|
||||
Reference in new issue
Block a user