fix(goaleval): read the verdict a thinking model leaves in reasoning (#10785)

With no recovery_model or guardian_model configured, the Goal evaluator
runs on the main model. A DeepSeek thinking model can stream its whole
answer as reasoning_content and finish with stop and an empty content
block. boundedllm.Call collected only ChunkText, so the evaluator got ""
and every goal paused with "empty goal evaluator response".

boundedllm.Config gains ReasoningAnswer. When a caller opts in and the
provider reports finish_reason=stop with no content, Call returns the
reasoning stream instead. A truncated stream still yields nothing, and
reasoning past MaxOutputBytes is refused only when it would become the
answer, so reasoning before real content is never charged. The goal
evaluator opts in because parseVerdict extracts the JSON object and fails
closed on anything else; free-text callers (advisor, prompt refine) keep
the content-only behaviour.

Test: TestEvaluateReadsVerdictFromReasoningOnlyStop fails on the base
with "empty goal evaluator response" and passes with the fix;
TestEvaluateIgnoresReasoningOfTruncatedStream and TestCallReasoningAnswer
cover the stop guard, opt-in and output budget.

Refs #9678

internal/model/boundedllm/bounded.go

@@ -54,6 +54,9 @@ type Config struct {
 	MaxSystemBytes int
 	// MaxTotalBytes is the hard cap on system + evidence. Zero uses DefaultMaxTotalBytes.
 	MaxTotalBytes int
+	// ReasoningAnswer returns the reasoning stream when a stop left content empty;
+	// only for callers whose contract extracts its answer from structure.
+	ReasoningAnswer bool
 }
 
 // Call runs one bounded no-tool request: system policy + a single user evidence
@@ -132,6 +135,7 @@ func Call(ctx context.Context, cfg Config, system, evidence string) (string, err
 	}
 
 	var text strings.Builder
+	reasoning := reasoningAnswer{keep: cfg.ReasoningAnswer, limit: maxOutputBytes}
 	for chunk := range ch {
 		switch chunk.Type {
 		case provider.ChunkText:
@@ -140,6 +144,8 @@ func Call(ctx context.Context, cfg Config, system, evidence string) (string, err
 				cancel()
 				return "", fmt.Errorf("bounded reviewer output exceeded %d bytes", maxOutputBytes)
 			}
+		case provider.ChunkReasoning:
+			reasoning.add(chunk.Text)
 		case provider.ChunkUsage:
 			if chunk.Usage != nil {
 				u := *chunk.Usage
@@ -155,5 +161,41 @@ func Call(ctx context.Context, cfg Config, system, evidence string) (string, err
 	if callCtx.Err() != nil && text.Len() == 0 {
 		return "", callCtx.Err()
 	}
+	if text.Len() == 0 {
+		return reasoning.answer(usage)
+	}
 	return text.String(), nil
 }
+
+// reasoningAnswer holds the reasoning stream of a caller that accepts it as the
+// answer. It is charged against the output budget only when it becomes one, so
+// a model that reasons at length before writing content is not refused.
+type reasoningAnswer struct {
+	keep     bool
+	limit    int
+	text     strings.Builder
+	overflow bool
+}
+
+func (r *reasoningAnswer) add(s string) {
+	if !r.keep || r.overflow {
+		return
+	}
+	r.text.WriteString(s)
+	if r.text.Len() > r.limit {
+		r.overflow = true
+		r.text.Reset()
+	}
+}
+
+// answer is the reasoning when the provider reported a clean stop; a truncated
+// stream's reasoning is unfinished thought, not an answer.
+func (r *reasoningAnswer) answer(usage *provider.Usage) (string, error) {
+	if !r.keep || usage == nil || usage.FinishReason != "stop" {
+		return "", nil
+	}
+	if r.overflow {
+		return "", fmt.Errorf("bounded reviewer reasoning exceeded %d bytes", r.limit)
+	}
+	return r.text.String(), nil
+}

internal/model/boundedllm/bounded_test.go

@@ -0,0 +1,53 @@
+package boundedllm
+
+import (
+	"context"
+	"strings"
+	"testing"
+
+	"reasonix/internal/contract/provider"
+)
+
+type chunkProvider []provider.Chunk
+
+func (chunkProvider) Name() string { return "chunks" }
+
+func (c chunkProvider) Stream(context.Context, provider.Request) (<-chan provider.Chunk, error) {
+	ch := make(chan provider.Chunk, len(c))
+	for _, chunk := range c {
+		ch <- chunk
+	}
+	close(ch)
+	return ch, nil
+}
+
+func stop() provider.Chunk {
+	return provider.Chunk{Type: provider.ChunkUsage, Usage: &provider.Usage{FinishReason: "stop"}}
+}
+
+func TestCallReasoningAnswer(t *testing.T) {
+	for _, tc := range []struct {
+		name    string
+		opt     bool
+		chunks  chunkProvider
+		want    string
+		wantErr bool
+	}{
+		{"content wins over reasoning", true, chunkProvider{{Type: provider.ChunkReasoning, Text: "think"}, {Type: provider.ChunkText, Text: "answer"}, stop()}, "answer", false},
+		{"reasoning-only stop answers when opted in", true, chunkProvider{{Type: provider.ChunkReasoning, Text: "ans"}, {Type: provider.ChunkReasoning, Text: "wer"}, stop()}, "answer", false},
+		{"reasoning-only stop stays empty without opt-in", false, chunkProvider{{Type: provider.ChunkReasoning, Text: "answer"}, stop()}, "", false},
+		{"reasoning without a stop is not an answer", true, chunkProvider{{Type: provider.ChunkReasoning, Text: "answer"}, {Type: provider.ChunkUsage, Usage: &provider.Usage{FinishReason: "length"}}}, "", false},
+		{"reasoning over the output budget fails", true, chunkProvider{{Type: provider.ChunkReasoning, Text: strings.Repeat("r", 65)}, stop()}, "", true},
+		{"long reasoning before content is not charged", true, chunkProvider{{Type: provider.ChunkReasoning, Text: strings.Repeat("r", 65)}, {Type: provider.ChunkText, Text: "answer"}, stop()}, "answer", false},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			got, err := Call(context.Background(), Config{Provider: tc.chunks, MaxOutputBytes: 64, ReasoningAnswer: tc.opt}, "sys", "ev")
+			if (err != nil) != tc.wantErr {
+				t.Fatalf("Call() error = %v, wantErr %v", err, tc.wantErr)
+			}
+			if got != tc.want {
+				t.Fatalf("Call() = %q, want %q", got, tc.want)
+			}
+		})
+	}
+}

internal/runtime/goaleval/evaluator.go

@@ -159,16 +159,17 @@ func (s *Session) Evaluate(ctx context.Context, evidence GoalEvidence) (Verdict,
 	defer s.mu.Unlock()
 
 	text, err := boundedllm.Call(ctx, boundedllm.Config{
-		Provider:       s.prov,
-		Pricing:        s.pricing,
-		ModelRef:       s.modelRef,
-		Sink:           s.sink,
-		UsageSource:    event.UsageSourceGoalEvaluator,
-		Timeout:        s.timeout,
-		MaxTokens:      MaxTokens,
-		MaxOutputBytes: MaxOutputBytes,
-		MaxSystemBytes: boundedllm.DefaultMaxSystemBytes,
-		MaxTotalBytes:  boundedllm.DefaultMaxTotalBytes,
+		Provider:        s.prov,
+		Pricing:         s.pricing,
+		ModelRef:        s.modelRef,
+		Sink:            s.sink,
+		UsageSource:     event.UsageSourceGoalEvaluator,
+		Timeout:         s.timeout,
+		MaxTokens:       MaxTokens,
+		MaxOutputBytes:  MaxOutputBytes,
+		MaxSystemBytes:  boundedllm.DefaultMaxSystemBytes,
+		MaxTotalBytes:   boundedllm.DefaultMaxTotalBytes,
+		ReasoningAnswer: true,
 	}, PolicyPrompt, payload)
 	if err != nil {
 		return Verdict{}, err

internal/runtime/goaleval/evaluator_test.go

@@ -158,3 +158,39 @@ func TestEvidenceIsBoundedAndUntrusted(t *testing.T) {
 		t.Fatalf("provider calls = %d, want 1", prov.calls)
 	}
 }
+
+type reasoningOnlyProvider struct {
+	reasoning string
+	finish    string
+}
+
+func (reasoningOnlyProvider) Name() string { return "reasoning-only" }
+
+func (r reasoningOnlyProvider) Stream(context.Context, provider.Request) (<-chan provider.Chunk, error) {
+	ch := make(chan provider.Chunk, 3)
+	ch <- provider.Chunk{Type: provider.ChunkReasoning, Text: r.reasoning}
+	ch <- provider.Chunk{Type: provider.ChunkUsage, Usage: &provider.Usage{FinishReason: r.finish}}
+	close(ch)
+	return ch, nil
+}
+
+func TestEvaluateReadsVerdictFromReasoningOnlyStop(t *testing.T) {
+	prov := reasoningOnlyProvider{
+		reasoning: "The contract is met.\n" + `{"outcome":"complete","reason":"all steps done"}`,
+		finish:    "stop",
+	}
+	v, err := evaluate(t, prov, GoalEvidence{GoalContract: "x"})
+	if err != nil {
+		t.Fatalf("Evaluate() error = %v, want the verdict carried in reasoning", err)
+	}
+	if v.Outcome != OutcomeComplete {
+		t.Fatalf("outcome = %q, want complete", v.Outcome)
+	}
+}
+
+func TestEvaluateIgnoresReasoningOfTruncatedStream(t *testing.T) {
+	prov := reasoningOnlyProvider{reasoning: `{"outcome":"complete","reason":"x"}`, finish: "length"}
+	if _, err := evaluate(t, prov, GoalEvidence{GoalContract: "x"}); err == nil {
+		t.Fatal("Evaluate() error = nil, want a truncated reasoning-only stream to fail closed")
+	}
+}