mirror of
https://github.com/esengine/DeepSeek-Reasonix.git
synced 2026-11-07 10:45:59 -10:00
yhh
394da0a313
internal/ was 141 packages side by side, so nothing but a reader's memory
said which were foundations and which assembled the rest. Each now sits in a
layer directory, and the directory is the declaration:
base L0 foundation with no domain in it
contract L1 types and interfaces the layers above agree on
model, tools, safety, ext, state, platform
L2 subsystems, peers of one another
runtime L3 the turn engine
session L4 the controller
assembly L5 the composition root
frontend L6 cli, serve, acp and the host adapters
Only paths change: package names, code and behaviour do not. Imports,
repolint and coverage baselines, sensitive paths, CI package lists and docs
follow the new paths; release notes keep the paths they were written with,
and sample paths inside tests and fixtures keep theirs.
repolint gains layer-order: a package may import its own layer and those
below it, read from where it lives. The 47 imports that point upward today
are recorded as that rule's opening debt (-allow-widen, a new rule's first
baseline) and can only go down. Most of it is config, which still reaches
into eight subsystems and is the next thing to take apart; the rest is
event, history and a few tools reaching into runtime for types that belong
in contract.
197 lines
8.0 KiB
Go
197 lines
8.0 KiB
Go
package main
|
|
|
|
import (
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
|
|
"reasonix/internal/base/testenv"
|
|
)
|
|
|
|
func writeTrajectory(t *testing.T, name string, lines []string) string {
|
|
t.Helper()
|
|
path := filepath.Join(testenv.TempDir(t), name)
|
|
if err := os.WriteFile(path, []byte(strings.Join(lines, "\n")), 0o644); err != nil {
|
|
t.Fatalf("write fixture: %v", err)
|
|
}
|
|
return path
|
|
}
|
|
|
|
func TestSummarizeOutcomeFromRecordedShadowSamples(t *testing.T) {
|
|
path := writeTrajectory(t, "shadow.trajectory.jsonl", []string{
|
|
`{"seq":1,"ts":500,"event":{"kind":"turn_started"}}`,
|
|
`{"seq":2,"ts":1000,"outcome_progress":{"round":1,"exploration":1,"legacy_gain":1}}`,
|
|
`{"seq":3,"ts":2000,"outcome_progress":{"round":2,"verification":1,"legacy_gain":2}}`,
|
|
`{"seq":4,"ts":3000,"outcome_progress":{"round":3,"churn":1,"legacy_gain":3}}`,
|
|
`{"seq":5,"ts":4000,"outcome_progress":{"round":4,"verification":1,"objective":1,"legacy_gain":2}}`,
|
|
`{"seq":6,"ts":5000,"outcome_progress":{"round":5,"churn":1,"legacy_gain":3}}`,
|
|
`{"seq":7,"ts":6000,"outcome_progress":{"round":6,"verification":1,"regression":1}}`,
|
|
`{"seq":8,"ts":7000,"event":{"kind":"turn_done"}}`,
|
|
})
|
|
s, err := summarizeTrajectory(path)
|
|
if err != nil {
|
|
t.Fatalf("summarizeTrajectory: %v", err)
|
|
}
|
|
o := s.Outcome
|
|
if o == nil || o.Backfilled {
|
|
t.Fatalf("outcome = %+v, want recorded (not backfilled)", o)
|
|
}
|
|
if o.Rounds != 6 || o.ProgressRounds != 5 {
|
|
t.Errorf("rounds=%d progress=%d, want 6/5", o.Rounds, o.ProgressRounds)
|
|
}
|
|
// Round 5 claimed legacy progress (a mutation) with no objective transition
|
|
// inside the redemption window — the false-progress case.
|
|
if o.FalseProgressRounds != 1 {
|
|
t.Errorf("false progress = %d, want 1", o.FalseProgressRounds)
|
|
}
|
|
if o.SolutionStallMax != 2 {
|
|
t.Errorf("solution stall max = %d, want 2", o.SolutionStallMax)
|
|
}
|
|
if o.Objective != 1 || o.Regression != 1 || o.BestScore != 1 || o.FinalScore != 0 {
|
|
t.Errorf("objective=%d regression=%d best=%d final=%d, want 1/1/1/0",
|
|
o.Objective, o.Regression, o.BestScore, o.FinalScore)
|
|
}
|
|
if !o.RegressedFromBest || o.SearchRegretMs != 3000 {
|
|
t.Errorf("regressed=%v regret=%d, want true/3000 (best at ts 4000, end at 7000)",
|
|
o.RegressedFromBest, o.SearchRegretMs)
|
|
}
|
|
}
|
|
|
|
func TestSummarizeOutcomeBackfillsFromVerificationReceipts(t *testing.T) {
|
|
args := `{\"command\":\"go test ./x\"}`
|
|
path := writeTrajectory(t, "old.trajectory.jsonl", []string{
|
|
`{"seq":1,"ts":1000,"event":{"kind":"turn_started"}}`,
|
|
`{"seq":2,"ts":2000,"event":{"kind":"tool_result","tool":{"name":"bash","args":"` + args + `","err":"exit 1","execution":{"verification":"failed"}}}}`,
|
|
`{"seq":3,"ts":3000,"event":{"kind":"tool_result","tool":{"name":"bash","args":"` + args + `","execution":{"verification":"passed"}}}}`,
|
|
`{"seq":4,"ts":4000,"event":{"kind":"tool_result","tool":{"name":"bash","args":"` + args + `","err":"exit 1","execution":{"verification":"failed"}}}}`,
|
|
`{"seq":5,"ts":5000,"event":{"kind":"turn_done"}}`,
|
|
})
|
|
s, err := summarizeTrajectory(path)
|
|
if err != nil {
|
|
t.Fatalf("summarizeTrajectory: %v", err)
|
|
}
|
|
o := s.Outcome
|
|
if o == nil || !o.Backfilled {
|
|
t.Fatalf("outcome = %+v, want a backfilled summary", o)
|
|
}
|
|
if o.Objective != 1 || o.Regression != 1 || o.BestScore != 1 || o.FinalScore != 0 {
|
|
t.Errorf("objective=%d regression=%d best=%d final=%d, want 1/1/1/0",
|
|
o.Objective, o.Regression, o.BestScore, o.FinalScore)
|
|
}
|
|
if !o.RegressedFromBest || o.SearchRegretMs != 2000 {
|
|
t.Errorf("regressed=%v regret=%d, want true/2000", o.RegressedFromBest, o.SearchRegretMs)
|
|
}
|
|
if o.FalseProgressRounds != 0 || o.ProgressRounds != 0 {
|
|
t.Errorf("backfill cannot price legacy claims, got progress=%d false=%d",
|
|
o.ProgressRounds, o.FalseProgressRounds)
|
|
}
|
|
// A subagent's verification must not pollute the parent series.
|
|
sub := writeTrajectory(t, "sub.trajectory.jsonl", []string{
|
|
`{"seq":1,"ts":1000,"event":{"kind":"turn_started"}}`,
|
|
`{"seq":2,"ts":2000,"event":{"kind":"tool_result","tool":{"name":"bash","args":"` + args + `","parentId":"task-1","execution":{"verification":"failed"}}}}`,
|
|
`{"seq":3,"ts":3000,"event":{"kind":"turn_done"}}`,
|
|
})
|
|
if s, err = summarizeTrajectory(sub); err != nil {
|
|
t.Fatalf("summarizeTrajectory: %v", err)
|
|
}
|
|
if s.Outcome != nil {
|
|
t.Errorf("subagent-only verification must yield no outcome summary, got %+v", s.Outcome)
|
|
}
|
|
}
|
|
|
|
func TestSummarizeOutcomeTracksDebtAndTTFDC(t *testing.T) {
|
|
path := writeTrajectory(t, "debt.trajectory.jsonl", []string{
|
|
`{"seq":1,"ts":1000,"event":{"kind":"turn_started"}}`,
|
|
`{"seq":2,"ts":2000,"outcome_progress":{"round":1,"churn":1,"legacy_gain":3,"debt_age":1}}`,
|
|
`{"seq":3,"ts":3000,"outcome_progress":{"round":2,"exploration":1,"legacy_gain":1,"debt_age":2}}`,
|
|
`{"seq":4,"ts":4000,"outcome_progress":{"round":3,"churn":1,"legacy_gain":3,"debt_age":3}}`,
|
|
`{"seq":5,"ts":9000,"outcome_progress":{"round":4,"discriminating":1,"verification":1}}`,
|
|
`{"seq":6,"ts":10000,"event":{"kind":"turn_done"}}`,
|
|
})
|
|
s, err := summarizeTrajectory(path)
|
|
if err != nil {
|
|
t.Fatalf("summarizeTrajectory: %v", err)
|
|
}
|
|
o := s.Outcome
|
|
if o == nil || o.DebtAgeMax != 3 {
|
|
t.Fatalf("outcome = %+v, want debt age max 3", o)
|
|
}
|
|
if o.TTFDCMs != 8000 {
|
|
t.Fatalf("TTFDC = %d, want 8000 (run start 1000 → first discriminating 9000)", o.TTFDCMs)
|
|
}
|
|
got := renderOutcomeProgress([]result{{Trajectory: s}})
|
|
for _, want := range []string{"**discriminating checks** in 1/1 runs (TTFDC p50 8.0s)", "**verification debt max** 3 rounds"} {
|
|
if !strings.Contains(got, want) {
|
|
t.Errorf("render missing %q in:\n%s", want, got)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestSummarizeOutcomeSumsDebtExposure(t *testing.T) {
|
|
path := writeTrajectory(t, "debt-area.trajectory.jsonl", []string{
|
|
`{"seq":1,"ts":1000,"event":{"kind":"turn_started"}}`,
|
|
`{"seq":2,"ts":2000,"outcome_progress":{"round":1,"churn":2,"debt_age":1,"blind_mutations":2}}`,
|
|
`{"seq":3,"ts":3000,"outcome_progress":{"round":2,"churn":1,"debt_age":2,"blind_mutations":3}}`,
|
|
`{"seq":4,"ts":4000,"outcome_progress":{"round":3,"exploration":1,"debt_age":3,"blind_mutations":3}}`,
|
|
`{"seq":5,"ts":9000,"outcome_progress":{"round":4,"discriminating":1,"verification":1}}`,
|
|
`{"seq":6,"ts":10000,"event":{"kind":"turn_done"}}`,
|
|
})
|
|
s, err := summarizeTrajectory(path)
|
|
if err != nil {
|
|
t.Fatalf("summarizeTrajectory: %v", err)
|
|
}
|
|
o := s.Outcome
|
|
if o == nil || o.DebtArea != 6 || o.BlindPeak != 3 {
|
|
t.Fatalf("outcome = %+v, want debt area 6 and blind peak 3", o)
|
|
}
|
|
}
|
|
|
|
func TestRenderOutcomeProgressAggregatesRuns(t *testing.T) {
|
|
results := []result{
|
|
{Trajectory: &trajectorySummary{Outcome: &outcomeSummary{
|
|
Rounds: 8, ProgressRounds: 5, FalseProgressRounds: 2,
|
|
Objective: 2, BestScore: 2, FinalScore: 2, VerificationRevisits: 3,
|
|
}}},
|
|
{Trajectory: &trajectorySummary{Outcome: &outcomeSummary{
|
|
Objective: 1, Regression: 1, BestScore: 1, FinalScore: 0,
|
|
RegressedFromBest: true, SearchRegretMs: 4000, Backfilled: true,
|
|
}}},
|
|
{Trajectory: &trajectorySummary{}}, // no outcome data: excluded
|
|
}
|
|
got := renderOutcomeProgress(results)
|
|
for _, want := range []string{
|
|
"**Outcome shadow** (2 runs)",
|
|
"**objective transitions** 3",
|
|
"**regressed from best** 1 (50%)",
|
|
"**false progress** 2/5 (40%)",
|
|
"**avg search regret** 4.0s",
|
|
"backfilled 1",
|
|
} {
|
|
if !strings.Contains(got, want) {
|
|
t.Errorf("render missing %q in:\n%s", want, got)
|
|
}
|
|
}
|
|
if renderOutcomeProgress(nil) != "" {
|
|
t.Error("no runs must render nothing")
|
|
}
|
|
}
|
|
|
|
// TestRenderOutcomeProgressWithoutRepeatedChecks pins the honest reading: with
|
|
// no check ever run twice, a fail→pass edge was never observable, so the ratio
|
|
// would be reporting its own blindness.
|
|
func TestRenderOutcomeProgressWithoutRepeatedChecks(t *testing.T) {
|
|
results := []result{
|
|
{Trajectory: &trajectorySummary{Outcome: &outcomeSummary{
|
|
Rounds: 6, ProgressRounds: 4, FalseProgressRounds: 4, FinalScore: 0,
|
|
}}},
|
|
}
|
|
got := renderOutcomeProgress(results)
|
|
if !strings.Contains(got, "**false progress** unmeasured (no check ran twice)") {
|
|
t.Errorf("render must say unmeasured, got:\n%s", got)
|
|
}
|
|
if strings.Contains(got, "100%") {
|
|
t.Errorf("render must not claim a ratio it could not observe, got:\n%s", got)
|
|
}
|
|
}
|