fix(bot): stop recovering nil dereferences that corrupt the heap on Windows

The Windows CI leg crashed twice inside internal/bot's test binary with
Go runtime faults:
- "Exception 0xc0000005 ... PC=0x0" (44150b0aa);
- "fatal error: acquireSudog: found s.elem != nil in cache" during GC
  mark termination (d636e93d5).
Both are heap corruption surfacing wherever it is next read.

golang/go#81238 is still open. On windows-latest runners (Windows Server
2025), a host whose CPU saves a large XSTATE (Intel AMX) writes the
exception context below the goroutine stack when a hardware exception is
dispatched, corrupting the Go heap. The pool mixes hosts, which is why it
is intermittent. It needs a real exception, a nil dereference for
example; an ordinary panic() does not trigger it. golang/go#77975, the
Go 1.26 overlapped-I/O corruption, was fixed in 1.26.2, and it involved
UDP receive, not these paths.

internal/bot dispatched such exceptions on purpose.
safeBotControllerRuntimeStatus, safeBotControllerWorkspaceRoot and
safeBotSetToolApprovalMode wrapped their call in recover(). Test fakes
embed a nil botController, so every call to a method a fake did not
define dereferenced nil and was recovered. Counted with temporary
instrumentation on one run of ./internal/bot: SetToolApprovalMode 7,
WorkspaceRoot 3. In production the controller is a complete
*control.Controller, so the recover only hid real bugs.

- The three wrappers keep their nil check and lose the recover.
- The eight fakes embed stubBotController, a nil botController plus
  no-op SetToolApprovalMode, WorkspaceRoot and RuntimeStatus. A fake
  missing a method the gateway starts calling now fails at once on every
  platform.
- ci.yml runs internal/bot under the race detector, which it never did,
  and the Windows test leg prints the runner CPU so the next runtime
  crash can be matched against the host.

Found on the way and not changed here: a panic inside getOrCreateSession
unwinds through finishTurnItem, which takes the gateway lock the
panicking frame already holds. The gateway deadlocks instead of
crashing; with a fake that still embedded nil, the test hung rather than
panicked.

Checked: go test -race over internal/bot/... and internal/botruntime
passes; golangci-lint is clean on darwin and on GOOS=windows; actionlint
passes.
This commit is contained in:
esengine committed 2026-09-17 00:56:18 -07:00
1 parent 2964f483d6
commit 3d5b0b68db
6 files changed
+37 -29

No files matched your search

+9 -1
View File
@@ -133,6 +133,14 @@ jobs:
if: env.RUN_STEPS == 'true' && runner.os != 'Windows'
run: python3 -m unittest discover -s tools/trajectory -t tools/trajectory
# A hardware exception on a Windows host whose CPU saves a large XSTATE
# (Intel AMX) can corrupt the Go heap (golang/go#81238). The runner pool
# mixes hosts, so a runtime crash is only attributable with the CPU on record.
- name: runner CPU
if: env.RUN_STEPS == 'true' && runner.os == 'Windows'
shell: pwsh
run: Get-CimInstance Win32_Processor | Select-Object -ExpandProperty Name
- name: test
if: env.RUN_STEPS == 'true' && runner.os != 'Windows'
env:
@@ -234,7 +242,7 @@ jobs:
if: github.event_name == 'pull_request'
env:
REASONIX_RELEASE_CACHE_GUARD: "1"
run: go test -race ./internal/agent/... ./internal/plugin/... ./internal/jobs/... ./internal/proc/... ./internal/sandbox/... ./internal/filelock/... ./internal/eventwire/... ./internal/remote/... ./internal/extension/... ./internal/boot/... ./internal/control/... ./internal/tool/...
run: go test -race ./internal/agent/... ./internal/plugin/... ./internal/jobs/... ./internal/proc/... ./internal/sandbox/... ./internal/filelock/... ./internal/eventwire/... ./internal/remote/... ./internal/extension/... ./internal/boot/... ./internal/control/... ./internal/tool/... ./internal/bot/...
- name: test -race (full)
if: github.event_name != 'pull_request'
+15
View File
@@ -0,0 +1,15 @@
package bot
import "reasonix/internal/control"
// stubBotController answers the calls the gateway makes on every session, so a
// fake states only what its test is about. Everything else reaches the nil
// botController and panics: a fake missing a method the gateway starts calling
// fails at once on every platform, instead of being recovered somewhere.
type stubBotController struct {
botController
}
func (stubBotController) SetToolApprovalMode(string) {}
func (stubBotController) WorkspaceRoot() string { return "" }
func (stubBotController) RuntimeStatus() control.RuntimeStatus { return control.RuntimeStatus{} }
+2 -2
View File
@@ -20,7 +20,7 @@ type closeProbeBotController struct {
}
type stopWaitBotController struct {
botController
stubBotController
started chan struct{}
release chan struct{}
}
@@ -199,7 +199,7 @@ func (a *typingHoldAdapter) SendTyping(context.Context, string) error {
}
type cancelPublishBotController struct {
botController
stubBotController
closeEntered chan struct{} // one send when Close begins
closeHold chan struct{} // Close parks here, pinning Stop inside the closeSessions loop
turnCtx chan error // ctx.Err() observed on RunTurn entry
+5 -5
View File
@@ -156,7 +156,7 @@ func (f *fakeReactionAdapter) cleanupMessages() []string {
}
type queueTestController struct {
botController
stubBotController
mu sync.Mutex
steers []string
rejectSteer bool
@@ -201,7 +201,7 @@ func (c *queueTestController) wasCanceled() bool {
}
type rotatingBotController struct {
botController
stubBotController
path string
newPath string
newCalls int
@@ -218,7 +218,7 @@ func (c *rotatingBotController) SessionPath() string { return c.path }
func (c *rotatingBotController) Close() { c.closed = true }
type runtimeStatusBotController struct {
botController
stubBotController
status control.RuntimeStatus
workspaceRoot string
sessionPath string
@@ -231,7 +231,7 @@ func (c *runtimeStatusBotController) SessionPath() string { ret
func (c *runtimeStatusBotController) Close() { c.closed = true }
type blockingApprovalController struct {
botController
stubBotController
emit func(event.Event)
emitted chan struct{}
approved chan struct{}
@@ -256,7 +256,7 @@ func (c *blockingApprovalController) Approve(id string, allow, session, persist
}
type blockingAskController struct {
botController
stubBotController
emit func(event.Event)
emitted chan struct{}
answered chan []event.AskAnswer
+2 -17
View File
@@ -119,16 +119,10 @@ func botSessionHasActiveWork(state *sessionState) bool {
return status.Running || status.PendingPrompt || status.BackgroundJobs > 0
}
func safeBotControllerRuntimeStatus(ctrl botController) (status control.RuntimeStatus, ok bool) {
func safeBotControllerRuntimeStatus(ctrl botController) (control.RuntimeStatus, bool) {
if ctrl == nil {
return control.RuntimeStatus{}, false
}
defer func() {
if recover() != nil {
status = control.RuntimeStatus{}
ok = false
}
}()
return ctrl.RuntimeStatus(), true
}
@@ -393,16 +387,10 @@ func sessionStateMatchesRuntime(state *sessionState, profile sessionRuntimeProfi
return true
}
func safeBotControllerWorkspaceRoot(ctrl botController) (root string, ok bool) {
func safeBotControllerWorkspaceRoot(ctrl botController) (string, bool) {
if ctrl == nil {
return "", false
}
defer func() {
if recover() != nil {
root = ""
ok = false
}
}()
return ctrl.WorkspaceRoot(), true
}
@@ -410,9 +398,6 @@ func safeBotSetToolApprovalMode(ctrl botController, mode string) {
if ctrl == nil {
return
}
defer func() {
_ = recover()
}()
ctrl.SetToolApprovalMode(mode)
}
+4 -4
View File
@@ -15,10 +15,10 @@ import (
// RunTurn until Approve is called. Every other method is a harmless stub so the
// gateway's turn/approve path can drive it without a real controller.
type approvalBlockingController struct {
botController // embedded nil interface: unused methods panic if ever called
started chan struct{}
released chan struct{}
approved chan struct{}
stubBotController
started chan struct{}
released chan struct{}
approved chan struct{}
}
func newApprovalBlockingController() *approvalBlockingController {