Compare commits

...
Author SHA1 Message Date
Gnanam eb67043095 fix(tui): make the stall watchdog visible before it fires, not just after
Zero already has a content-stall watchdog built specifically for this
exact case (providerio.ErrStreamStalled's own doc comment: "observed on
chatgpt/gpt-5.x and ollama reasoning models") - a heartbeating-but-silent
upstream is aborted after idleTimeout * StreamContentStallFactor (5min *
2 = 10min by default). It works. The gap is that until it fires, the
"still generating... Xs" cue is the ONLY signal the user has, and it
looks completely identical whether real (if slow) content is still
coming or the stream is already dead and just hasn't been caught yet -
confirmed against a live report where the token count sat frozen for
7+ minutes with no way to tell from the UI alone.

Past half the effective idle timeout, quietGenerationHint() now escalates
to name what's actually happening and when Zero will act on its own:
"still generating... Xs - unusually quiet, Zero will auto-recover by
~Ys if it doesn't resume". Exported StreamContentStallFactor from
providerio (was unexported) so the TUI can compute the same ceiling the
watchdog itself uses, instead of duplicating the constant.

New test: TestQuietGenerationHintEscalatesPastHalfIdleTimeout, using
ZERO_STREAM_IDLE_TIMEOUT to shrink the timeout so the test runs fast
while exercising the exact same math the real watchdog uses.
2026-07-01 20:58:14 +05:30
4 changed files with 61 additions and 7 deletions

No files matched your search

+6 -6
View File
@@ -25,7 +25,7 @@ var ErrStreamIdle = errors.New("idle timeout (upstream stopped sending data)")
// ErrStreamStalled reports that a streaming upstream kept the connection alive
// (SSE keep-alives reset the idle watchdog, so it never fired) but produced no
// actual output for streamContentStallFactor × the idle timeout. Without this an
// actual output for StreamContentStallFactor × the idle timeout. Without this an
// upstream that heartbeats-but-stalls — observed on chatgpt/gpt-5.x and ollama
// reasoning models — would hang the agent indefinitely.
var ErrStreamStalled = errors.New("stream stalled (upstream kept the connection alive but produced no output)")
@@ -38,7 +38,7 @@ var ErrStreamStalled = errors.New("stream stalled (upstream kept the connection
// stuck or very slow.
func StreamTimeoutMessage(err error, idleTimeout time.Duration) string {
if errors.Is(err, ErrStreamStalled) {
return fmt.Sprintf("no output for %s (the model kept the connection alive but produced nothing — it may be stuck; try a faster model or lower reasoning effort)", idleTimeout*streamContentStallFactor)
return fmt.Sprintf("no output for %s (the model kept the connection alive but produced nothing — it may be stuck; try a faster model or lower reasoning effort)", idleTimeout*StreamContentStallFactor)
}
return fmt.Sprintf("idle timeout after %s (upstream stopped sending data)", idleTimeout)
}
@@ -53,13 +53,13 @@ func StreamTimeoutMessage(err error, idleTimeout time.Duration) string {
// Override globally with ZERO_STREAM_IDLE_TIMEOUT.
const DefaultStreamIdleTimeout = 5 * time.Minute
// streamContentStallFactor bounds a heartbeat-but-no-output stream. Keep-alives
// StreamContentStallFactor bounds a heartbeat-but-no-output stream. Keep-alives
// reset the idle watchdog (a heartbeating upstream is not "dead"), but if NO real
// data line arrives for streamContentStallFactor × the idle timeout, the stream is
// data line arrives for StreamContentStallFactor × the idle timeout, the stream is
// treated as stalled and aborted (ErrStreamStalled). It is > 1 so a slow-but-
// producing request — one that emits data between heartbeats — is never killed; the
// content watchdog only catches a stream that heartbeats while producing nothing.
const streamContentStallFactor = 2
const StreamContentStallFactor = 2
// streamIdleTimeoutEnv is the global override for the stream idle timeout. It
// accepts a Go duration ("5m", "300s", "90s") or a bare number of seconds
@@ -285,7 +285,7 @@ func ScanSSEDataWithContext(
// Content watchdog: only real data lines reset it (keep-alives do not), so a
// stream that heartbeats without producing output is bounded instead of
// hanging forever.
contentTimeout := idleTimeout * streamContentStallFactor
contentTimeout := idleTimeout * StreamContentStallFactor
content := time.NewTimer(contentTimeout)
defer content.Stop()
contentC = content.C
@@ -205,7 +205,7 @@ func TestUpstreamUnreachable(t *testing.T) {
// A heartbeating-but-output-less upstream must not hang forever: SSE keep-alives
// feed the idle watchdog (so it never fires), but the content watchdog
// (idleTimeout × streamContentStallFactor) aborts with ErrStreamStalled when no
// (idleTimeout × StreamContentStallFactor) aborts with ErrStreamStalled when no
// real data line arrives. This is the gpt-5.x / ollama "still generating forever"
// hang.
func TestScanSSEDataWithContextAbortsOnContentStall(t *testing.T) {
+14
View File
@@ -25,6 +25,7 @@ import (
"github.com/Gitlawb/zero/internal/notify"
"github.com/Gitlawb/zero/internal/providerhealth"
"github.com/Gitlawb/zero/internal/providermodeldiscovery"
"github.com/Gitlawb/zero/internal/providers/providerio"
"github.com/Gitlawb/zero/internal/sandbox"
"github.com/Gitlawb/zero/internal/sessions"
"github.com/Gitlawb/zero/internal/streamjson"
@@ -2806,6 +2807,15 @@ const quietWorkingHint = 8 * time.Second
// quietGenerationHint returns a "still generating…" cue with an advancing
// quiet-timer when the active turn has produced no streamed output for a while,
// else "". The advancing number is itself the liveness signal.
//
// Past half the provider's idle timeout, the cue escalates to name what's
// actually happening and when Zero will act on its own: a heartbeating-but-
// silent stream (observed on chatgpt/gpt-5.x and ollama reasoning models,
// see providerio.ErrStreamStalled) is bounded by the content-stall watchdog at
// idleTimeout × providerio.StreamContentStallFactor, but until it fires this
// exact same plain "still generating… Xs" text is indistinguishable from a
// genuine hang — the ticking number was the only signal, and it looks
// identical whether real (if slow) content is coming or nothing ever will.
func (m model) quietGenerationHint() string {
if m.activeRunID == 0 {
return ""
@@ -2821,6 +2831,10 @@ func (m model) quietGenerationHint() string {
if quiet < quietWorkingHint {
return ""
}
if idleTimeout := providerio.ResolveStreamIdleTimeout(0); idleTimeout > 0 && quiet >= idleTimeout/2 {
ceiling := idleTimeout * providerio.StreamContentStallFactor
return fmt.Sprintf("still generating… %s — unusually quiet, Zero will auto-recover by ~%s if it doesn't resume", formatWorkingElapsed(quiet), formatWorkingElapsed(ceiling))
}
return "still generating… " + formatWorkingElapsed(quiet)
}
+40
View File
@@ -106,6 +106,46 @@ func TestQuietGenerationHint(t *testing.T) {
}
}
// TestQuietGenerationHintEscalatesPastHalfIdleTimeout: a heartbeating-but-
// silent stream (chatgpt/gpt-5.x, ollama reasoning models — see
// providerio.ErrStreamStalled) and a genuinely healthy-but-slow one look
// identical under the plain "still generating… Xs" cue — the ticking number
// is the only signal, whether real content is still coming or nothing ever
// will. Past half the provider's idle timeout the cue must say so explicitly
// and name when Zero's own content-stall watchdog will act, rather than
// leaving the user to guess whether this is a hang.
func TestQuietGenerationHintEscalatesPastHalfIdleTimeout(t *testing.T) {
// 30s idle timeout: half (15s) sits comfortably above quietWorkingHint (8s),
// leaving a clean window to observe the plain cue before it escalates.
t.Setenv("ZERO_STREAM_IDLE_TIMEOUT", "30s")
base := time.Date(2026, 6, 25, 12, 0, 0, 0, time.UTC)
m := model{now: func() time.Time { return base }}
m.activeRunID = 7
m.turnStartedAt = base
// Quiet past quietWorkingHint but under half the (30s) idle timeout -> the
// plain cue, same as TestQuietGenerationHint's normal case.
m.lastStreamActivity = base.Add(-10 * time.Second)
got := m.quietGenerationHint()
if !strings.Contains(got, "still generating") {
t.Fatalf("under half the idle timeout: want the plain cue, got %q", got)
}
if strings.Contains(got, "auto-recover") {
t.Fatalf("under half the idle timeout: should not have escalated yet, got %q", got)
}
// Quiet for >= half the idle timeout -> escalates to name the watchdog and
// its ceiling (idleTimeout * providerio.StreamContentStallFactor = 1m00s here).
m.lastStreamActivity = base.Add(-15 * time.Second)
got = m.quietGenerationHint()
if !strings.Contains(got, "unusually quiet") || !strings.Contains(got, "auto-recover") {
t.Fatalf("past half the idle timeout: want an escalated cue naming the watchdog, got %q", got)
}
if !strings.Contains(got, "1m00s") {
t.Fatalf("escalated cue should name the content-stall ceiling (idleTimeout * StreamContentStallFactor = 1m00s), got %q", got)
}
}
// TestQuietHintHiddenWhenSidebarShowsIt: when the context sidebar is up (it
// carries the "generating…" pulse in ACTIVITY), the working line must NOT also
// show the hint — it appears in exactly one place.