mirror of
https://github.com/Twigpine/zero.git
synced 2026-11-08 01:09:46 -10:00
Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
eb67043095 |
No files matched your search
@@ -25,7 +25,7 @@ var ErrStreamIdle = errors.New("idle timeout (upstream stopped sending data)")
|
||||
|
||||
// ErrStreamStalled reports that a streaming upstream kept the connection alive
|
||||
// (SSE keep-alives reset the idle watchdog, so it never fired) but produced no
|
||||
// actual output for streamContentStallFactor × the idle timeout. Without this an
|
||||
// actual output for StreamContentStallFactor × the idle timeout. Without this an
|
||||
// upstream that heartbeats-but-stalls — observed on chatgpt/gpt-5.x and ollama
|
||||
// reasoning models — would hang the agent indefinitely.
|
||||
var ErrStreamStalled = errors.New("stream stalled (upstream kept the connection alive but produced no output)")
|
||||
@@ -38,7 +38,7 @@ var ErrStreamStalled = errors.New("stream stalled (upstream kept the connection
|
||||
// stuck or very slow.
|
||||
func StreamTimeoutMessage(err error, idleTimeout time.Duration) string {
|
||||
if errors.Is(err, ErrStreamStalled) {
|
||||
return fmt.Sprintf("no output for %s (the model kept the connection alive but produced nothing — it may be stuck; try a faster model or lower reasoning effort)", idleTimeout*streamContentStallFactor)
|
||||
return fmt.Sprintf("no output for %s (the model kept the connection alive but produced nothing — it may be stuck; try a faster model or lower reasoning effort)", idleTimeout*StreamContentStallFactor)
|
||||
}
|
||||
return fmt.Sprintf("idle timeout after %s (upstream stopped sending data)", idleTimeout)
|
||||
}
|
||||
@@ -53,13 +53,13 @@ func StreamTimeoutMessage(err error, idleTimeout time.Duration) string {
|
||||
// Override globally with ZERO_STREAM_IDLE_TIMEOUT.
|
||||
const DefaultStreamIdleTimeout = 5 * time.Minute
|
||||
|
||||
// streamContentStallFactor bounds a heartbeat-but-no-output stream. Keep-alives
|
||||
// StreamContentStallFactor bounds a heartbeat-but-no-output stream. Keep-alives
|
||||
// reset the idle watchdog (a heartbeating upstream is not "dead"), but if NO real
|
||||
// data line arrives for streamContentStallFactor × the idle timeout, the stream is
|
||||
// data line arrives for StreamContentStallFactor × the idle timeout, the stream is
|
||||
// treated as stalled and aborted (ErrStreamStalled). It is > 1 so a slow-but-
|
||||
// producing request — one that emits data between heartbeats — is never killed; the
|
||||
// content watchdog only catches a stream that heartbeats while producing nothing.
|
||||
const streamContentStallFactor = 2
|
||||
const StreamContentStallFactor = 2
|
||||
|
||||
// streamIdleTimeoutEnv is the global override for the stream idle timeout. It
|
||||
// accepts a Go duration ("5m", "300s", "90s") or a bare number of seconds
|
||||
@@ -285,7 +285,7 @@ func ScanSSEDataWithContext(
|
||||
// Content watchdog: only real data lines reset it (keep-alives do not), so a
|
||||
// stream that heartbeats without producing output is bounded instead of
|
||||
// hanging forever.
|
||||
contentTimeout := idleTimeout * streamContentStallFactor
|
||||
contentTimeout := idleTimeout * StreamContentStallFactor
|
||||
content := time.NewTimer(contentTimeout)
|
||||
defer content.Stop()
|
||||
contentC = content.C
|
||||
|
||||
@@ -205,7 +205,7 @@ func TestUpstreamUnreachable(t *testing.T) {
|
||||
|
||||
// A heartbeating-but-output-less upstream must not hang forever: SSE keep-alives
|
||||
// feed the idle watchdog (so it never fires), but the content watchdog
|
||||
// (idleTimeout × streamContentStallFactor) aborts with ErrStreamStalled when no
|
||||
// (idleTimeout × StreamContentStallFactor) aborts with ErrStreamStalled when no
|
||||
// real data line arrives. This is the gpt-5.x / ollama "still generating forever"
|
||||
// hang.
|
||||
func TestScanSSEDataWithContextAbortsOnContentStall(t *testing.T) {
|
||||
|
||||
@@ -25,6 +25,7 @@ import (
|
||||
"github.com/Gitlawb/zero/internal/notify"
|
||||
"github.com/Gitlawb/zero/internal/providerhealth"
|
||||
"github.com/Gitlawb/zero/internal/providermodeldiscovery"
|
||||
"github.com/Gitlawb/zero/internal/providers/providerio"
|
||||
"github.com/Gitlawb/zero/internal/sandbox"
|
||||
"github.com/Gitlawb/zero/internal/sessions"
|
||||
"github.com/Gitlawb/zero/internal/streamjson"
|
||||
@@ -2806,6 +2807,15 @@ const quietWorkingHint = 8 * time.Second
|
||||
// quietGenerationHint returns a "still generating…" cue with an advancing
|
||||
// quiet-timer when the active turn has produced no streamed output for a while,
|
||||
// else "". The advancing number is itself the liveness signal.
|
||||
//
|
||||
// Past half the provider's idle timeout, the cue escalates to name what's
|
||||
// actually happening and when Zero will act on its own: a heartbeating-but-
|
||||
// silent stream (observed on chatgpt/gpt-5.x and ollama reasoning models,
|
||||
// see providerio.ErrStreamStalled) is bounded by the content-stall watchdog at
|
||||
// idleTimeout × providerio.StreamContentStallFactor, but until it fires this
|
||||
// exact same plain "still generating… Xs" text is indistinguishable from a
|
||||
// genuine hang — the ticking number was the only signal, and it looks
|
||||
// identical whether real (if slow) content is coming or nothing ever will.
|
||||
func (m model) quietGenerationHint() string {
|
||||
if m.activeRunID == 0 {
|
||||
return ""
|
||||
@@ -2821,6 +2831,10 @@ func (m model) quietGenerationHint() string {
|
||||
if quiet < quietWorkingHint {
|
||||
return ""
|
||||
}
|
||||
if idleTimeout := providerio.ResolveStreamIdleTimeout(0); idleTimeout > 0 && quiet >= idleTimeout/2 {
|
||||
ceiling := idleTimeout * providerio.StreamContentStallFactor
|
||||
return fmt.Sprintf("still generating… %s — unusually quiet, Zero will auto-recover by ~%s if it doesn't resume", formatWorkingElapsed(quiet), formatWorkingElapsed(ceiling))
|
||||
}
|
||||
return "still generating… " + formatWorkingElapsed(quiet)
|
||||
}
|
||||
|
||||
|
||||
@@ -106,6 +106,46 @@ func TestQuietGenerationHint(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestQuietGenerationHintEscalatesPastHalfIdleTimeout: a heartbeating-but-
|
||||
// silent stream (chatgpt/gpt-5.x, ollama reasoning models — see
|
||||
// providerio.ErrStreamStalled) and a genuinely healthy-but-slow one look
|
||||
// identical under the plain "still generating… Xs" cue — the ticking number
|
||||
// is the only signal, whether real content is still coming or nothing ever
|
||||
// will. Past half the provider's idle timeout the cue must say so explicitly
|
||||
// and name when Zero's own content-stall watchdog will act, rather than
|
||||
// leaving the user to guess whether this is a hang.
|
||||
func TestQuietGenerationHintEscalatesPastHalfIdleTimeout(t *testing.T) {
|
||||
// 30s idle timeout: half (15s) sits comfortably above quietWorkingHint (8s),
|
||||
// leaving a clean window to observe the plain cue before it escalates.
|
||||
t.Setenv("ZERO_STREAM_IDLE_TIMEOUT", "30s")
|
||||
base := time.Date(2026, 6, 25, 12, 0, 0, 0, time.UTC)
|
||||
m := model{now: func() time.Time { return base }}
|
||||
m.activeRunID = 7
|
||||
m.turnStartedAt = base
|
||||
|
||||
// Quiet past quietWorkingHint but under half the (30s) idle timeout -> the
|
||||
// plain cue, same as TestQuietGenerationHint's normal case.
|
||||
m.lastStreamActivity = base.Add(-10 * time.Second)
|
||||
got := m.quietGenerationHint()
|
||||
if !strings.Contains(got, "still generating") {
|
||||
t.Fatalf("under half the idle timeout: want the plain cue, got %q", got)
|
||||
}
|
||||
if strings.Contains(got, "auto-recover") {
|
||||
t.Fatalf("under half the idle timeout: should not have escalated yet, got %q", got)
|
||||
}
|
||||
|
||||
// Quiet for >= half the idle timeout -> escalates to name the watchdog and
|
||||
// its ceiling (idleTimeout * providerio.StreamContentStallFactor = 1m00s here).
|
||||
m.lastStreamActivity = base.Add(-15 * time.Second)
|
||||
got = m.quietGenerationHint()
|
||||
if !strings.Contains(got, "unusually quiet") || !strings.Contains(got, "auto-recover") {
|
||||
t.Fatalf("past half the idle timeout: want an escalated cue naming the watchdog, got %q", got)
|
||||
}
|
||||
if !strings.Contains(got, "1m00s") {
|
||||
t.Fatalf("escalated cue should name the content-stall ceiling (idleTimeout * StreamContentStallFactor = 1m00s), got %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestQuietHintHiddenWhenSidebarShowsIt: when the context sidebar is up (it
|
||||
// carries the "generating…" pulse in ACTIVITY), the working line must NOT also
|
||||
// show the hint — it appears in exactly one place.
|
||||
|
||||
Reference in new issue
Block a user