mirror of
https://github.com/Twigpine/zero.git
synced 2026-11-08 09:49:34 -10:00
Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
f40da4ee88 |
No files matched your search
@@ -54,10 +54,8 @@ jobs:
|
||||
# is published under the pushed git tag. If they disagree, install.sh/install.ps1
|
||||
# and the npm postinstall all build a download URL that 404s. Fail fast so a
|
||||
# forgotten version bump can never publish mis-versioned, un-installable assets.
|
||||
# Matches on the ref (not event_name) because release-please dispatches this
|
||||
# workflow at the tag ref — its GITHUB_TOKEN-created tag cannot fire `push`.
|
||||
- name: Check tag matches package.json version
|
||||
if: ${{ startsWith(github.ref, 'refs/tags/v') }}
|
||||
if: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') }}
|
||||
shell: bash
|
||||
run: |
|
||||
pkg_version="$(node -p "require('./package.json').version")"
|
||||
@@ -80,15 +78,13 @@ jobs:
|
||||
path: dist/release/*
|
||||
if-no-files-found: error
|
||||
|
||||
# On a `v*` tag ref (pushed manually, or dispatched at the tag by the
|
||||
# release-please workflow), publish the packaged archive + .sha256 as GitHub
|
||||
# Release assets so the documented `scripts/install.sh` / `install.ps1` path
|
||||
# (which reads /releases/latest) works. Uses the preinstalled `gh` (no new
|
||||
# action dependency); each matrix OS creates the release if absent
|
||||
# (best-effort — release-please normally creates it first) then uploads its
|
||||
# own assets with --clobber. Tags come from merging the release-please PR.
|
||||
# On a `v*` tag push, publish the packaged archive + .sha256 as GitHub Release
|
||||
# assets so the documented `scripts/install.sh` / `install.ps1` path (which reads
|
||||
# /releases/latest) works. Uses the preinstalled `gh` (no new action dependency);
|
||||
# each matrix OS creates the release if absent (best-effort) then uploads its own
|
||||
# assets with --clobber. NOTE: maintainers must push the tag — this never tags.
|
||||
- name: Publish to GitHub Release
|
||||
if: ${{ startsWith(github.ref, 'refs/tags/v') }}
|
||||
if: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') }}
|
||||
shell: bash
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
@@ -96,83 +92,3 @@ jobs:
|
||||
tag="${GITHUB_REF_NAME}"
|
||||
gh release create "$tag" --repo "$GITHUB_REPOSITORY" --title "$tag" --generate-notes 2>/dev/null || true
|
||||
gh release upload "$tag" dist/release/* --repo "$GITHUB_REPOSITORY" --clobber
|
||||
|
||||
# Publishes the npm wrapper (@gitlawb/zero) once every platform's release
|
||||
# assets are live — the package's postinstall downloads its binary from the
|
||||
# GitHub Release, so publishing before the assets exist would ship a package
|
||||
# that fails to install.
|
||||
#
|
||||
# Auth is npm OIDC trusted publishing: no NPM_TOKEN secret. The package's
|
||||
# npmjs.com settings must list this repo + workflow file as a trusted
|
||||
# publisher, and provenance is then generated automatically. Note npm only
|
||||
# allows configuring a trusted publisher on an EXISTING package, so the very
|
||||
# first @gitlawb/zero publish is a one-time manual `npm publish` bootstrap.
|
||||
publish-npm:
|
||||
name: Publish npm package
|
||||
needs: package
|
||||
if: ${{ startsWith(github.ref, 'refs/tags/v') }}
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
id-token: write # OIDC token exchange for npm trusted publishing + provenance
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
|
||||
|
||||
# No registry-url here on purpose: setup-node's registry-url writes an
|
||||
# .npmrc line referencing $NODE_AUTH_TOKEN, and npm hard-fails on the
|
||||
# unset env var. Trusted publishing needs no token at all.
|
||||
- name: Setup Node
|
||||
uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
|
||||
with:
|
||||
node-version: 24
|
||||
|
||||
# OIDC trusted publishing needs npm >= 11.5.1. Use the npm bundled with
|
||||
# the pinned Node toolchain (Node 24 ships >= 11.5.1) rather than pulling
|
||||
# npm@latest from the registry inside the publish path — an unpinned
|
||||
# install there would be avoidable supply-chain exposure. Fail fast if a
|
||||
# future runner/toolchain change ever regresses the bundled npm.
|
||||
- name: Check npm supports trusted publishing
|
||||
run: |
|
||||
npm --version | node -e '
|
||||
const version = require("fs").readFileSync(0, "utf8").trim();
|
||||
const [major, minor, patch] = version.split(".").map(Number);
|
||||
const ok =
|
||||
major > 11 ||
|
||||
(major === 11 && (minor > 5 || (minor === 5 && patch >= 1)));
|
||||
if (!ok) {
|
||||
console.error(
|
||||
`::error::bundled npm ${version} is too old for OIDC trusted ` +
|
||||
`publishing (needs >= 11.5.1) — bump node-version in this job.`,
|
||||
);
|
||||
process.exit(1);
|
||||
}
|
||||
console.log(`npm ${version} supports trusted publishing.`);
|
||||
'
|
||||
|
||||
# Probe the exact URLs postinstall will build (internal/release/release.go
|
||||
# asset-name scheme) as an anonymous client. This also blocks publishing
|
||||
# while the repo is still private — private release assets 404 without
|
||||
# auth, exactly as they would for an installing user.
|
||||
- name: Verify release assets are downloadable
|
||||
shell: bash
|
||||
run: |
|
||||
version="$(node -p "require('./package.json').version")"
|
||||
base="https://github.com/${GITHUB_REPOSITORY}/releases/download/v${version}"
|
||||
for asset in \
|
||||
"zero-v${version}-linux-x64.tar.gz" \
|
||||
"zero-v${version}-linux-arm64.tar.gz" \
|
||||
"zero-v${version}-macos-x64.tar.gz" \
|
||||
"zero-v${version}-macos-arm64.tar.gz" \
|
||||
"zero-v${version}-windows-x64.zip"; do
|
||||
for url in "${base}/${asset}" "${base}/${asset}.sha256"; do
|
||||
if ! curl -fsSLI -o /dev/null "$url"; then
|
||||
echo "::error::${url} is not publicly downloadable — npm postinstall would fail. Is the repo public and are all assets uploaded?" >&2
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
done
|
||||
echo "all release assets for v${version} are publicly downloadable."
|
||||
|
||||
- name: Publish to npm
|
||||
run: npm publish --access public
|
||||
@@ -1,35 +0,0 @@
|
||||
name: Release Please
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
|
||||
permissions:
|
||||
contents: write # push the release-PR branch, create the tag + GitHub Release
|
||||
pull-requests: write # open/update the release PR
|
||||
issues: write # create/apply release-tracking labels on the release PR
|
||||
actions: write # dispatch release-artifacts.yml once a release is cut
|
||||
|
||||
jobs:
|
||||
release-please:
|
||||
name: Release PR / tag
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Run release-please
|
||||
id: release
|
||||
uses: googleapis/release-please-action@a02a34c4d625f9be7cb89156071d8567266a2445 # v4.2.0
|
||||
|
||||
# Tags created with the default GITHUB_TOKEN never fire `on: push: tags`
|
||||
# workflows (GitHub suppresses workflows triggered by workflow-created
|
||||
# events), so release-artifacts.yml would not run for a release-please
|
||||
# tag. workflow_dispatch invoked through the API is exempt from that
|
||||
# suppression — dispatch the artifacts build at the new tag explicitly.
|
||||
- name: Build and publish release artifacts
|
||||
if: ${{ steps.release.outputs.release_created }}
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
run: |
|
||||
gh workflow run release-artifacts.yml \
|
||||
--repo "$GITHUB_REPOSITORY" \
|
||||
--ref "${{ steps.release.outputs.tag_name }}"
|
||||
@@ -1 +0,0 @@
|
||||
{}
|
||||
+21
-36
@@ -210,29 +210,27 @@ func Run(ctx context.Context, prompt string, provider Provider, options Options)
|
||||
}
|
||||
}
|
||||
|
||||
// forwardedVisibleText flags forwarded final PROSE (OnText) specifically —
|
||||
// the only output whose re-stream on a stall retry would visibly DUPLICATE
|
||||
// answer text the user already read. Transient working indicators
|
||||
// (reasoning previews, tool-call "writing…" previews) are deliberately NOT
|
||||
// counted: a turn that streamed only those then stalled — the common
|
||||
// gpt-5.x / ollama "froze mid-write_file" case — is safe to re-issue, since
|
||||
// the TUI resets the tool-call preview on the next tool-call-start and a
|
||||
// stalled turn is never appended to `messages` (it returns on the error
|
||||
// before the append), so the retry re-sends clean context with no
|
||||
// conversation-state duplication.
|
||||
forwardedVisibleText := false
|
||||
// Wrapped callbacks flag ANY output forwarded this turn, so a stall retry
|
||||
// never re-streams on top of output the user already saw — even across the
|
||||
// reactive-compaction retry below, which overwrites `collected`.
|
||||
forwardedAnything := false
|
||||
mark := func() { forwardedAnything = true }
|
||||
// OnUsage is telemetry, not visible output, so it does NOT mark
|
||||
// forwardedAnything: a provider that emits an early usage event (e.g. prompt
|
||||
// tokens) then stalls with no content must still be safely retried. The gate
|
||||
// only guards against re-streaming output the user actually SAW.
|
||||
forwardingOpts := zeroruntime.CollectOptions{OnUsage: options.OnUsage}
|
||||
if options.OnText != nil {
|
||||
forwardingOpts.OnText = func(s string) { forwardedVisibleText = true; options.OnText(s) }
|
||||
forwardingOpts.OnText = func(s string) { mark(); options.OnText(s) }
|
||||
}
|
||||
if options.OnReasoning != nil {
|
||||
forwardingOpts.OnReasoning = func(s string) { options.OnReasoning(s) }
|
||||
forwardingOpts.OnReasoning = func(s string) { mark(); options.OnReasoning(s) }
|
||||
}
|
||||
if options.OnToolCallStart != nil {
|
||||
forwardingOpts.OnToolCallStart = func(id, name string) { options.OnToolCallStart(id, name) }
|
||||
forwardingOpts.OnToolCallStart = func(id, name string) { mark(); options.OnToolCallStart(id, name) }
|
||||
}
|
||||
if options.OnToolCallDelta != nil {
|
||||
forwardingOpts.OnToolCallDelta = func(id, fragment string) { options.OnToolCallDelta(id, fragment) }
|
||||
forwardingOpts.OnToolCallDelta = func(id, fragment string) { mark(); options.OnToolCallDelta(id, fragment) }
|
||||
}
|
||||
// recoverStreamError applies the same non-stall recovery the initial stream
|
||||
// gets to ANY collected error — including one from a reissued (stall-retry)
|
||||
@@ -286,28 +284,15 @@ func Run(ctx context.Context, prompt string, provider Provider, options Options)
|
||||
result.Messages = copyMessages(messages)
|
||||
return result, ctx.Err()
|
||||
}
|
||||
// A stream idle/stall timeout is safely re-issued when the turn committed NO
|
||||
// answer text — no forwarded visible prose (forwardedVisibleText) and no
|
||||
// collected final text (collected.Text). This covers two cases:
|
||||
// 1. Nothing streamed at all before the connection died (the original
|
||||
// macOS stale-pooled-connection hang past the response-header timeout).
|
||||
// 2. The model streamed transient reasoning and began a tool call (e.g. a
|
||||
// large write_file) then froze mid-arguments — the common gpt-5.x /
|
||||
// ollama heartbeat-pause stall. That partial tool call is NEVER executed
|
||||
// (a turn with collected.Error returns before dispatch below) and NEVER
|
||||
// appended to `messages`, so a retry re-issues clean context; the only
|
||||
// re-render is transient previews, not duplicated answer text. This is
|
||||
// why the gate no longer excludes collected.ToolCalls: an incomplete
|
||||
// tool call from a timed-out stream is discard-and-retry, not output.
|
||||
// A turn that forwarded real prose is NOT retried (it would duplicate visible
|
||||
// answer text) and falls through to the error return below. Capped +
|
||||
// exponential backoff, with a user-visible notice per attempt.
|
||||
// A stream idle/stall timeout with NO output forwarded yet this turn can be
|
||||
// safely re-issued on a fresh connection — nothing was shown, so there's no
|
||||
// duplication (this recovers the macOS stale-pooled-connection hang when it
|
||||
// slips past the transport's response-header timeout). A turn that already
|
||||
// streamed partial text/tool-calls is NOT retried (it would duplicate) and
|
||||
// falls through to the error return below. Capped + exponential backoff.
|
||||
for attempt := 1; attempt <= maxStreamStallRetries &&
|
||||
isStreamTimeoutError(collected.Error) && !forwardedVisibleText &&
|
||||
collected.Text == ""; attempt++ {
|
||||
if notify := stallRetryNoticeFor(options); notify != nil {
|
||||
notify(attempt, maxStreamStallRetries)
|
||||
}
|
||||
isStreamTimeoutError(collected.Error) && !forwardedAnything &&
|
||||
collected.Text == "" && len(collected.ToolCalls) == 0; attempt++ {
|
||||
if err := sleepWithContext(ctx, backoffFor(attempt)); err != nil {
|
||||
result.Messages = copyMessages(messages)
|
||||
return result, err
|
||||
|
||||
@@ -40,21 +40,6 @@ func reconnectNoticeFor(options Options) reconnectNotifier {
|
||||
}
|
||||
}
|
||||
|
||||
// stallRetryNoticeFor builds a notifier for the loop's content-stall retry (a
|
||||
// stream that connected and produced only transient output, then went silent).
|
||||
// Distinct wording from reconnectNoticeFor's "connection lost" because the
|
||||
// connection was fine — the model stalled. Surfaced through OnReasoning, the
|
||||
// non-content channel, so it never folds into the answer text. Nil when there
|
||||
// is no reasoning sink (the retry still happens silently).
|
||||
func stallRetryNoticeFor(options Options) reconnectNotifier {
|
||||
if options.OnReasoning == nil {
|
||||
return nil
|
||||
}
|
||||
return func(attempt, max int) {
|
||||
options.OnReasoning(fmt.Sprintf("\n[no output — model stalled; retrying %d/%d…]\n", attempt, max))
|
||||
}
|
||||
}
|
||||
|
||||
// streamWithReconnect issues request via provider.StreamCompletion and, on a
|
||||
// transient disconnect error, retries the connect up to maxStreamReconnects
|
||||
// times with exponential backoff. It returns the live stream on success, or the
|
||||
|
||||
@@ -10,25 +10,19 @@ import (
|
||||
)
|
||||
|
||||
// stallProvider connects successfully (HTTP 200) but the stream emits a stall/idle
|
||||
// timeout error for the first stallBefore calls, then succeeds with "done". The
|
||||
// pre-stall emission simulates different partial-output shapes:
|
||||
// - partialText: forwarded PROSE (StreamEventText) — must NOT be retried
|
||||
// (re-issuing would duplicate visible answer text).
|
||||
// - reasoningText: transient reasoning preview — IS retried.
|
||||
// - partialToolCall: a tool call started (+ arg delta) but never ended, the
|
||||
// "froze mid-write_file" case — IS retried (the incomplete call is never
|
||||
// executed or committed).
|
||||
// timeout error for the first stallBefore calls, then succeeds with "done". When
|
||||
// partialText is set it emits that text BEFORE the stall, simulating a stall after
|
||||
// partial output (which must NOT be retried — re-issuing would duplicate it).
|
||||
type stallProvider struct {
|
||||
calls int32
|
||||
stallBefore int32
|
||||
partialText string
|
||||
reasoningText string
|
||||
partialToolCall string
|
||||
calls int32
|
||||
stallBefore int32
|
||||
partialText string
|
||||
reasoningText string
|
||||
}
|
||||
|
||||
func (p *stallProvider) StreamCompletion(_ context.Context, _ zeroruntime.CompletionRequest) (<-chan zeroruntime.StreamEvent, error) {
|
||||
n := atomic.AddInt32(&p.calls, 1)
|
||||
ch := make(chan zeroruntime.StreamEvent, 5)
|
||||
ch := make(chan zeroruntime.StreamEvent, 3)
|
||||
if n <= p.stallBefore {
|
||||
if p.partialText != "" {
|
||||
ch <- zeroruntime.StreamEvent{Type: zeroruntime.StreamEventText, Content: p.partialText}
|
||||
@@ -36,16 +30,9 @@ func (p *stallProvider) StreamCompletion(_ context.Context, _ zeroruntime.Comple
|
||||
if p.reasoningText != "" {
|
||||
ch <- zeroruntime.StreamEvent{Type: zeroruntime.StreamEventReasoning, Content: p.reasoningText}
|
||||
}
|
||||
if p.partialToolCall != "" {
|
||||
// A tool call that starts and streams a partial argument fragment but
|
||||
// never gets StreamEventToolCallEnd, then the stream errors — exactly
|
||||
// the gpt-5.x "began the big write_file then froze" shape.
|
||||
ch <- zeroruntime.StreamEvent{Type: zeroruntime.StreamEventToolCallStart, ToolCallID: "tc_1", ToolName: p.partialToolCall}
|
||||
ch <- zeroruntime.StreamEvent{Type: zeroruntime.StreamEventToolCallDelta, ToolCallID: "tc_1", ArgumentsFragment: `{"path":"x.html","content":"<!doctype`}
|
||||
}
|
||||
ch <- zeroruntime.StreamEvent{
|
||||
Type: zeroruntime.StreamEventError,
|
||||
Error: "provider stream error: no output for 6m (the model produced nothing)",
|
||||
Error: "provider stream error: no output for 10m (the model produced nothing)",
|
||||
}
|
||||
close(ch)
|
||||
return ch, nil
|
||||
@@ -85,54 +72,21 @@ func TestRunDoesNotRetryStallAfterPartialOutput(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// A stall after only transient REASONING was forwarded (no prose text, no
|
||||
// completed tool call) IS retried: reasoning is a "thinking" preview, not
|
||||
// committed answer text, so re-streaming it on a clearly-signalled retry is
|
||||
// acceptable and recovers the common "reasoned then froze" stall. Only
|
||||
// forwarded final prose (OnText) blocks the retry (collected.Text is empty
|
||||
// here, so this exercises the transient-vs-prose gate specifically).
|
||||
func TestRunRetriesStallAfterOnlyReasoning(t *testing.T) {
|
||||
// A stall after only REASONING was forwarded (no text/tool-calls) must NOT be
|
||||
// retried: the user already saw the reasoning, so re-issuing with callbacks
|
||||
// re-enabled would duplicate it. collected.Text is empty here, so this exercises
|
||||
// the turn-level forwarded-output gate specifically, not the text check.
|
||||
func TestRunDoesNotRetryStallAfterForwardedReasoning(t *testing.T) {
|
||||
p := &stallProvider{stallBefore: 1, reasoningText: "thinking hard"}
|
||||
result, err := Run(context.Background(), "go", p, Options{
|
||||
_, err := Run(context.Background(), "go", p, Options{
|
||||
Registry: tools.NewRegistry(),
|
||||
OnReasoning: func(string) {},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("a stall after only reasoning should retry to success, got %v", err)
|
||||
if err == nil {
|
||||
t.Fatal("a stall after forwarded reasoning must NOT be retried; want an error")
|
||||
}
|
||||
if result.FinalAnswer != "done" {
|
||||
t.Fatalf("final answer = %q, want %q", result.FinalAnswer, "done")
|
||||
}
|
||||
if got := atomic.LoadInt32(&p.calls); got != 2 {
|
||||
t.Fatalf("want 2 calls (1 stall + 1 retry), got %d", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A stall mid-way through an INCOMPLETE tool call (started + partial args, never
|
||||
// ended) IS retried — the exact gpt-5.x / ollama "began the big write_file then
|
||||
// froze" case. The incomplete call is never executed (a turn returns on the
|
||||
// stall error before dispatch) and never committed to history, so re-issuing is
|
||||
// safe and recovers to success. This is the primary scenario this change fixes.
|
||||
func TestRunRetriesStallAfterIncompleteToolCall(t *testing.T) {
|
||||
starts := 0
|
||||
p := &stallProvider{stallBefore: 1, reasoningText: "planning", partialToolCall: "write_file"}
|
||||
result, err := Run(context.Background(), "go", p, Options{
|
||||
Registry: tools.NewRegistry(),
|
||||
OnReasoning: func(string) {},
|
||||
OnToolCallStart: func(string, string) { starts++ },
|
||||
OnToolCallDelta: func(string, string) {},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("a stall mid-incomplete-tool-call should retry to success, got %v", err)
|
||||
}
|
||||
if result.FinalAnswer != "done" {
|
||||
t.Fatalf("final answer = %q, want %q", result.FinalAnswer, "done")
|
||||
}
|
||||
if got := atomic.LoadInt32(&p.calls); got != 2 {
|
||||
t.Fatalf("want 2 calls (1 stall + 1 retry), got %d", got)
|
||||
}
|
||||
if starts != 1 {
|
||||
t.Fatalf("the incomplete tool call should have been forwarded once before the stall, got %d starts", starts)
|
||||
if got := atomic.LoadInt32(&p.calls); got != 1 {
|
||||
t.Fatalf("reasoning-then-stall must not retry, got %d calls", got)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -455,15 +455,11 @@ func TestHarnessTimeoutCancelsBlockedAgent(t *testing.T) {
|
||||
|
||||
// The timeout is longer than materialization of the small fixture even on
|
||||
// slower Windows CI runners, so the agent is reached before it blocks until
|
||||
// the deadline fires. 10s wasn't generous enough in practice — it failed
|
||||
// on an otherwise-unrelated PR merge (materialization alone consumed the
|
||||
// full budget on a contended Windows runner, so the agent was never
|
||||
// reached); widened for headroom, since the assertions below still bound
|
||||
// worst-case test time via that same deadline, not this constant.
|
||||
// the deadline fires.
|
||||
report := harness.Run(context.Background(), suitePath, suite, BenchmarkInput{
|
||||
TaskID: "document-stream-json-verify-events",
|
||||
WorkRoot: t.TempDir(),
|
||||
Timeout: 30 * time.Second,
|
||||
Timeout: 10 * time.Second,
|
||||
})
|
||||
|
||||
if !agentReached {
|
||||
|
||||
@@ -6,12 +6,10 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"math"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"regexp"
|
||||
"runtime"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -27,9 +25,9 @@ var ErrStreamIdle = errors.New("idle timeout (upstream stopped sending data)")
|
||||
|
||||
// ErrStreamStalled reports that a streaming upstream kept the connection alive
|
||||
// (SSE keep-alives reset the idle watchdog, so it never fired) but produced no
|
||||
// actual output for ContentStallTimeout(idle). Without this an upstream that
|
||||
// heartbeats-but-stalls — observed on chatgpt/gpt-5.x and ollama reasoning
|
||||
// models — would hang the agent indefinitely.
|
||||
// actual output for streamContentStallFactor × the idle timeout. Without this an
|
||||
// upstream that heartbeats-but-stalls — observed on chatgpt/gpt-5.x and ollama
|
||||
// reasoning models — would hang the agent indefinitely.
|
||||
var ErrStreamStalled = errors.New("stream stalled (upstream kept the connection alive but produced no output)")
|
||||
|
||||
// StreamTimeoutMessage returns the human-readable detail for a stream-timeout
|
||||
@@ -40,7 +38,7 @@ var ErrStreamStalled = errors.New("stream stalled (upstream kept the connection
|
||||
// stuck or very slow.
|
||||
func StreamTimeoutMessage(err error, idleTimeout time.Duration) string {
|
||||
if errors.Is(err, ErrStreamStalled) {
|
||||
return fmt.Sprintf("no output for %s (the model kept the connection alive but produced nothing — it may be stuck; try a faster model or lower reasoning effort)", ContentStallTimeout(idleTimeout))
|
||||
return fmt.Sprintf("no output for %s (the model kept the connection alive but produced nothing — it may be stuck; try a faster model or lower reasoning effort)", idleTimeout*streamContentStallFactor)
|
||||
}
|
||||
return fmt.Sprintf("idle timeout after %s (upstream stopped sending data)", idleTimeout)
|
||||
}
|
||||
@@ -55,32 +53,13 @@ func StreamTimeoutMessage(err error, idleTimeout time.Duration) string {
|
||||
// Override globally with ZERO_STREAM_IDLE_TIMEOUT.
|
||||
const DefaultStreamIdleTimeout = 5 * time.Minute
|
||||
|
||||
// ContentStallTimeout bounds a heartbeat-but-no-output stream: keep-alives reset
|
||||
// the idle watchdog (a heartbeating upstream is not "dead"), but if NO real data
|
||||
// line arrives for this long the stream is aborted (ErrStreamStalled). Scaled
|
||||
// above idleTimeout (so a slow-but-producing request that emits data between
|
||||
// heartbeats is never killed — the watchdog only ever fires when NOTHING real
|
||||
// arrives), but only 1.2× rather than the old 2×: a genuine heartbeat-pause
|
||||
// stall on chatgpt/gpt-5.x rarely recovers, so a ~10-minute dead wait (at the 5m
|
||||
// idle default) was a terrible UX for a doomed turn. At the default idle this is
|
||||
// 6 minutes; it still scales with ZERO_STREAM_IDLE_TIMEOUT. A returned value <= 0
|
||||
// (idle watchdog disabled) leaves the content watchdog off too.
|
||||
func ContentStallTimeout(idleTimeout time.Duration) time.Duration {
|
||||
if idleTimeout <= 0 {
|
||||
return 0
|
||||
}
|
||||
// 1.2× computed as idle + idle/5 (not idle*6/5), plus a clamp: a
|
||||
// pathologically large ZERO_STREAM_IDLE_TIMEOUT could make idle*6 overflow
|
||||
// int64 and wrap to a NEGATIVE duration, which would arm the content timer
|
||||
// to fire immediately and abort every stream. Clamping to the max duration
|
||||
// on overflow just means "effectively no content watchdog" — the sane
|
||||
// result for an absurd idle timeout.
|
||||
extra := idleTimeout / 5
|
||||
if idleTimeout > math.MaxInt64-extra {
|
||||
return math.MaxInt64
|
||||
}
|
||||
return idleTimeout + extra
|
||||
}
|
||||
// streamContentStallFactor bounds a heartbeat-but-no-output stream. Keep-alives
|
||||
// reset the idle watchdog (a heartbeating upstream is not "dead"), but if NO real
|
||||
// data line arrives for streamContentStallFactor × the idle timeout, the stream is
|
||||
// treated as stalled and aborted (ErrStreamStalled). It is > 1 so a slow-but-
|
||||
// producing request — one that emits data between heartbeats — is never killed; the
|
||||
// content watchdog only catches a stream that heartbeats while producing nothing.
|
||||
const streamContentStallFactor = 2
|
||||
|
||||
// streamIdleTimeoutEnv is the global override for the stream idle timeout. It
|
||||
// accepts a Go duration ("5m", "300s", "90s") or a bare number of seconds
|
||||
@@ -143,21 +122,6 @@ func NormalizeBaseURL(baseURL string, defaultBaseURL string, label string) (stri
|
||||
// unaffected (the stream-idle + content watchdogs cover that).
|
||||
// - IdleConnTimeout is shortened so a connection that went idle across a pause
|
||||
// is closed (and re-dialed fresh) rather than reused after it has gone stale.
|
||||
// - DisableKeepAlives on darwin only: ResponseHeaderTimeout/IdleConnTimeout
|
||||
// above catch a reused connection that's fully dead (never responds) or
|
||||
// idle past the timeout, but not one that's alive-but-severely-degraded —
|
||||
// e.g. reused shortly after a prior request on the same host (well within
|
||||
// 30s, common across quick retries of the same turn), where it still
|
||||
// delivers real bytes, just at a crippled rate, resetting Zero's stream
|
||||
// idle/content-stall watchdogs (which only fire on true silence) without
|
||||
// ever recovering. That degraded-not-dead case is indistinguishable from
|
||||
// genuine backend slowness from inside the stream, so the only reliable
|
||||
// fix is removing pooling from the equation entirely on the one platform
|
||||
// where this class of bug has actually reproduced. A fresh TCP+TLS
|
||||
// handshake per request costs low tens of milliseconds — negligible next
|
||||
// to the minutes-long stalls this avoids — and this doesn't touch
|
||||
// Linux/Windows, where the underlying OS doesn't keep dead/degraded
|
||||
// pooled connections around as long.
|
||||
var sharedHTTPClient = func() *http.Client {
|
||||
transport := http.DefaultTransport.(*http.Transport).Clone()
|
||||
// 120s, not 60s: a slow cloud proxy (e.g. ollama `*:cloud`) can withhold its
|
||||
@@ -168,7 +132,6 @@ var sharedHTTPClient = func() *http.Client {
|
||||
// content-stall watchdogs.
|
||||
transport.ResponseHeaderTimeout = 120 * time.Second
|
||||
transport.IdleConnTimeout = 30 * time.Second
|
||||
transport.DisableKeepAlives = runtime.GOOS == "darwin"
|
||||
return &http.Client{Transport: transport}
|
||||
}()
|
||||
|
||||
@@ -322,7 +285,7 @@ func ScanSSEDataWithContext(
|
||||
// Content watchdog: only real data lines reset it (keep-alives do not), so a
|
||||
// stream that heartbeats without producing output is bounded instead of
|
||||
// hanging forever.
|
||||
contentTimeout := ContentStallTimeout(idleTimeout)
|
||||
contentTimeout := idleTimeout * streamContentStallFactor
|
||||
content := time.NewTimer(contentTimeout)
|
||||
defer content.Stop()
|
||||
contentC = content.C
|
||||
|
||||
@@ -4,9 +4,7 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"math"
|
||||
"net/http"
|
||||
"runtime"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -205,36 +203,11 @@ func TestUpstreamUnreachable(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// ContentStallTimeout is 1.2× the idle timeout, disabled (0) when idle is
|
||||
// disabled, and must never overflow to a negative duration for an absurdly
|
||||
// large idle timeout (which would arm the content timer to fire immediately
|
||||
// and abort every stream).
|
||||
func TestContentStallTimeout(t *testing.T) {
|
||||
cases := []struct {
|
||||
idle time.Duration
|
||||
want time.Duration
|
||||
}{
|
||||
{5 * time.Minute, 6 * time.Minute},
|
||||
{30 * time.Second, 36 * time.Second},
|
||||
{100 * time.Millisecond, 120 * time.Millisecond},
|
||||
{0, 0},
|
||||
{-1, 0},
|
||||
}
|
||||
for _, c := range cases {
|
||||
if got := ContentStallTimeout(c.idle); got != c.want {
|
||||
t.Fatalf("ContentStallTimeout(%v) = %v, want %v", c.idle, got, c.want)
|
||||
}
|
||||
}
|
||||
// Overflow guard: idle*6 would wrap negative; the clamp keeps it positive.
|
||||
if got := ContentStallTimeout(math.MaxInt64); got <= 0 {
|
||||
t.Fatalf("ContentStallTimeout(MaxInt64) = %v, must stay positive (no overflow wrap)", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A heartbeating-but-output-less upstream must not hang forever: SSE keep-alives
|
||||
// feed the idle watchdog (so it never fires), but the content watchdog
|
||||
// (ContentStallTimeout(idle)) aborts with ErrStreamStalled when no real data
|
||||
// line arrives. This is the gpt-5.x / ollama "still generating forever" hang.
|
||||
// (idleTimeout × streamContentStallFactor) aborts with ErrStreamStalled when no
|
||||
// real data line arrives. This is the gpt-5.x / ollama "still generating forever"
|
||||
// hang.
|
||||
func TestScanSSEDataWithContextAbortsOnContentStall(t *testing.T) {
|
||||
pr, pw := io.Pipe()
|
||||
defer func() { _ = pw.Close() }()
|
||||
@@ -262,8 +235,7 @@ func TestScanSSEDataWithContextAbortsOnContentStall(t *testing.T) {
|
||||
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
// idle 100ms → content stall at 120ms (ContentStallTimeout = idle*1.2).
|
||||
// Keep-alives reset idle but not content.
|
||||
// idle 100ms → content stall at 200ms. Keep-alives reset idle but not content.
|
||||
done <- ScanSSEDataWithContext(context.Background(), cancel, pr, 100*time.Millisecond, func(string) bool { return true })
|
||||
}()
|
||||
|
||||
@@ -286,7 +258,7 @@ func TestScanSSEDataWithContextContentResetsOnData(t *testing.T) {
|
||||
pr, pw := io.Pipe()
|
||||
|
||||
go func() {
|
||||
// One data line every 30ms for ~300ms (well past the 120ms content window,
|
||||
// One data line every 30ms for ~300ms (well past the 200ms content window,
|
||||
// with comfortable slack under the 100ms idle timeout), so the content
|
||||
// watchdog keeps resetting, then close cleanly.
|
||||
for i := 0; i < 10; i++ {
|
||||
@@ -318,16 +290,16 @@ func TestScanSSEDataWithContextContentResetsOnData(t *testing.T) {
|
||||
}
|
||||
|
||||
// StreamTimeoutMessage must give a stalled stream a distinct, accurate detail —
|
||||
// it reports the content window (ContentStallTimeout) and must NOT claim the
|
||||
// upstream "stopped sending data" (keep-alives were still arriving).
|
||||
// it reports the content window (idle × factor) and must NOT claim the upstream
|
||||
// "stopped sending data" (keep-alives were still arriving).
|
||||
func TestStreamTimeoutMessage(t *testing.T) {
|
||||
idle := 5 * time.Minute
|
||||
if msg := StreamTimeoutMessage(ErrStreamIdle, idle); !strings.Contains(msg, "idle timeout after 5m") {
|
||||
t.Fatalf("idle message = %q, want it to mention the 5m idle timeout", msg)
|
||||
}
|
||||
stalled := StreamTimeoutMessage(ErrStreamStalled, idle)
|
||||
if !strings.Contains(stalled, "no output for 6m") {
|
||||
t.Fatalf("stalled message = %q, want it to report the 6m content window (idle 5m × 1.2)", stalled)
|
||||
if !strings.Contains(stalled, "no output for 10m") {
|
||||
t.Fatalf("stalled message = %q, want it to report the 10m content window", stalled)
|
||||
}
|
||||
if strings.Contains(stalled, "stopped sending data") {
|
||||
t.Fatalf("stalled message must not claim the upstream stopped sending data: %q", stalled)
|
||||
@@ -352,17 +324,6 @@ func TestHTTPClientReturnsStallHardenedSharedClient(t *testing.T) {
|
||||
if tr.IdleConnTimeout != 30*time.Second {
|
||||
t.Fatalf("IdleConnTimeout = %v, want 30s", tr.IdleConnTimeout)
|
||||
}
|
||||
// DisableKeepAlives is the stronger mitigation scoped to darwin only: the
|
||||
// two timeouts above catch a reused connection that's fully dead or idle
|
||||
// too long, but not one that's alive-and-degraded (still delivers real
|
||||
// bytes, just at a crippled rate) — which resets Zero's stream watchdogs
|
||||
// without ever recovering. Removing pooling from the equation entirely is
|
||||
// only worth its reconnect cost on the platform this class of bug has
|
||||
// actually reproduced on.
|
||||
wantDisableKeepAlives := runtime.GOOS == "darwin"
|
||||
if tr.DisableKeepAlives != wantDisableKeepAlives {
|
||||
t.Fatalf("DisableKeepAlives = %v, want %v (GOOS=%s)", tr.DisableKeepAlives, wantDisableKeepAlives, runtime.GOOS)
|
||||
}
|
||||
if HTTPClient(nil) != got {
|
||||
t.Fatal("HTTPClient(nil) must return a shared instance so the conn pool is reused")
|
||||
}
|
||||
|
||||
@@ -124,16 +124,10 @@ func TestOutputToolBlocksUntilTaskCompletes(t *testing.T) {
|
||||
tool := NewOutputTool(manager)
|
||||
tool.PollInterval = time.Millisecond
|
||||
|
||||
// timeout is a ceiling, not a target: the test still returns as soon as the
|
||||
// 5ms write lands (PollInterval keeps it responsive), it just tolerates a
|
||||
// slow/contended CI runner without failing. A 100ms budget against a 5ms
|
||||
// write only left 20x margin, which Windows CI runners (coarser default
|
||||
// timer granularity, heavier scheduling contention than Linux/macOS) blew
|
||||
// through and failed the test on an otherwise-unrelated PR merge.
|
||||
result := tool.Run(context.Background(), map[string]any{
|
||||
"task_id": "child_task",
|
||||
"block": true,
|
||||
"timeout": 2000,
|
||||
"timeout": 100,
|
||||
})
|
||||
|
||||
if result.Status != tools.StatusOK || !strings.Contains(result.Output, "output:\nfinished") {
|
||||
|
||||
+4
-21
@@ -25,7 +25,6 @@ import (
|
||||
"github.com/Gitlawb/zero/internal/notify"
|
||||
"github.com/Gitlawb/zero/internal/providerhealth"
|
||||
"github.com/Gitlawb/zero/internal/providermodeldiscovery"
|
||||
"github.com/Gitlawb/zero/internal/providers/providerio"
|
||||
"github.com/Gitlawb/zero/internal/sandbox"
|
||||
"github.com/Gitlawb/zero/internal/sessions"
|
||||
"github.com/Gitlawb/zero/internal/streamjson"
|
||||
@@ -2235,18 +2234,15 @@ func (m model) footerView(width int) string {
|
||||
footer.WriteString(fitStyledLine(joinHeaderLine(" "+left, right, width), width))
|
||||
}
|
||||
footer.WriteString("\n")
|
||||
// A message typed while a run is active is queued for the next turn; show its
|
||||
// preview directly ABOVE the input box (not below), so it reads as "waiting to
|
||||
// send" sitting on top of what you're currently typing.
|
||||
if queued := renderQueuedMessagePreview(m.queuedMessage, width); queued != "" {
|
||||
footer.WriteString(queued)
|
||||
footer.WriteString("\n")
|
||||
}
|
||||
footer.WriteString(m.composerBox(width))
|
||||
if hint := m.composerDescriptionHint(width); hint != "" {
|
||||
footer.WriteString("\n")
|
||||
footer.WriteString(hint)
|
||||
}
|
||||
if queued := renderQueuedMessagePreview(m.queuedMessage, width); queued != "" {
|
||||
footer.WriteString("\n")
|
||||
footer.WriteString(queued)
|
||||
}
|
||||
footer.WriteString("\n")
|
||||
footer.WriteString(m.statusLine(width))
|
||||
return footer.String()
|
||||
@@ -2822,15 +2818,6 @@ const quietWorkingHint = 8 * time.Second
|
||||
// quietGenerationHint returns a "still generating…" cue with an advancing
|
||||
// quiet-timer when the active turn has produced no streamed output for a while,
|
||||
// else "". The advancing number is itself the liveness signal.
|
||||
//
|
||||
// Past half the provider's idle timeout, the cue escalates to name what's
|
||||
// actually happening and when Zero will act on its own: a heartbeating-but-
|
||||
// silent stream (observed on chatgpt/gpt-5.x and ollama reasoning models,
|
||||
// see providerio.ErrStreamStalled) is bounded by the content-stall watchdog at
|
||||
// providerio.ContentStallTimeout(idle), but until it fires this exact same
|
||||
// plain "still generating… Xs" text is indistinguishable from a genuine hang —
|
||||
// the ticking number was the only signal, and it looks identical whether real
|
||||
// (if slow) content is coming or nothing ever will.
|
||||
func (m model) quietGenerationHint() string {
|
||||
if m.activeRunID == 0 {
|
||||
return ""
|
||||
@@ -2846,10 +2833,6 @@ func (m model) quietGenerationHint() string {
|
||||
if quiet < quietWorkingHint {
|
||||
return ""
|
||||
}
|
||||
if idleTimeout := providerio.ResolveStreamIdleTimeout(0); idleTimeout > 0 && quiet >= idleTimeout/2 {
|
||||
ceiling := providerio.ContentStallTimeout(idleTimeout)
|
||||
return fmt.Sprintf("still generating… %s — unusually quiet, Zero will auto-recover by ~%s if it doesn't resume", formatWorkingElapsed(quiet), formatWorkingElapsed(ceiling))
|
||||
}
|
||||
return "still generating… " + formatWorkingElapsed(quiet)
|
||||
}
|
||||
|
||||
|
||||
@@ -55,38 +55,6 @@ func TestQueuedPromptPreviewAppearsInView(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestQueuedPromptPreviewSitsAboveComposer: the queued-message preview must
|
||||
// render ABOVE the input box, not below it — a message waiting to send should
|
||||
// sit on top of what you're currently typing.
|
||||
func TestQueuedPromptPreviewSitsAboveComposer(t *testing.T) {
|
||||
m := newQueuedMessageTestModel(t)
|
||||
m.pending = true
|
||||
m.activeRunID = 1
|
||||
m.runID = 1
|
||||
m.width = 96
|
||||
m.input.SetValue("summarize the failing test output")
|
||||
updated, _ := m.Update(testKey(tea.KeyEnter))
|
||||
next := updated.(model)
|
||||
|
||||
footer := plainRender(t, next.footerView(96))
|
||||
lines := strings.Split(footer, "\n")
|
||||
queuedLine, composerTopLine := -1, -1
|
||||
for i, ln := range lines {
|
||||
if queuedLine < 0 && strings.Contains(ln, "queued") {
|
||||
queuedLine = i
|
||||
}
|
||||
if composerTopLine < 0 && strings.Contains(ln, "╭") { // composer box top border
|
||||
composerTopLine = i
|
||||
}
|
||||
}
|
||||
if queuedLine < 0 || composerTopLine < 0 {
|
||||
t.Fatalf("expected both a queued line and a composer box; queued=%d composer=%d\n%s", queuedLine, composerTopLine, footer)
|
||||
}
|
||||
if queuedLine >= composerTopLine {
|
||||
t.Fatalf("queued preview (line %d) must sit ABOVE the composer box top border (line %d):\n%s", queuedLine, composerTopLine, footer)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAgentResponseLaunchesQueuedPrompt(t *testing.T) {
|
||||
provider := &scriptedProvider{scripts: [][]zeroruntime.StreamEvent{{
|
||||
{Type: zeroruntime.StreamEventText, Content: "queued answer"},
|
||||
|
||||
@@ -106,46 +106,6 @@ func TestQuietGenerationHint(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestQuietGenerationHintEscalatesPastHalfIdleTimeout: a heartbeating-but-
|
||||
// silent stream (chatgpt/gpt-5.x, ollama reasoning models — see
|
||||
// providerio.ErrStreamStalled) and a genuinely healthy-but-slow one look
|
||||
// identical under the plain "still generating… Xs" cue — the ticking number
|
||||
// is the only signal, whether real content is still coming or nothing ever
|
||||
// will. Past half the provider's idle timeout the cue must say so explicitly
|
||||
// and name when Zero's own content-stall watchdog will act, rather than
|
||||
// leaving the user to guess whether this is a hang.
|
||||
func TestQuietGenerationHintEscalatesPastHalfIdleTimeout(t *testing.T) {
|
||||
// 30s idle timeout: half (15s) sits comfortably above quietWorkingHint (8s),
|
||||
// leaving a clean window to observe the plain cue before it escalates.
|
||||
t.Setenv("ZERO_STREAM_IDLE_TIMEOUT", "30s")
|
||||
base := time.Date(2026, 6, 25, 12, 0, 0, 0, time.UTC)
|
||||
m := model{now: func() time.Time { return base }}
|
||||
m.activeRunID = 7
|
||||
m.turnStartedAt = base
|
||||
|
||||
// Quiet past quietWorkingHint but under half the (30s) idle timeout -> the
|
||||
// plain cue, same as TestQuietGenerationHint's normal case.
|
||||
m.lastStreamActivity = base.Add(-10 * time.Second)
|
||||
got := m.quietGenerationHint()
|
||||
if !strings.Contains(got, "still generating") {
|
||||
t.Fatalf("under half the idle timeout: want the plain cue, got %q", got)
|
||||
}
|
||||
if strings.Contains(got, "auto-recover") {
|
||||
t.Fatalf("under half the idle timeout: should not have escalated yet, got %q", got)
|
||||
}
|
||||
|
||||
// Quiet for >= half the idle timeout -> escalates to name the watchdog and
|
||||
// its ceiling (providerio.ContentStallTimeout(30s) = 30s*1.2 = 36s here).
|
||||
m.lastStreamActivity = base.Add(-15 * time.Second)
|
||||
got = m.quietGenerationHint()
|
||||
if !strings.Contains(got, "unusually quiet") || !strings.Contains(got, "auto-recover") {
|
||||
t.Fatalf("past half the idle timeout: want an escalated cue naming the watchdog, got %q", got)
|
||||
}
|
||||
if !strings.Contains(got, "36s") {
|
||||
t.Fatalf("escalated cue should name the content-stall ceiling (ContentStallTimeout(30s) = 36s), got %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestQuietHintHiddenWhenSidebarShowsIt: when the context sidebar is up (it
|
||||
// carries the "generating…" pulse in ACTIVITY), the working line must NOT also
|
||||
// show the hint — it appears in exactly one place.
|
||||
|
||||
@@ -1,12 +0,0 @@
|
||||
{
|
||||
"$schema": "https://raw.githubusercontent.com/googleapis/release-please/main/schemas/config.json",
|
||||
"release-type": "node",
|
||||
"include-component-in-tag": false,
|
||||
"bootstrap-sha": "02f0c0929bec65f5fc41bdcab3eb518d2b0b329a",
|
||||
"packages": {
|
||||
".": {
|
||||
"package-name": "@gitlawb/zero",
|
||||
"release-as": "0.1.0"
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user