mirror of
https://github.com/Twigpine/zero.git
synced 2026-11-08 09:49:34 -10:00
Compare commits
11
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
c6ad2a43a3 | ||
|
|
bb4857d1e1 | ||
|
|
ce4b1e7443 | ||
|
|
fb37335a84 | ||
|
|
19cbd11236 | ||
|
|
7d51113ca8 | ||
|
|
136d65397f | ||
|
|
e4fc65fab6 | ||
|
|
389e242802 | ||
|
|
f1b0b4a993 | ||
|
|
a59909e3eb |
No files matched your search
@@ -25,7 +25,9 @@ type TokenResolver func(ctx context.Context, forceRefresh bool) (header string,
|
||||
//
|
||||
// base carries the API-key auth config (key + default header/scheme). setExtra
|
||||
// (optional) sets the provider's non-auth headers on every attempt. Transient
|
||||
// retries (429/503/529) are handled by the inner SendWithRetry.
|
||||
// retries are handled by the inner SendWithRetry: 429/503/529 on their own
|
||||
// budget, and provably pre-send transport failures (refused or timed-out
|
||||
// connect, DNS, TLS handshake) on a separate, shorter one.
|
||||
func SendWithAuthRetry(
|
||||
ctx context.Context,
|
||||
client *http.Client,
|
||||
|
||||
@@ -3,6 +3,9 @@ package providerio
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
@@ -17,13 +20,22 @@ import (
|
||||
// (previously only the OpenAI provider retried; Anthropic and Gemini surfaced the
|
||||
// first failure).
|
||||
//
|
||||
// What is retried: ONLY 429 (rate limit) and 503 (service unavailable) — the
|
||||
// statuses where the server explicitly did NOT accept the request, so replaying
|
||||
// it cannot duplicate work. Other 5xx (500/502/504) and transport/network errors
|
||||
// are NOT retried: a completion POST is non-idempotent and may have reached and
|
||||
// been processed by the server, so replaying it could duplicate
|
||||
// (billable) work. Only the INITIAL request is ever in scope; once the response
|
||||
// body starts streaming it is never re-issued.
|
||||
// What is retried: 429 (rate limit) and 503 (service unavailable) — the statuses
|
||||
// where the server explicitly did NOT accept the request — plus PROVABLY
|
||||
// pre-send transport failures (DNS resolution, a refused/unreachable dial, a TLS
|
||||
// handshake timeout) where no request bytes ever left this host. In every one of
|
||||
// those cases we KNOW the server did not receive and process the request, so a
|
||||
// replay cannot duplicate work. Other 5xx (500/502/504) and every ambiguous or
|
||||
// post-send transport error (connection reset, broken pipe, EOF, a generic i/o
|
||||
// timeout, context deadline) are NOT retried: a completion POST is
|
||||
// non-idempotent and may already have reached and been processed by the server,
|
||||
// so replaying it could duplicate (billable) work. Only the INITIAL request is
|
||||
// ever in scope; once the response body starts streaming it is never re-issued.
|
||||
// Pre-send classification is by syscall errno (portable across POSIX and
|
||||
// Windows), scoped to an Op=="dial" *net.OpError so the same errno raised on a
|
||||
// post-send read/write can't be misread as pre-send. NXDOMAIN is treated as
|
||||
// permanent (not retried), and the pre-send path fails fast on its own
|
||||
// sub-second schedule rather than the seconds-long 429 backoff.
|
||||
|
||||
const defaultMaxRetryAttempts = 6
|
||||
|
||||
@@ -37,12 +49,32 @@ const maxBackoff = 30 * time.Second
|
||||
// schedule is 2s, 4s, 8s, 16s, then maxBackoff. A var so tests can shrink it.
|
||||
var retryBackoffBase = 2 * time.Second
|
||||
|
||||
// preSendMaxAttempts bounds retries of a provably pre-send transport failure.
|
||||
// Unlike a 429 (whose window is measured in seconds), a refused/unreachable dial
|
||||
// either recovers within a few hundred ms or is a deterministic misconfiguration
|
||||
// (wrong host/port, dead local daemon) that will never recover — so a short,
|
||||
// low-count schedule fails fast instead of stalling the agent ~60s on the
|
||||
// rate-limit schedule. This bounds the pre-send path independently of the caller's
|
||||
// maxAttempts, which is tuned for 429/503.
|
||||
const preSendMaxAttempts = 3
|
||||
|
||||
// preSendBackoffBase is the first wait before replaying a pre-send transport
|
||||
// failure. Modeled on the mid-stream reconnect base (internal/agent/reconnect.go),
|
||||
// it is sub-second so a transient dial blip recovers quickly and a permanent one
|
||||
// fails fast. A var so tests can shrink it.
|
||||
var preSendBackoffBase = 500 * time.Millisecond
|
||||
|
||||
// SendWithRetry issues an HTTP request, retrying ONLY the safe-to-replay server
|
||||
// responses (429 and 503, see ShouldRetryStatus) up to maxAttempts — backing off
|
||||
// between tries and honoring a server Retry-After header and context
|
||||
// cancellation. Other 5xx and transport/network errors are returned immediately,
|
||||
// never replayed (see the package note). The request is rebuilt from body each
|
||||
// attempt; setHeader (if non-nil) sets headers on every attempt.
|
||||
// cancellation. The one transport-error exception is a PROVABLY pre-send failure
|
||||
// (see isPreSendTransportError): a refused/unreachable dial or DNS failure where
|
||||
// no request bytes left this host, retried on its own short schedule
|
||||
// (preSendMaxAttempts / preSendBackoff). Every other 5xx and every ambiguous or
|
||||
// post-send transport error is returned immediately, never replayed, because a
|
||||
// non-idempotent completion POST may already have been received. The request is
|
||||
// rebuilt from body each attempt; setHeader (if non-nil) sets headers on every
|
||||
// attempt.
|
||||
//
|
||||
// It returns the final *http.Response (which the caller inspects for a non-2xx
|
||||
// status, exactly as before) or an error for a network failure / context
|
||||
@@ -60,7 +92,47 @@ func SendWithRetry(
|
||||
if maxAttempts <= 0 {
|
||||
maxAttempts = defaultMaxRetryAttempts
|
||||
}
|
||||
for attempt := 1; ; attempt++ {
|
||||
if client == nil {
|
||||
client = http.DefaultClient
|
||||
}
|
||||
// Redirects are still FOLLOWED, so a redirecting gateway keeps working, but
|
||||
// entering one makes that attempt non-retryable. The hazard is that client.Do
|
||||
// can transmit the original POST to the first host, follow a 307/308, and then
|
||||
// fail the dial to the redirect target: that error is an Op=="dial" error, so
|
||||
// isPreSendTransportError would read it as "nothing was sent" and replay a
|
||||
// completion the first host already received. Observing the redirect rather
|
||||
// than blocking it keeps both properties, since a request that was redirected
|
||||
// is never replayed while redirecting endpoints stay usable. The caller's own
|
||||
// CheckRedirect still runs and still decides whether to proceed.
|
||||
redirected := false
|
||||
callerCheckRedirect := client.CheckRedirect
|
||||
redirectAwareClient := *client
|
||||
redirectAwareClient.CheckRedirect = func(request *http.Request, via []*http.Request) error {
|
||||
redirected = true
|
||||
if callerCheckRedirect != nil {
|
||||
return callerCheckRedirect(request, via)
|
||||
}
|
||||
// Mirror net/http's default policy, which a nil CheckRedirect would have
|
||||
// applied: stop after 10 hops.
|
||||
if len(via) >= 10 {
|
||||
return errors.New("stopped after 10 redirects")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
// The two retry budgets are fully independent, each with its own counter and
|
||||
// its own backoff ordinal, because they answer different questions: a
|
||||
// dial/DNS/TLS failure is a fast local blip (preSendMaxAttempts, sub-second
|
||||
// schedule) while a 429/503 is a server-side window measured in seconds
|
||||
// (maxAttempts, retryBackoffBase). A single shared loop counter coupled them
|
||||
// in both directions: status retries starved the pre-send budget, and
|
||||
// pre-send retries shortened the status window and started its backoff at a
|
||||
// later rung. Neither kind of failure now consumes or advances the other.
|
||||
preSendAttempts := 0
|
||||
statusAttempts := 0
|
||||
for {
|
||||
// Per attempt: only a redirect entered by THIS attempt can make its own
|
||||
// transport error unsafe to replay.
|
||||
redirected = false
|
||||
request, err := http.NewRequestWithContext(ctx, method, url, bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -70,32 +142,78 @@ func SendWithRetry(
|
||||
}
|
||||
|
||||
connectSpan := trace.FromContext(ctx).Span(trace.SpanProviderConnect)
|
||||
response, err := client.Do(request)
|
||||
response, err := redirectAwareClient.Do(request)
|
||||
connectSpan.End()
|
||||
if err != nil {
|
||||
// A transport failure on a POST does NOT mean the server didn't receive
|
||||
// it — the request may have arrived and be generating a (billable,
|
||||
// non-idempotent) completion while only the response/connection failed.
|
||||
// Replaying it could duplicate that work, so surface the error instead
|
||||
// of retrying. Only 429/503 responses (below), where we KNOW the request
|
||||
// was not accepted, are safe to retry.
|
||||
// Context cancellation always surfaces as cancellation, never a retry.
|
||||
if ctx.Err() != nil {
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
// A transport failure on a POST does NOT usually mean the server didn't
|
||||
// receive it — the request may have arrived and be generating a
|
||||
// (billable, non-idempotent) completion while only the response or
|
||||
// connection failed. Replaying that could duplicate work, so it is
|
||||
// surfaced immediately. The one safe exception is a PROVABLY pre-send
|
||||
// failure (see isPreSendTransportError): the request never left this
|
||||
// host, so a replay cannot duplicate anything — exactly like a 429/503
|
||||
// where we KNOW the request was not accepted. The pre-send path uses
|
||||
// its own short bound and sub-second schedule (preSendMaxAttempts /
|
||||
// preSendBackoff), not the seconds-long 429 schedule, so a permanent
|
||||
// dial failure fails fast instead of stalling the agent for ~60s.
|
||||
// A redirect means the original POST already left this host, so the
|
||||
// dial that failed belongs to a later hop and replaying would duplicate
|
||||
// a completion the first host may already have processed.
|
||||
if isPreSendTransportError(err) && !redirected {
|
||||
preSendAttempts++
|
||||
if preSendAttempts < preSendMaxAttempts {
|
||||
if r := trace.FromContext(ctx); r != nil {
|
||||
r.Counter(trace.CounterRetryCount, 1)
|
||||
}
|
||||
if preSendBackoff(ctx, preSendAttempts) {
|
||||
continue
|
||||
}
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if ShouldRetryStatus(response.StatusCode) && attempt < maxAttempts {
|
||||
if r := trace.FromContext(ctx); r != nil {
|
||||
r.Counter(trace.CounterRetryCount, 1)
|
||||
// Deliberately NOT guarded on `redirected`, unlike the transport-error
|
||||
// branch above. The two differ in what is known about the request. A
|
||||
// transport failure after a redirect leaves it unknown whether the
|
||||
// redirect target received and began processing the POST, so replaying
|
||||
// could duplicate billable work. Here every hop answered: a 307/308 is the
|
||||
// origin declining and delegating, and a retryable status is the target
|
||||
// declining as well, so nothing accepted the request and a replay
|
||||
// duplicates nothing.
|
||||
//
|
||||
// The exposure is also identical to a retryable status with no redirect in
|
||||
// play, which this package has always retried. Guarding on redirects here
|
||||
// would not close a gap, it would make a rate-limited completion behind a
|
||||
// redirecting gateway fail outright instead of backing off.
|
||||
// The transport hop succeeded, so the connect phase is healthy again and
|
||||
// the pre-send budget starts over. It bounds CONSECUTIVE connect failures,
|
||||
// not the lifetime of the call: without this reset, two early dial retries
|
||||
// followed by a 429 and then one more dial blip would surface a transport
|
||||
// error while the status budget still had room, which is the coupling
|
||||
// these two counters were split apart to remove. Total work stays bounded,
|
||||
// since every reset costs a status attempt and those are capped.
|
||||
preSendAttempts = 0
|
||||
|
||||
if ShouldRetryStatus(response.StatusCode) {
|
||||
statusAttempts++
|
||||
if statusAttempts < maxAttempts {
|
||||
if r := trace.FromContext(ctx); r != nil {
|
||||
r.Counter(trace.CounterRetryCount, 1)
|
||||
}
|
||||
wait := RetryAfter(response)
|
||||
_ = response.Body.Close()
|
||||
if Backoff(ctx, statusAttempts, wait) {
|
||||
continue
|
||||
}
|
||||
// Backoff aborted: the only reason it returns false is ctx cancellation.
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
wait := RetryAfter(response)
|
||||
_ = response.Body.Close()
|
||||
if Backoff(ctx, attempt, wait) {
|
||||
continue
|
||||
}
|
||||
// Backoff aborted: the only reason it returns false is ctx cancellation.
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
|
||||
// Success, a non-retryable status, or retries exhausted on a retryable
|
||||
@@ -109,6 +227,119 @@ func SendWithRetry(
|
||||
}
|
||||
}
|
||||
|
||||
// isPreSendTransportError reports whether a transport error PROVES no request
|
||||
// bytes reached the server, so replaying the request cannot duplicate a
|
||||
// billable completion. Only narrow, unambiguous pre-connect failures qualify: a
|
||||
// recoverable DNS lookup failure, a refused/unreachable dial, and a TLS
|
||||
// handshake timeout (the handshake completes before any HTTP request bytes are
|
||||
// written).
|
||||
//
|
||||
// The refused/unreachable errnos (ECONNREFUSED/ENETUNREACH/EHOSTUNREACH) are
|
||||
// matched by syscall errno via errors.Is — portable across platforms, since Go
|
||||
// maps the WSA* codes on Windows too — but ONLY when carried by an Op=="dial"
|
||||
// *net.OpError. errors.Is matches an errno anywhere in the chain regardless of
|
||||
// which socket operation raised it, and the kernel raises these same errnos on
|
||||
// an established connection when a route drops mid-generation; scoping to the
|
||||
// dial phase is what makes "no request bytes were sent" structural rather than
|
||||
// argued, so a post-send read/write can never replay a non-idempotent POST.
|
||||
// NXDOMAIN is treated as permanent (deterministic) and is NOT retried.
|
||||
//
|
||||
// Ambiguous or post-send failures are deliberately excluded and checked FIRST,
|
||||
// so an error that is both is never treated as pre-send: a reset or broken pipe
|
||||
// can follow a request that was already sent, and a bare "i/o timeout" covers
|
||||
// read timeouts after send as well as dial timeouts, so none of these prove the
|
||||
// request was not received and a non-idempotent POST must not be replayed on
|
||||
// them.
|
||||
func isPreSendTransportError(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
// DNS resolution precedes any connection, so a lookup failure proves nothing
|
||||
// was sent — but only retry a lookup that could plausibly succeed on a replay.
|
||||
// NXDOMAIN (IsNotFound) is authoritative and deterministic (a mistyped or
|
||||
// non-existent host), so retrying it just stalls the agent through the whole
|
||||
// backoff schedule before failing anyway; a timeout or temporary SERVFAIL is
|
||||
// worth another try. errors.As unwraps url.Error/OpError to reach the DNSError.
|
||||
var dnsErr *net.DNSError
|
||||
if errors.As(err, &dnsErr) {
|
||||
return !dnsErr.IsNotFound
|
||||
}
|
||||
// Post-send / ambiguous failures, excluded first. EOF and reset are matched
|
||||
// by identity so wording (or a hostname containing "eof") can't fool them.
|
||||
if errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) || isConnResetErrno(err) {
|
||||
return false
|
||||
}
|
||||
// A CONNECT-phase timeout is provably pre-send for the same structural reason
|
||||
// a refused dial is: the connection was never established, so no request
|
||||
// bytes left this host. It has to be admitted BEFORE the wording exclusions
|
||||
// below, because net surfaces it with "i/o timeout" in the message, which the
|
||||
// blanket exclusion would otherwise reject. The gate is what keeps that safe:
|
||||
// a post-send read timeout is also Timeout(), but carries Op "read", so it
|
||||
// never reaches here.
|
||||
if opErr := connectOpError(err); opErr != nil && opErr.Timeout() {
|
||||
return true
|
||||
}
|
||||
msg := strings.ToLower(err.Error())
|
||||
if strings.Contains(msg, "connection reset") ||
|
||||
strings.Contains(msg, "broken pipe") ||
|
||||
strings.Contains(msg, "i/o timeout") {
|
||||
return false
|
||||
}
|
||||
// A refused/unreachable connection is provably pre-send ONLY when it was
|
||||
// raised during the DIAL (connection-establishment) phase. The same errnos
|
||||
// are also raised by the kernel on an ESTABLISHED connection — when a route
|
||||
// drops or an ICMP unreachable arrives mid-generation — which is post-send.
|
||||
// errors.Is matches an errno anywhere in the chain regardless of the
|
||||
// operation, so we first require an Op=="dial" *net.OpError: that makes "no
|
||||
// request bytes left this host" structural, so a post-send read/write carrying
|
||||
// one of these can never replay a non-idempotent POST. dialPreSendErrnos is
|
||||
// platform-specific: on Windows a refused dial carries the raw WSA* errno
|
||||
// (WSAECONNREFUSED etc.), which does NOT satisfy errors.Is against the POSIX
|
||||
// syscall.ECONNREFUSED, so the Windows list adds those codes. The string
|
||||
// markers only catch a POSIX dial error already flattened past its errno.
|
||||
if connectOpError(err) != nil {
|
||||
for _, errno := range dialPreSendErrnos {
|
||||
if errors.Is(err, errno) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
switch {
|
||||
case strings.Contains(msg, "connection refused"),
|
||||
strings.Contains(msg, "network is unreachable"),
|
||||
strings.Contains(msg, "no route to host"):
|
||||
return true
|
||||
}
|
||||
}
|
||||
// A TLS handshake times out after the TCP dial connects but before any HTTP
|
||||
// request bytes are written, so it is structurally pre-send. net/http surfaces
|
||||
// it as its own error string, not a *net.OpError, so it is matched by wording
|
||||
// independent of the dial gate.
|
||||
if strings.Contains(msg, "tls handshake timeout") {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// connectOpError returns the outermost *net.OpError in err's chain when it comes
|
||||
// from the connection-establishment phase, else nil. Go tags a direct dial with
|
||||
// Op "dial" and a failed CONNECT through an HTTP proxy with Op "proxyconnect";
|
||||
// a failure on an already-established connection carries Op "read"/"write".
|
||||
// This is how isPreSendTransportError scopes the pre-send match to the connect
|
||||
// phase so a post-send failure can't be misclassified as pre-send.
|
||||
//
|
||||
// "proxyconnect" belongs here for the same reason "dial" does: when HTTPS_PROXY
|
||||
// is set and the CONNECT fails, the tunnel to the model host was never built, so
|
||||
// no request bytes reached it. Omitting it meant every pre-send failure behind a
|
||||
// corporate proxy fell through to no-retry, which is the environment where
|
||||
// transient connect failures are most common.
|
||||
func connectOpError(err error) *net.OpError {
|
||||
var opErr *net.OpError
|
||||
if errors.As(err, &opErr) && (opErr.Op == "dial" || opErr.Op == "proxyconnect") {
|
||||
return opErr
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ShouldRetryStatus reports whether an HTTP status is safe to retry for a
|
||||
// non-idempotent completion POST: 429 (Too Many Requests), 503 (Service
|
||||
// Unavailable), and 529 (Anthropic's "overloaded"). All mean the server
|
||||
@@ -137,6 +368,39 @@ func Backoff(ctx context.Context, attempt int, retryAfter time.Duration) bool {
|
||||
}
|
||||
}
|
||||
|
||||
// preSendBackoff waits before pre-send retry attempt N (1-based) on a short
|
||||
// sub-second schedule (500ms, 1s, ...), returning false if the context is
|
||||
// cancelled during the wait. Separate from Backoff's seconds-long rate-limit
|
||||
// schedule because a pre-send dial failure recovers in milliseconds or never.
|
||||
func preSendBackoff(ctx context.Context, attempt int) bool {
|
||||
timer := time.NewTimer(preSendBackoffWait(attempt))
|
||||
defer timer.Stop()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return false
|
||||
case <-timer.C:
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
// preSendBackoffWait computes the pre-send wait before retry attempt N (1-based):
|
||||
// exponential from preSendBackoffBase (500ms, 1s, 2s, ...), capped at maxBackoff.
|
||||
// The exponent is clamped so a large attempt count cannot overflow.
|
||||
func preSendBackoffWait(attempt int) time.Duration {
|
||||
exponent := attempt - 1
|
||||
if exponent > 5 {
|
||||
exponent = 5
|
||||
}
|
||||
if exponent < 0 {
|
||||
exponent = 0
|
||||
}
|
||||
wait := preSendBackoffBase * time.Duration(1<<exponent)
|
||||
if wait > maxBackoff {
|
||||
wait = maxBackoff
|
||||
}
|
||||
return wait
|
||||
}
|
||||
|
||||
// backoffWait computes the wait before retry attempt N (1-based): Retry-After
|
||||
// when supplied, else exponential from retryBackoffBase, both capped at
|
||||
// maxBackoff. The exponent is clamped so a large attempt count cannot overflow.
|
||||
|
||||
@@ -0,0 +1,32 @@
|
||||
//go:build !windows && !plan9
|
||||
|
||||
package providerio
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"syscall"
|
||||
)
|
||||
|
||||
// dialPreSendErrnos are the syscall errnos a refused or unreachable DIAL raises
|
||||
// on this platform. On POSIX the standard syscall constants are exactly what the
|
||||
// net package surfaces, so matching them via errors.Is is sufficient. Plan 9 is
|
||||
// excluded because it does not define these BSD-style errno constants (it uses
|
||||
// string errors), so the constants below would not compile there; Zero targets
|
||||
// linux/darwin/windows, so no Plan 9 variant is needed.
|
||||
var dialPreSendErrnos = []error{
|
||||
syscall.ECONNREFUSED,
|
||||
syscall.ENETUNREACH,
|
||||
syscall.EHOSTUNREACH,
|
||||
// A connect timeout is normally caught by the Timeout() check on the connect
|
||||
// gate; this covers the case where the errno survives but the timeout flag
|
||||
// does not, e.g. an error rebuilt from the errno alone.
|
||||
syscall.ETIMEDOUT,
|
||||
}
|
||||
|
||||
// isConnResetErrno reports whether err carries a connection-reset errno, the
|
||||
// post-send exclusion in isPreSendTransportError. It lives here (per platform)
|
||||
// so retry.go references no syscall constants and stays buildable on Plan 9,
|
||||
// which defines none of them.
|
||||
func isConnResetErrno(err error) bool {
|
||||
return errors.Is(err, syscall.ECONNRESET)
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
//go:build plan9
|
||||
|
||||
package providerio
|
||||
|
||||
// Plan 9 has no BSD-style errno constants (it uses string errors), so the
|
||||
// errno-based classification is unavailable here: isConnResetErrno returns false
|
||||
// and dialPreSendErrnos is empty. retry.go still classifies via its string
|
||||
// markers on this platform, the pre-send inclusions (connection refused /
|
||||
// network unreachable / no route to host) and the separate post-send
|
||||
// "connection reset" exclusion each keep working through the wording checks.
|
||||
// Zero targets linux/darwin/windows, so this variant only keeps the package
|
||||
// building on Plan 9 rather than adding real support.
|
||||
var dialPreSendErrnos = []error{}
|
||||
|
||||
func isConnResetErrno(error) bool { return false }
|
||||
@@ -0,0 +1,40 @@
|
||||
//go:build windows
|
||||
|
||||
package providerio
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"syscall"
|
||||
|
||||
"golang.org/x/sys/windows"
|
||||
)
|
||||
|
||||
// dialPreSendErrnos are the errnos a refused or unreachable DIAL raises on
|
||||
// Windows. A real Windows dial failure carries the raw Winsock error
|
||||
// (WSAECONNREFUSED = 10061, WSAENETUNREACH = 10051, WSAEHOSTUNREACH = 10065), and
|
||||
// errors.Is against the POSIX syscall.ECONNREFUSED returns FALSE there because
|
||||
// Go's Windows syscall.ECONNREFUSED is a distinct value the net package never
|
||||
// produces. So the WSA codes must be matched explicitly, or a refused dial would
|
||||
// silently never be retried on Windows. The POSIX constants stay in the list too
|
||||
// so an error built with them (e.g. a test fixture) still classifies.
|
||||
var dialPreSendErrnos = []error{
|
||||
syscall.ECONNREFUSED,
|
||||
syscall.ENETUNREACH,
|
||||
syscall.EHOSTUNREACH,
|
||||
syscall.ETIMEDOUT,
|
||||
windows.WSAECONNREFUSED,
|
||||
windows.WSAENETUNREACH,
|
||||
windows.WSAEHOSTUNREACH,
|
||||
// Same reasoning as the POSIX ETIMEDOUT entry, with the Winsock code a real
|
||||
// Windows connect timeout actually carries.
|
||||
windows.WSAETIMEDOUT,
|
||||
}
|
||||
|
||||
// isConnResetErrno reports whether err carries a connection-reset errno, the
|
||||
// post-send exclusion in isPreSendTransportError. It matches the same
|
||||
// syscall.ECONNRESET the shared path used before the errno classification was
|
||||
// split out per platform, so behavior on Windows is unchanged; it lives here
|
||||
// so retry.go references no syscall constants and stays buildable on Plan 9.
|
||||
func isConnResetErrno(err error) bool {
|
||||
return errors.Is(err, syscall.ECONNRESET)
|
||||
}
|
||||
@@ -1,15 +1,34 @@
|
||||
//go:build !plan9
|
||||
|
||||
package providerio
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"sync/atomic"
|
||||
"syscall"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// wrapDialErrno builds the error shape a dial failure has (a *net.OpError
|
||||
// wrapping an *os.SyscallError wrapping the errno) so errors.Is reaches the
|
||||
// errno exactly as in production. These fixtures use the POSIX syscall.Exxx
|
||||
// constants, which do NOT reproduce a real Windows dial: that carries distinct
|
||||
// windows.WSA* errnos (Go's syscall.ECONNREFUSED is a value the net package
|
||||
// never produces on Windows). They still classify on Windows only because
|
||||
// dialPreSendErrnos there also lists the POSIX constants for exactly this
|
||||
// fixture reason; the real Windows dial branch is covered separately by
|
||||
// TestIsPreSendTransportErrorRealRefusedDial, which makes an actual refused dial.
|
||||
func wrapDialErrno(op string, errno syscall.Errno) error {
|
||||
return &net.OpError{Op: op, Net: "tcp", Err: os.NewSyscallError("connectex", errno)}
|
||||
}
|
||||
|
||||
type roundTripperFunc func(*http.Request) (*http.Response, error)
|
||||
|
||||
func (f roundTripperFunc) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) }
|
||||
@@ -35,6 +54,152 @@ func TestSendWithRetryDoesNotReplayTransportErrors(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// A PROVABLY pre-send transport failure (no request bytes left this host) is
|
||||
// safe to replay and must be retried, bounded by preSendMaxAttempts (its own
|
||||
// short schedule), unlike an ambiguous post-send failure. Real dial failures
|
||||
// arrive as an Op=="dial" *net.OpError, so the errno cases are the production
|
||||
// shape on every platform (Windows included); the string case exercises the
|
||||
// wording fallback for a dial error already flattened past its errno.
|
||||
func TestSendWithRetryReplaysProvablyPreSendErrors(t *testing.T) {
|
||||
shrinkBackoff(t)
|
||||
cases := map[string]error{
|
||||
"errno refused (dial)": wrapDialErrno("dial", syscall.ECONNREFUSED),
|
||||
"errno network unreachable": wrapDialErrno("dial", syscall.ENETUNREACH),
|
||||
"errno host unreachable": wrapDialErrno("dial", syscall.EHOSTUNREACH),
|
||||
"dial string fallback": &net.OpError{Op: "dial", Net: "tcp", Err: errors.New("connect: connection refused")},
|
||||
"tls handshake timeout": errors.New("net/http: TLS handshake timeout"),
|
||||
"dns timeout": &net.DNSError{Err: "server misbehaving", Name: "nope.invalid", IsTimeout: true},
|
||||
}
|
||||
for name, transportErr := range cases {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
var calls int32
|
||||
client := &http.Client{Transport: roundTripperFunc(func(*http.Request) (*http.Response, error) {
|
||||
atomic.AddInt32(&calls, 1)
|
||||
return nil, transportErr
|
||||
})}
|
||||
// maxAttempts=6 (the default) proves the pre-send path is bounded by
|
||||
// preSendMaxAttempts, not the caller's 429-tuned maxAttempts.
|
||||
resp, err := SendWithRetry(context.Background(), client, http.MethodPost, "http://example.invalid", []byte("{}"), nil, 6)
|
||||
if resp != nil {
|
||||
_ = resp.Body.Close()
|
||||
}
|
||||
if err == nil {
|
||||
t.Fatal("expected the transport error to surface after retries are exhausted")
|
||||
}
|
||||
if got := atomic.LoadInt32(&calls); got != preSendMaxAttempts {
|
||||
t.Fatalf("pre-send error tried %d times, want %d (preSendMaxAttempts)", got, preSendMaxAttempts)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// An ambiguous transport failure that could have followed a sent request must
|
||||
// NOT be replayed: a non-idempotent completion POST could duplicate billable
|
||||
// work. This is the safety line the fix must not cross.
|
||||
func TestSendWithRetryDoesNotReplayAmbiguousTransportErrors(t *testing.T) {
|
||||
for name, transportErr := range map[string]error{
|
||||
"generic i/o timeout": errors.New("dial tcp 1.2.3.4:443: i/o timeout"),
|
||||
"broken pipe": errors.New("write tcp: broken pipe"),
|
||||
"unexpected eof": io.ErrUnexpectedEOF,
|
||||
// The pre-send errnos are also raised on an ESTABLISHED connection: a route
|
||||
// dropping mid-generation surfaces on the pending read/write, which is
|
||||
// post-send. Scoping to Op=="dial" must keep these from replaying the POST.
|
||||
"host unreachable on read is post-send": wrapDialErrno("read", syscall.EHOSTUNREACH),
|
||||
"net unreachable on write is post-send": wrapDialErrno("write", syscall.ENETUNREACH),
|
||||
// NXDOMAIN is authoritative and deterministic, so retrying it would only
|
||||
// stall the agent before failing anyway.
|
||||
"dns nxdomain": &net.DNSError{Err: "no such host", Name: "nope.invalid", IsNotFound: true},
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
var calls int32
|
||||
client := &http.Client{Transport: roundTripperFunc(func(*http.Request) (*http.Response, error) {
|
||||
atomic.AddInt32(&calls, 1)
|
||||
return nil, transportErr
|
||||
})}
|
||||
resp, err := SendWithRetry(context.Background(), client, http.MethodPost, "http://example.invalid", []byte("{}"), nil, 3)
|
||||
if resp != nil {
|
||||
_ = resp.Body.Close()
|
||||
}
|
||||
if err == nil {
|
||||
t.Fatal("expected the transport error to surface")
|
||||
}
|
||||
if got := atomic.LoadInt32(&calls); got != 1 {
|
||||
t.Fatalf("ambiguous transport error replayed %d times, want 1 (no retry)", got)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// A REAL refused dial (nothing listening on the target) must classify as
|
||||
// pre-send on every platform. This is the regression the errno-constant fixtures
|
||||
// miss: on Windows the kernel raises WSAECONNREFUSED, which errors.Is does NOT
|
||||
// match against syscall.ECONNREFUSED, so before dialPreSendErrnos carried the WSA
|
||||
// codes a refused dial returned false here and Windows never retried it. Using a
|
||||
// real Dial exercises the platform's true error shape.
|
||||
func TestIsPreSendTransportErrorRealRefusedDial(t *testing.T) {
|
||||
// Reserve an ephemeral port, then close it, so a dial there is refused.
|
||||
ln, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatalf("listen: %v", err)
|
||||
}
|
||||
addr := ln.Addr().String()
|
||||
if cerr := ln.Close(); cerr != nil {
|
||||
t.Fatalf("close listener: %v", cerr)
|
||||
}
|
||||
conn, dialErr := (&net.Dialer{Timeout: 2 * time.Second}).Dial("tcp", addr)
|
||||
if dialErr == nil {
|
||||
_ = conn.Close()
|
||||
t.Skip("expected a refused dial but the port accepted a connection")
|
||||
}
|
||||
if !isPreSendTransportError(dialErr) {
|
||||
t.Fatalf("real refused dial not classified pre-send: %v", dialErr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsPreSendTransportError(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
err error
|
||||
want bool
|
||||
}{
|
||||
{"nil", nil, false},
|
||||
// DNS: a lookup that could recover is pre-send; NXDOMAIN is permanent.
|
||||
{"dns timeout", &net.DNSError{Err: "server misbehaving", Name: "x.invalid", IsTimeout: true}, true},
|
||||
{"dns temporary", &net.DNSError{Err: "server misbehaving", Name: "x.invalid", IsTemporary: true}, true},
|
||||
{"dns nxdomain is permanent", &net.DNSError{Err: "no such host", Name: "x.invalid", IsNotFound: true}, false},
|
||||
{"tls handshake timeout", errors.New("net/http: TLS handshake timeout"), true},
|
||||
// Errno-wrapped DIAL failures: how a real refused/unreachable dial arrives
|
||||
// on EVERY platform, including Windows where the wording differs entirely.
|
||||
{"errno refused (dial)", wrapDialErrno("dial", syscall.ECONNREFUSED), true},
|
||||
{"errno network unreachable (dial)", wrapDialErrno("dial", syscall.ENETUNREACH), true},
|
||||
{"errno host unreachable (dial)", wrapDialErrno("dial", syscall.EHOSTUNREACH), true},
|
||||
{"dial operror string fallback", &net.OpError{Op: "dial", Net: "tcp", Err: errors.New("connect: connection refused")}, true},
|
||||
// The SAME errnos raised on a post-send read/write are NOT pre-send — the
|
||||
// kernel reports them on an established connection when a route drops.
|
||||
{"host unreachable on read is post-send", wrapDialErrno("read", syscall.EHOSTUNREACH), false},
|
||||
{"net unreachable on write is post-send", wrapDialErrno("write", syscall.ENETUNREACH), false},
|
||||
{"errno reset is post-send", wrapDialErrno("read", syscall.ECONNRESET), false},
|
||||
// A refused/unreachable error already flattened past its *net.OpError can't
|
||||
// be proven pre-send, so it is NOT retried (conservative direction).
|
||||
{"flattened refused, no opError", errors.New("dial tcp 127.0.0.1:1: connect: connection refused"), false},
|
||||
{"connection reset", errors.New("read tcp: connection reset by peer"), false},
|
||||
{"broken pipe", errors.New("write tcp: broken pipe"), false},
|
||||
{"unexpected eof", io.ErrUnexpectedEOF, false},
|
||||
{"eof", io.EOF, false},
|
||||
{"generic io timeout", errors.New("dial tcp: i/o timeout"), false},
|
||||
{"context deadline", context.DeadlineExceeded, false},
|
||||
// Exclusion is checked before inclusion, even for a dial OpError: an "i/o
|
||||
// timeout" in the message wins over the refused wording.
|
||||
{"exclusion wins over inclusion", &net.OpError{Op: "dial", Net: "tcp", Err: errors.New("connection refused / i/o timeout")}, false},
|
||||
{"unrelated", errors.New("some other error"), false},
|
||||
}
|
||||
for _, c := range cases {
|
||||
if got := isPreSendTransportError(c.err); got != c.want {
|
||||
t.Errorf("%s: isPreSendTransportError(%v) = %v, want %v", c.name, c.err, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestShouldRetryStatus(t *testing.T) {
|
||||
cases := map[int]bool{
|
||||
http.StatusOK: false,
|
||||
@@ -94,12 +259,17 @@ func TestBackoffWaitsThenReturnsTrue(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// shrinkBackoff makes retry waits negligible for the duration of a test.
|
||||
// shrinkBackoff makes both retry schedules (429 and pre-send) negligible for the
|
||||
// duration of a test.
|
||||
func shrinkBackoff(t *testing.T) {
|
||||
t.Helper()
|
||||
saved := retryBackoffBase
|
||||
savedRetry, savedPreSend := retryBackoffBase, preSendBackoffBase
|
||||
retryBackoffBase = time.Millisecond
|
||||
t.Cleanup(func() { retryBackoffBase = saved })
|
||||
preSendBackoffBase = time.Millisecond
|
||||
t.Cleanup(func() {
|
||||
retryBackoffBase = savedRetry
|
||||
preSendBackoffBase = savedPreSend
|
||||
})
|
||||
}
|
||||
|
||||
func TestBackoffWaitSchedule(t *testing.T) {
|
||||
@@ -126,6 +296,30 @@ func TestBackoffWaitSchedule(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// The pre-send schedule is sub-second and doubles, far shorter than the 429
|
||||
// schedule: a permanent dial failure fails in ~1.5s across preSendMaxAttempts
|
||||
// (500ms + 1s) instead of stalling the agent ~60s on 2/4/8/16/30s. This is the
|
||||
// second half of the fix for a mistyped host / dead local daemon hanging a turn.
|
||||
func TestPreSendBackoffWaitSchedule(t *testing.T) {
|
||||
cases := []struct {
|
||||
attempt int
|
||||
want time.Duration
|
||||
}{
|
||||
{1, 500 * time.Millisecond},
|
||||
{2, 1 * time.Second},
|
||||
{3, 2 * time.Second},
|
||||
// The exponent clamps at 5 (500ms<<5 = 16s) so a large attempt count can't
|
||||
// overflow; in practice preSendMaxAttempts caps retries at attempt 2, so
|
||||
// only the 500ms/1s rungs are ever reached.
|
||||
{50, 16 * time.Second},
|
||||
}
|
||||
for _, c := range cases {
|
||||
if got := preSendBackoffWait(c.attempt); got != c.want {
|
||||
t.Errorf("preSendBackoffWait(%d) = %v, want %v", c.attempt, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSendWithRetryRetriesThenSucceeds(t *testing.T) {
|
||||
shrinkBackoff(t)
|
||||
var hits int32
|
||||
@@ -193,3 +387,316 @@ func TestSendWithRetryReturnsLastResponseAfterMaxAttempts(t *testing.T) {
|
||||
t.Fatalf("server hit %d times, want 2 (maxAttempts)", got)
|
||||
}
|
||||
}
|
||||
|
||||
// Redirects are still followed, so a gateway that answers the completion endpoint
|
||||
// with a 307/308 keeps working: the caller gets the final response, not the 3xx.
|
||||
func TestSendWithRetryFollowsRedirects(t *testing.T) {
|
||||
var calls int32
|
||||
client := &http.Client{Transport: roundTripperFunc(func(r *http.Request) (*http.Response, error) {
|
||||
if atomic.AddInt32(&calls, 1) == 1 {
|
||||
return &http.Response{
|
||||
StatusCode: http.StatusTemporaryRedirect,
|
||||
Header: http.Header{"Location": {"https://redirect-target.invalid/v1"}},
|
||||
Body: http.NoBody,
|
||||
Request: r,
|
||||
}, nil
|
||||
}
|
||||
return &http.Response{StatusCode: http.StatusOK, Body: http.NoBody, Request: r}, nil
|
||||
})}
|
||||
|
||||
resp, err := SendWithRetry(context.Background(), client, http.MethodPost, "https://origin.invalid/v1", []byte("{}"), nil, 3)
|
||||
if err != nil {
|
||||
t.Fatalf("a redirected completion must succeed: %v", err)
|
||||
}
|
||||
_ = resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("status = %d, want 200 (the redirect should have been followed)", resp.StatusCode)
|
||||
}
|
||||
if got := atomic.LoadInt32(&calls); got != 2 {
|
||||
t.Fatalf("transport called %d times, want 2 (origin then redirect target)", got)
|
||||
}
|
||||
}
|
||||
|
||||
// Once an attempt has entered a redirect, the original POST has already left this
|
||||
// host, so a dial failure on the redirect hop must NOT be replayed even though it
|
||||
// arrives as an Op=="dial" error that isPreSendTransportError would otherwise
|
||||
// treat as provably pre-send. Replaying it would re-bill a completion the first
|
||||
// host may already have processed.
|
||||
func TestSendWithRetryDoesNotReplayAfterRedirect(t *testing.T) {
|
||||
shrinkBackoff(t)
|
||||
var calls int32
|
||||
client := &http.Client{Transport: roundTripperFunc(func(r *http.Request) (*http.Response, error) {
|
||||
if atomic.AddInt32(&calls, 1) == 1 {
|
||||
return &http.Response{
|
||||
StatusCode: http.StatusTemporaryRedirect,
|
||||
Header: http.Header{"Location": {"https://redirect-target.invalid/v1"}},
|
||||
Body: http.NoBody,
|
||||
Request: r,
|
||||
}, nil
|
||||
}
|
||||
// The dial to the redirect target fails: pre-send in shape, but post-send
|
||||
// in fact, because the origin already received the POST.
|
||||
return nil, wrapDialErrno("dial", syscall.ECONNREFUSED)
|
||||
})}
|
||||
|
||||
resp, err := SendWithRetry(context.Background(), client, http.MethodPost, "https://origin.invalid/v1", []byte("{}"), nil, 6)
|
||||
if resp != nil {
|
||||
_ = resp.Body.Close()
|
||||
}
|
||||
if err == nil {
|
||||
t.Fatal("expected the redirect-hop dial failure to surface as an error")
|
||||
}
|
||||
// Exactly two: the origin request and the single redirect hop. A third would
|
||||
// mean the redirected POST was replayed.
|
||||
if got := atomic.LoadInt32(&calls); got != 2 {
|
||||
t.Fatalf("redirected POST was replayed: transport called %d times, want 2", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A caller that supplies its own CheckRedirect keeps it: the retry path observes
|
||||
// redirects, it does not take the decision away from the caller.
|
||||
func TestSendWithRetryPreservesCallerRedirectPolicy(t *testing.T) {
|
||||
var policyCalls int32
|
||||
client := &http.Client{
|
||||
CheckRedirect: func(*http.Request, []*http.Request) error {
|
||||
atomic.AddInt32(&policyCalls, 1)
|
||||
return http.ErrUseLastResponse
|
||||
},
|
||||
Transport: roundTripperFunc(func(r *http.Request) (*http.Response, error) {
|
||||
return &http.Response{
|
||||
StatusCode: http.StatusTemporaryRedirect,
|
||||
Header: http.Header{"Location": {"https://redirect-target.invalid/v1"}},
|
||||
Body: http.NoBody,
|
||||
Request: r,
|
||||
}, nil
|
||||
}),
|
||||
}
|
||||
|
||||
resp, err := SendWithRetry(context.Background(), client, http.MethodPost, "https://origin.invalid/v1", []byte("{}"), nil, 3)
|
||||
if err != nil {
|
||||
t.Fatalf("caller policy asked to stop at the 3xx, so it must surface: %v", err)
|
||||
}
|
||||
_ = resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusTemporaryRedirect {
|
||||
t.Fatalf("status = %d, want the 307 surfaced per the caller's policy", resp.StatusCode)
|
||||
}
|
||||
if got := atomic.LoadInt32(&policyCalls); got != 1 {
|
||||
t.Fatalf("caller CheckRedirect called %d times, want 1 (it must not be discarded)", got)
|
||||
}
|
||||
}
|
||||
|
||||
// The pre-send retry budget is independent of the 429/503 status retries: two
|
||||
// rate-limit responses must not consume the pre-send allowance, so the first
|
||||
// refused dial after them still gets its own preSendMaxAttempts tries.
|
||||
func TestSendWithRetryPreSendBudgetSurvivesStatusRetries(t *testing.T) {
|
||||
shrinkBackoff(t)
|
||||
var calls int32
|
||||
client := &http.Client{Transport: roundTripperFunc(func(r *http.Request) (*http.Response, error) {
|
||||
if atomic.AddInt32(&calls, 1) <= 2 {
|
||||
return &http.Response{StatusCode: http.StatusTooManyRequests, Body: http.NoBody, Request: r}, nil
|
||||
}
|
||||
return nil, wrapDialErrno("dial", syscall.ECONNREFUSED)
|
||||
})}
|
||||
|
||||
resp, err := SendWithRetry(context.Background(), client, http.MethodPost, "https://x.invalid/v1", []byte("{}"), nil, 6)
|
||||
if resp != nil {
|
||||
_ = resp.Body.Close()
|
||||
}
|
||||
if err == nil {
|
||||
t.Fatal("expected the exhausted pre-send failure to surface as an error")
|
||||
}
|
||||
// Two status responses (calls 1-2), then the pre-send failure gets its own
|
||||
// budget: preSendMaxAttempts=3 means it retries twice more before returning.
|
||||
if got := atomic.LoadInt32(&calls); got != 2+preSendMaxAttempts {
|
||||
t.Fatalf("pre-send budget eaten by status retries: transport called %d times, want %d (2 status + %d pre-send)", got, 2+preSendMaxAttempts, preSendMaxAttempts)
|
||||
}
|
||||
}
|
||||
|
||||
// The inverse sequence: pre-send retries must not consume the 429/503 budget or
|
||||
// advance its backoff rung either. Two safely replayed refused dials followed by
|
||||
// rate limiting must still leave the status path its full maxAttempts window,
|
||||
// otherwise a couple of dial blips silently shorten every later rate-limit wait.
|
||||
func TestSendWithRetryStatusBudgetSurvivesPreSendRetries(t *testing.T) {
|
||||
shrinkBackoff(t)
|
||||
const maxAttempts = 3
|
||||
var calls int32
|
||||
client := &http.Client{Transport: roundTripperFunc(func(r *http.Request) (*http.Response, error) {
|
||||
if atomic.AddInt32(&calls, 1) <= 2 {
|
||||
return nil, wrapDialErrno("dial", syscall.ECONNREFUSED)
|
||||
}
|
||||
return &http.Response{StatusCode: http.StatusTooManyRequests, Body: http.NoBody, Request: r}, nil
|
||||
})}
|
||||
|
||||
resp, err := SendWithRetry(context.Background(), client, http.MethodPost, "https://x.invalid/v1", []byte("{}"), nil, maxAttempts)
|
||||
if err != nil {
|
||||
t.Fatalf("exhausted status retries surface the response, not an error: %v", err)
|
||||
}
|
||||
_ = resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusTooManyRequests {
|
||||
t.Fatalf("status = %d, want 429", resp.StatusCode)
|
||||
}
|
||||
// Calls 1-2 are the retried dial failures; the status path then gets its own
|
||||
// full window (maxAttempts requests) before the 429 is surfaced.
|
||||
if got := atomic.LoadInt32(&calls); got != 2+maxAttempts {
|
||||
t.Fatalf("status budget eaten by pre-send retries: transport called %d times, want %d (2 pre-send + %d status)", got, 2+maxAttempts, maxAttempts)
|
||||
}
|
||||
}
|
||||
|
||||
// A redirect followed by a RETRYABLE STATUS is still retried, and deliberately
|
||||
// so. This is the case a reviewer flagged as a possible replay of billable work,
|
||||
// and the asymmetry with the transport-error guard above is intentional rather
|
||||
// than an oversight, so it is pinned here.
|
||||
//
|
||||
// The two situations differ in what is known about the request:
|
||||
//
|
||||
// - Transport error after a redirect: the original POST reached the origin and
|
||||
// the dial to the redirect target then failed, so whether the target received
|
||||
// and began processing it is UNKNOWN. Replaying could duplicate work, which
|
||||
// is why that path refuses to retry.
|
||||
// - Retryable status after a redirect: every hop answered. A 307/308 is the
|
||||
// origin declining to process and delegating, and a 429/503/529 is the target
|
||||
// declining as well. Nothing accepted the request, so replaying duplicates
|
||||
// nothing.
|
||||
//
|
||||
// The risk here is also exactly the risk of a 429 without any redirect, which
|
||||
// this package has always retried; the redirect adds no new ambiguity. Refusing
|
||||
// to retry would instead make a rate-limited completion behind a redirecting
|
||||
// gateway fail outright rather than back off.
|
||||
func TestSendWithRetryRetriesRedirectedRetryableStatus(t *testing.T) {
|
||||
var calls int32
|
||||
client := &http.Client{Transport: roundTripperFunc(func(r *http.Request) (*http.Response, error) {
|
||||
switch atomic.AddInt32(&calls, 1) {
|
||||
case 1, 3:
|
||||
// The origin declines and delegates, both times.
|
||||
return &http.Response{
|
||||
StatusCode: http.StatusTemporaryRedirect,
|
||||
Header: http.Header{"Location": {"https://redirect-target.invalid/v1"}},
|
||||
Body: http.NoBody,
|
||||
Request: r,
|
||||
}, nil
|
||||
case 2:
|
||||
// The target declines too: rate limited, so nothing was accepted.
|
||||
return &http.Response{StatusCode: http.StatusTooManyRequests, Body: http.NoBody, Request: r}, nil
|
||||
default:
|
||||
return &http.Response{StatusCode: http.StatusOK, Body: http.NoBody, Request: r}, nil
|
||||
}
|
||||
})}
|
||||
|
||||
resp, err := SendWithRetry(context.Background(), client, http.MethodPost, "https://origin.invalid/v1", []byte("{}"), nil, 3)
|
||||
if err != nil {
|
||||
t.Fatalf("a rate-limited completion behind a redirect must still retry: %v", err)
|
||||
}
|
||||
_ = resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("status = %d, want 200 after the retry succeeded", resp.StatusCode)
|
||||
}
|
||||
// origin, target(429), origin, target(200).
|
||||
if got := atomic.LoadInt32(&calls); got != 4 {
|
||||
t.Fatalf("transport called %d times, want 4 (redirect, 429, redirect, 200)", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A CONNECT-phase timeout is provably pre-send: the connection was never
|
||||
// established, so no request bytes left this host. It reaches the classifier
|
||||
// carrying "i/o timeout", which the blanket wording exclusion rejects, so
|
||||
// admitting it depends on the connect gate running first.
|
||||
//
|
||||
// The post-send counterpart is the point of the table: a read timeout is also
|
||||
// Timeout(), and must stay excluded, or a reply that timed out mid-generation
|
||||
// would replay a completion the server already billed.
|
||||
func TestPreSendClassifiesConnectTimeouts(t *testing.T) {
|
||||
timeoutErr := &timeoutError{}
|
||||
for name, testCase := range map[string]struct {
|
||||
err error
|
||||
want bool
|
||||
}{
|
||||
"dial timeout": {
|
||||
err: &net.OpError{Op: "dial", Net: "tcp", Err: timeoutErr},
|
||||
want: true,
|
||||
},
|
||||
"proxy connect timeout": {
|
||||
err: &net.OpError{Op: "proxyconnect", Net: "tcp", Err: timeoutErr},
|
||||
want: true,
|
||||
},
|
||||
"read timeout stays post-send": {
|
||||
err: &net.OpError{Op: "read", Net: "tcp", Err: timeoutErr},
|
||||
want: false,
|
||||
},
|
||||
"write timeout stays post-send": {
|
||||
err: &net.OpError{Op: "write", Net: "tcp", Err: timeoutErr},
|
||||
want: false,
|
||||
},
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
if got := isPreSendTransportError(testCase.err); got != testCase.want {
|
||||
t.Fatalf("isPreSendTransportError(%v) = %v, want %v", testCase.err, got, testCase.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// A failed CONNECT through an HTTP proxy is pre-send for the same structural
|
||||
// reason a refused dial is, but net tags it Op "proxyconnect", so a gate
|
||||
// matching only "dial" left every pre-send failure behind a corporate proxy
|
||||
// unretried.
|
||||
func TestPreSendClassifiesProxyConnectRefusal(t *testing.T) {
|
||||
err := &net.OpError{Op: "proxyconnect", Net: "tcp", Err: syscall.ECONNREFUSED}
|
||||
if !isPreSendTransportError(err) {
|
||||
t.Fatal("a refused proxy CONNECT was not classified pre-send; no request bytes reached the model host")
|
||||
}
|
||||
// The op still has to be a connect phase. The same errno on an established
|
||||
// connection is post-send and must not replay a POST.
|
||||
post := &net.OpError{Op: "read", Net: "tcp", Err: syscall.ECONNREFUSED}
|
||||
if isPreSendTransportError(post) {
|
||||
t.Fatal("a read-phase error was classified pre-send")
|
||||
}
|
||||
}
|
||||
|
||||
// timeoutError is a net.Error whose Timeout() is true, matching the shape
|
||||
// net/http surfaces for a connect deadline.
|
||||
type timeoutError struct{}
|
||||
|
||||
func (timeoutError) Error() string { return "i/o timeout" }
|
||||
func (timeoutError) Timeout() bool { return true }
|
||||
func (timeoutError) Temporary() bool { return true }
|
||||
|
||||
// The pre-send budget bounds CONSECUTIVE connect failures, not the lifetime of
|
||||
// the call. Without a reset on a successful hop, an early run of dial blips
|
||||
// permanently spends the budget, so a later blip surfaces as a transport error
|
||||
// while the status budget still has room. That is the coupling the two counters
|
||||
// were split apart to remove, in the one ordering the earlier tests did not
|
||||
// cover: pre-send, then status, then pre-send again.
|
||||
func TestPreSendBudgetResetsAfterSuccessfulHop(t *testing.T) {
|
||||
refused := &net.OpError{Op: "dial", Net: "tcp", Err: syscall.ECONNREFUSED}
|
||||
var calls int32
|
||||
client := &http.Client{Transport: roundTripperFunc(func(r *http.Request) (*http.Response, error) {
|
||||
switch atomic.AddInt32(&calls, 1) {
|
||||
case 1, 2:
|
||||
// Two connect failures, spending the pre-send budget under the old
|
||||
// lifetime accounting.
|
||||
return nil, refused
|
||||
case 3:
|
||||
// A hop that reaches the server. It declines, so the request was not
|
||||
// accepted and retrying stays safe.
|
||||
return &http.Response{StatusCode: http.StatusTooManyRequests, Body: http.NoBody, Request: r}, nil
|
||||
case 4:
|
||||
// One more connect blip. With a lifetime cap this surfaces as an
|
||||
// error; the call should still recover.
|
||||
return nil, refused
|
||||
default:
|
||||
return &http.Response{StatusCode: http.StatusOK, Body: http.NoBody, Request: r}, nil
|
||||
}
|
||||
})}
|
||||
|
||||
resp, err := SendWithRetry(context.Background(), client, http.MethodPost, "https://origin.invalid/v1", []byte("{}"), nil, 3)
|
||||
if err != nil {
|
||||
t.Fatalf("interleaved connect and status failures returned %v; the pre-send budget did not reset after the hop that reached the server", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("status = %d, want 200", resp.StatusCode)
|
||||
}
|
||||
if got := atomic.LoadInt32(&calls); got != 5 {
|
||||
t.Fatalf("transport called %d times, want 5 (dial, dial, 429, dial, 200)", got)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user