fix: tier authentication failures and retry transient provider errors

The merged branches disagreed on authentication failures: feature/1.0.5
marked every runtime auth failure 'unknown' so readiness never flipped,
while main's 22f811c classified any auth-looking diagnostic as
LOCAL_AGENT_AUTH_REQUIRED. Together with broad regexes (a bare
"forbidden" anywhere, a credential regex matching "login" or "403"),
proxy refusals stopped Agents while real sign-outs left them "ready".

- New pure module runs/failure-signals.cjs classifies CLI text:
  explicit sign-in or API-key statements are definitive; line-leading
  403/forbidden/unauthorized and permission_error are ambiguous; tool
  output that merely mentions 403 is neither. It also recognises rate
  limits (with Retry-After hints), network failures, provider 5xx and
  context overflow.
- Definitive failures stay LOCAL_AGENT_AUTH_REQUIRED and now mark the
  credential 'missing' (the catalog clears it after the retry window or
  when native credentials verify). Ambiguous refusals become
  LOCAL_AGENT_ACCESS_DENIED, are retried once, and never touch readiness.
- Process failures whose diagnostics show throttling, network loss or a
  provider outage are retried in that category. Rate limits honour
  Retry-After up to 60 s and wait at least 2 s without a hint.
- A resumed session that no longer fits the model context is classified
  as LOCAL_AGENT_SESSION_INVALID so the existing fresh-session retry
  rebuilds it; Rust panics and JSON parse errors no longer discard a
  healthy session.
- Automatic retries no longer re-run the whole Agent catalog refresh
  before each attempt, which flipped busy Agents to unavailable.

Checks: failure-signals (new), failure-policy, cli-process-support,
cli-adapters-agent-protocols, agent-runtime-contract and the auth/retry
tests in local-workspace and local-workspace-auto pass on this tree,
including three readiness tests that failed on bb55fad.
This commit is contained in:
Ryder Sun committed 2026-10-15 17:37:57 +08:00
1 parent 1033d086d3
commit 2da40db63d
12 files changed
+386 -43

No files matched your search

+21 -7
View File
@@ -1,3 +1,4 @@
const { authFailureCertainty } = require('../runs/failure-signals.cjs')
const AGENT_OUTCOMES = Object.freeze([
'accepted',
'running',
@@ -219,19 +220,25 @@ function terminalAuthenticationDiagnostic(value) {
: ''
}
// True only for explicit sign-in or API-key statements. Ambiguous refusals
// (HTTP 403, proxy or region blocks, "provider rejected") are classified as
// LOCAL_AGENT_ACCESS_DENIED so they are retried instead of signing the Agent out.
function authConfigurationFailure(detail) {
return String(detail || '').split(/\r?\n/).some((line) => {
const text = line.trim().replace(/^(?:error|fatal)\s*:\s*/i, '')
return Boolean(terminalAuthenticationDiagnostic(text))
|| /^(?:(?:invalid|expired|missing|incorrect)\s+(?:api[ _-]?key|(?:access|refresh|auth)[ _-]?token|credentials?)\b|(?:authentication|authorization)\s+(?:failed|required)\b|(?:please\s+)?(?:log|sign)\s+in\b|not\s+logged\s+in\b|select an auth type\b|provider rejected\b)/i.test(text)
|| /^(?:认证|鉴权|登录)(?:失败|过期)|^(?:令牌|凭据|密钥)(?:无效|缺失|已过期)|^请(?:先|重新)登录/.test(text)
})
return authFailureCertainty(detail) === 'definitive'
}
function authFailureCode(detail) {
const certainty = authFailureCertainty(detail)
if (certainty === 'definitive') return 'LOCAL_AGENT_AUTH_REQUIRED'
if (certainty === 'ambiguous') return 'LOCAL_AGENT_ACCESS_DENIED'
return ''
}
function failureCategory(code) {
const value = String(code || '')
if (value === 'LOCAL_AGENT_SESSION_INVALID') return 'session'
if (value === 'LOCAL_AGENT_AUTH_REQUIRED') return 'authentication'
if (value === 'LOCAL_AGENT_ACCESS_DENIED') return 'provider'
if (value === 'LOCAL_AGENT_TIMEOUT') return 'timeout'
if (value === 'LOCAL_AGENT_EXECUTION_STOPPED') return 'cancellation'
if (value === 'LOCAL_AGENT_REFUSED') return 'refusal'
@@ -317,6 +324,9 @@ function requireTerminalAgentResult(result) {
// error, so the catalog can mark the Agent as needing a fresh login.
if (failure.category !== 'authentication' && authConfigurationFailure(diagnostic)) {
failure = agentRuntimeFailure('LOCAL_AGENT_AUTH_REQUIRED')
} else if (failure.code === 'LOCAL_AGENT_PROCESS_FAILED'
&& authFailureCertainty(normalized.diagnostic) === 'ambiguous') {
failure = agentRuntimeFailure('LOCAL_AGENT_ACCESS_DENIED')
}
throw agentRuntimeError(
failure.code,
@@ -326,7 +336,10 @@ function requireTerminalAgentResult(result) {
}
const authenticationDiagnostic = terminalAuthenticationDiagnostic(normalized.text)
if (authenticationDiagnostic) {
throw agentRuntimeError('LOCAL_AGENT_AUTH_REQUIRED', authenticationDiagnostic)
throw agentRuntimeError(
authFailureCode(authenticationDiagnostic) || 'LOCAL_AGENT_ACCESS_DENIED',
authenticationDiagnostic,
)
}
return normalized
}
@@ -355,6 +368,7 @@ module.exports = {
agentRuntimeError,
agentRuntimeFailure,
authConfigurationFailure,
authFailureCode,
isCodeReviewAgentKind,
isReviewOnlyAgentKind,
normalizeAgentResult,
+13 -5
View File
@@ -4,8 +4,9 @@ const { NETWORK_ENV_KEYS, networkEnvironment } = require('./cli-network-environm
const { redactChildSecrets } = require('./cli-runtime-events.cjs')
const {
agentRuntimeError,
authConfigurationFailure,
authFailureCode,
} = require('../agent-runtime-contract.cjs')
const { authFailureCertainty, isContextOverflow } = require('../../runs/failure-signals.cjs')
const { validateOpenClawRuntimeGuard } = require('./openclaw-runtime.cjs')
const TERMINATE_GRACE_MS = 500
@@ -34,8 +35,17 @@ function agentExecutionError(code, diagnostic = '') {
function invalidSessionFailure(detail) {
const text = String(detail || '').trim()
if (!text || authConfigurationFailure(text)
if (!text || authFailureCertainty(text)
|| /\b(?:api|access|refresh|auth|session)[ _-]?token\b/i.test(text)) return false
// A resumed native session that no longer fits the model context cannot be
// continued; rebuilding it with compact continuation context is the fix.
if (isContextOverflow(text)) return true
// Runtime crashes ("thread 'main' panicked ... invalid ...") and parser
// errors mention threads or invalid input without saying anything about the
// stored session. Discarding a healthy session for them loses context.
if (/\bpanicked\b|\bSyntaxError\b|\binvalid JSON\b|\bUnexpected token\b|\bstack backtrace\b/i.test(text)) {
return false
}
return /\b(?:no|unknown|invalid|expired|missing|stale)\s+(?:saved\s+)?(?:session|conversation|thread)\b|\b(?:session|conversation|thread)(?:\s+(?:id|key|reference))?\b[^\n]{0,100}\b(?:not found|does not exist|no longer exists|invalid|unknown|expired|has expired|was deleted|cannot be resumed)\b/i
.test(text)
}
@@ -45,9 +55,7 @@ function failedAgentProcessError(detail, options = {}) {
return agentExecutionError(
options.sessionRef && invalidSessionFailure(detail)
? 'LOCAL_AGENT_SESSION_INVALID'
: authConfigurationFailure(detail)
? 'LOCAL_AGENT_AUTH_REQUIRED'
: 'LOCAL_AGENT_PROCESS_FAILED',
: authFailureCode(detail) || 'LOCAL_AGENT_PROCESS_FAILED',
detail,
)
}
+51 -7
View File
@@ -1,3 +1,11 @@
const {
authFailureCertainty,
isNetworkFailure,
isProviderUnavailable,
isRateLimited,
retryAfterFromText,
} = require('./failure-signals.cjs')
const FAILURE_CATEGORIES = Object.freeze([
'authentication',
'compatibility',
@@ -121,23 +129,47 @@ function retryAfterMs(error, now) {
const parsed = parseRetryAfter(headerValue(headers, 'retry-after'), now)
if (parsed != null) return parsed
}
return null
// Local CLIs surface provider throttling as text ("retry after 20s").
return retryAfterFromText(error?.diagnostic || error?.message)
}
// A local CLI process failure carries a code-derived category (usually
// protocol). Its diagnostic often proves a transient provider condition, so
// refine it before deciding whether to retry and how long to wait.
function transientCategoryFromText(detail) {
if (!detail) return ''
if (isRateLimited(detail)) return 'rate_limit'
if (isNetworkFailure(detail)) return 'network'
if (isProviderUnavailable(detail)) return 'provider'
return ''
}
function classifiedCategory(error, code, status) {
const explicit = String(error?.failure?.category || '')
if (FAILURE_CATEGORY_SET.has(explicit)) return explicit
const detail = String(error?.diagnostic || error?.message || '')
if (FAILURE_CATEGORY_SET.has(explicit)) {
if (['protocol', 'execution'].includes(explicit)) {
const refined = transientCategoryFromText(error?.diagnostic || '')
if (refined) return refined
}
return explicit
}
if (code === 'LOCAL_BUDGET_EXHAUSTED') return 'budget'
if (/PERMISSION/.test(code)) return 'permission'
if (code === 'LOCAL_AGENT_EXECUTION_STOPPED' || /CANCEL|ABORT/.test(code)) {
return 'cancellation'
}
if (code === 'LOCAL_AGENT_SESSION_INVALID') return 'session'
if (status === 401 || status === 403 || code === 'LOCAL_AGENT_AUTH_REQUIRED'
|| /unauthori[sz]ed|forbidden|invalid token/i.test(detail)) return 'authentication'
// Only explicit sign-in statements prove that credentials are gone. A 403,
// a proxy refusal, or "unauthorized" inside unrelated output is retried as
// a provider-side refusal instead of stopping the Agent for the run.
const authCertainty = authFailureCertainty(detail)
if (status === 401 || code === 'LOCAL_AGENT_AUTH_REQUIRED'
|| authCertainty === 'definitive') return 'authentication'
if (status === 403 || code === 'LOCAL_AGENT_ACCESS_DENIED'
|| authCertainty === 'ambiguous') return 'provider'
if (status === 429 || /RATE_LIMIT|TOO_MANY_REQUESTS/.test(code)
|| /rate limit|too many requests/i.test(detail)) return 'rate_limit'
|| isRateLimited(detail)) return 'rate_limit'
if (NETWORK_CODES.has(code) || /NETWORK|CONNECTION/.test(code)
|| /socket hang up|temporary failure|network error/i.test(detail)) return 'network'
if (status != null && status >= 500) return 'provider'
@@ -244,18 +276,30 @@ function retryDecision(input, options = {}) {
if (safety?.sideEffectsPossible === true && safety.idempotencyMode !== 'durable') {
return Object.freeze({ ...base, action: 'human_gate', exhausted: false })
}
const maxDelayMs = Math.max(1, boundedInteger(
const configuredMaxDelayMs = Math.max(1, boundedInteger(
options.maxDelayMs, 10 * 1000, 24 * 60 * 60 * 1000,
))
// Provider throttling needs a longer and server-directed wait than a crashed
// process. Honor Retry-After up to the rate-limit cap instead of hammering
// the provider with the generic sub-second backoff.
const rateLimited = failure.category === 'rate_limit'
const maxDelayMs = rateLimited
? Math.max(configuredMaxDelayMs, boundedInteger(
options.rateLimitMaxDelayMs, 60 * 1000, 24 * 60 * 60 * 1000,
))
: configuredMaxDelayMs
const backoff = boundedBackoffDelay(attempt, { ...options, maxDelayMs })
const requestedDelay = failure.retryAfterMs == null
? 0
: Math.min(maxDelayMs, boundedInteger(failure.retryAfterMs, 0, maxDelayMs))
const minimumDelay = rateLimited && failure.retryAfterMs == null
? Math.min(maxDelayMs, boundedInteger(options.rateLimitMinDelayMs, 0, maxDelayMs))
: 0
return Object.freeze({
...base,
action: 'retry',
exhausted: false,
delayMs: Math.max(backoff, requestedDelay),
delayMs: Math.max(backoff, requestedDelay, minimumDelay),
})
}
+129
View File
@@ -0,0 +1,129 @@
// Pure text classifiers for local Agent failure diagnostics.
//
// Agent CLIs report failures as free text on stderr or inside structured error
// results. Classification decides whether Meldwork retries, asks the user to
// sign in again, or rebuilds a native session, so every rule here errs on the
// side of "ambiguous": only explicit login and API-key statements count as a
// definitive authentication failure. A bare 403, a proxy refusal, or a word
// such as "unauthorized" inside unrelated output must never mark an installed
// Agent as signed out.
const MAX_SIGNAL_TEXT_CHARS = 64 * 1024
const LOG_PREFIX = /^(?:\[[^\]]{1,40}\]\s*|(?:error|fatal|err|warning)\s*[:!-]\s*|[✗✘×!]\s*)+/i
const DEFINITIVE_AUTH_LINE = [
/^(?:invalid|expired|missing|incorrect|revoked)\s+(?:api[ _-]?key|(?:access|refresh|auth|oauth|bearer)[ _-]?token|credentials?)\b/i,
/^(?:authentication|authorization)\s+(?:failed|required)\b/i,
/^(?:please\s+)?(?:log|sign)\s+in\b/i,
/^(?:please\s+)?run\s+\S*\s*(?:\/)?login\b/i,
/^not\s+(?:logged|signed)\s+in\b/i,
/^you\s+are\s+not\s+(?:logged|signed)\s+in\b/i,
/^select an auth type\b/i,
/^oauth\s+token\s+(?:has\s+)?(?:expired|is\s+invalid|was\s+revoked)\b/i,
/^HTTP\s+401\s*:/i,
/^provider rejected\b/i,
/^(?:认证|鉴权|登录)(?:失败|过期)/,
/^(?:令牌|凭据|密钥)(?:无效|缺失|已过期)/,
/^请(?:先|重新)登录/,
/^未登录/,
]
const DEFINITIVE_AUTH_ANYWHERE = /"type"\s*:\s*"authentication_error"|\binvalid_api_key\b|\bincorrect api key provided\b|\binvalid x-api-key\b/i
// Ambiguous refusals must lead a diagnostic line. Tool output such as
// "Failed to download docs: HTTP 403 Forbidden" mentions a refusal of some
// unrelated resource and stays a plain process failure.
const AMBIGUOUS_AUTH_LINE = [
/^HTTP\s+403\s*:/i,
/^(?:403\s+)?forbidden\b/i,
/^unauthori[sz]ed\b/i,
/^access denied\b/i,
/^invalid token\b/i,
]
const AMBIGUOUS_AUTH_ANYWHERE = /"type"\s*:\s*"permission_error"/i
const RATE_LIMIT = /\b429\b|too many requests|rate[ _-]?limit(?:ed|ing)?\b|\boverloaded(?:_error)?\b|\bquota (?:exceeded|exhausted)\b|\bresource[ _-]?exhausted\b|\binsufficient_quota\b|\b529\b/i
const NETWORK = /\b(?:ECONNRESET|ECONNREFUSED|ECONNABORTED|ENOTFOUND|ETIMEDOUT|EAI_AGAIN|ENETUNREACH|ENETDOWN|EHOSTUNREACH)\b|socket hang up|fetch failed|network (?:error|is unreachable)|connection (?:reset|refused|closed|timed out)|getaddrinfo|tls handshake|ssl handshake|proxy (?:error|connection)/i
const PROVIDER_UNAVAILABLE = /\b(?:HTTP\s*)?50[0234]\b\s*(?::|-)?\s*(?:internal server error|bad gateway|service unavailable|gateway timeout)|\bservice unavailable\b|\bbad gateway\b|\bgateway timeout\b|\binternal server error\b|"type"\s*:\s*"api_error"|\bupstream connect error\b/i
const CONTEXT_OVERFLOW = /prompt is too long|context[_ ]length[_ ]exceeded|maximum context length|context window (?:exceeded|is full|limit)|input is too long|too many (?:input )?tokens|exceeds? the (?:maximum|model'?s?) context|context_window_exceeded|request too large for model/i
function signalText(value) {
return String(value || '').slice(0, MAX_SIGNAL_TEXT_CHARS)
}
function diagnosticLines(value) {
return signalText(value)
.split(/\r?\n/)
.map(line => line.trim().replace(LOG_PREFIX, '').trim())
.filter(Boolean)
}
// Returns 'definitive', 'ambiguous', or ''.
function authFailureCertainty(value) {
const text = signalText(value)
if (!text.trim()) return ''
const lines = diagnosticLines(text)
if (DEFINITIVE_AUTH_ANYWHERE.test(text)
|| lines.some(line => DEFINITIVE_AUTH_LINE.some(pattern => pattern.test(line)))) {
return 'definitive'
}
if (lines.some(line => AMBIGUOUS_AUTH_LINE.some(pattern => pattern.test(line)))
|| AMBIGUOUS_AUTH_ANYWHERE.test(text)) {
return 'ambiguous'
}
return ''
}
function isRateLimited(value) {
return RATE_LIMIT.test(signalText(value))
}
function isNetworkFailure(value) {
return NETWORK.test(signalText(value))
}
function isProviderUnavailable(value) {
return PROVIDER_UNAVAILABLE.test(signalText(value))
}
function isContextOverflow(value) {
return CONTEXT_OVERFLOW.test(signalText(value))
}
const RETRY_AFTER_PATTERNS = [
/retry[- ]after["']?\s*[:=]?\s*(\d+(?:\.\d+)?)\s*(ms|milliseconds?|s|secs?|seconds?|m|mins?|minutes?)?/i,
/(?:try|retry) again in\s+(\d+(?:\.\d+)?)\s*(ms|milliseconds?|s|secs?|seconds?|m|mins?|minutes?)?/i,
/(?:reset|resets) in\s+(\d+(?:\.\d+)?)\s*(ms|milliseconds?|s|secs?|seconds?|m|mins?|minutes?)/i,
]
const MAX_RETRY_AFTER_MS = 5 * 60 * 1000
function retryAfterFromText(value) {
const text = signalText(value)
for (const pattern of RETRY_AFTER_PATTERNS) {
const match = text.match(pattern)
if (!match) continue
const amount = Number(match[1])
if (!Number.isFinite(amount) || amount < 0) continue
const unit = String(match[2] || 's').toLowerCase()
const multiplier = unit.startsWith('ms') || unit.startsWith('milli')
? 1
: unit.startsWith('m') ? 60 * 1000 : 1000
return Math.min(MAX_RETRY_AFTER_MS, Math.round(amount * multiplier))
}
return null
}
module.exports = {
authFailureCertainty,
isContextOverflow,
isNetworkFailure,
isProviderUnavailable,
isRateLimited,
retryAfterFromText,
}
@@ -24,7 +24,6 @@ const {
attachmentType,
cleanText,
cleanProgressSteps,
credentialFailure,
normalizeAttachmentMetadata,
parseAutoReply,
settleWithin,
@@ -1958,8 +1957,11 @@ class LocalWorkspaceAgentInvocation {
: agentCancelled
? agentStoppedError()
: caughtError
if (credentialFailure(error) && context.deferCredentialFailure !== true) {
this.markRuntimeCredential(kind, 'unknown')
// Only an explicit sign-in failure changes readiness. Network, quota and
// provider refusals stay run-scoped so a healthy Agent remains usable.
if (error?.code === 'LOCAL_AGENT_AUTH_REQUIRED'
&& context.deferCredentialFailure !== true) {
this.markRuntimeCredential(kind, 'missing')
}
const status = parentTimedOut
? 'timeout'
@@ -3,7 +3,6 @@ const { MAX_MESSAGE_TEXT_CHARS } = require('./local-workspace-contracts.cjs')
const {
cleanText,
credentialFailure,
normalizeKnowledgeBaseHint,
normalizeSkillHint,
} = require('./local-workspace-inputs.cjs')
@@ -123,6 +122,11 @@ class LocalWorkspaceAutoRunner {
this.retryMaxDelayMs = Number.isFinite(options.retryMaxDelayMs)
? Math.max(this.retryBaseDelayMs, Math.floor(options.retryMaxDelayMs))
: 2000
// Provider throttling without a Retry-After hint waits at least this long
// so automatic retries do not burn the remaining quota within a second.
this.retryRateLimitMinDelayMs = Number.isFinite(options.retryRateLimitMinDelayMs)
? Math.max(0, Math.floor(options.retryRateLimitMinDelayMs))
: 2000
this.sleep = options.retrySleep || abortableDelay
}
@@ -306,14 +310,18 @@ class LocalWorkspaceAutoRunner {
}
// Session recovery is owned by LocalWorkspaceAgentInvocation so a stale
// native Session gets exactly one fresh attempt across every run mode.
// An ambiguous refusal (HTTP 403, proxy or region block) gets exactly
// one more attempt: a transient gateway refusal recovers, while a real
// permission problem fails fast without signing the Agent out.
const maxAttempts = failure.category === 'timeout' && options.retryTimeouts !== true
? 1
: 4
: failure.code === 'LOCAL_AGENT_ACCESS_DENIED' ? 2 : 4
const decision = retryDecision(failure, {
attempt,
maxAttempts,
baseDelayMs: this.retryBaseDelayMs,
maxDelayMs: this.retryMaxDelayMs,
rateLimitMinDelayMs: this.retryRateLimitMinDelayMs,
safety,
})
if (decision.action === 'human_gate') {
@@ -403,10 +411,9 @@ class LocalWorkspaceAutoRunner {
finalOutcome: 'failed',
...safety,
})
this.markRuntimeCredential(kind, 'unknown')
try { await this.refreshAgents() } catch {
if (controller.signal.aborted) throw new Error('LOCAL_AGENT_EXECUTION_STOPPED')
}
// Transient retries keep the Agent's readiness untouched. Refreshing
// the whole catalog here re-ran discovery and credential probes while
// the Agent was busy and could flip a healthy Agent to unavailable.
await this.sleep(decision.delayMs, controller.signal)
}
}
@@ -524,11 +531,11 @@ class LocalWorkspaceAutoRunner {
continue
}
if (control) return { result: null, removed: false, control, error }
if (!unauthorizedFailure(error)) {
if (credentialFailure(error)) this.markRuntimeCredential(kind, 'unknown')
throw error
}
this.markRuntimeCredential(kind, 'unknown')
if (!unauthorizedFailure(error)) throw error
// Only explicit sign-in failures reach this point. Mark the Agent as
// needing a fresh login; the catalog clears it on the next refresh
// after the retry window or as soon as native credentials verify.
this.markRuntimeCredential(kind, 'missing')
this.recordAttempt(group, controller, {
agentKind: kind,
phase,
@@ -34,11 +34,6 @@ function defaultAgentLabel(kind) {
return AGENT_LABELS[kind] || String(kind || 'Agent')
}
function credentialFailure(error) {
return /api[ _-]?key|credential|auth(?:entication|orization)?|login|log in|unauthorized|forbidden|401|403|令牌|凭据|登录|认证/i
.test(String(error?.message || error || ''))
}
function cleanText(value, limit = MAX_MESSAGE_TEXT_CHARS) {
return String(value || '').trim().slice(0, limit)
}
@@ -61,7 +56,6 @@ module.exports = {
STABLE_USER_TURNS_PER_EDGE,
cleanInline,
cleanText,
credentialFailure,
defaultAgentLabel,
isSupportedAgentKind,
}
@@ -40,7 +40,6 @@ module.exports = {
cleanProgressSteps: runtimeContracts.cleanProgressSteps,
cleanRunMaxRounds: runtimeContracts.cleanRunMaxRounds,
cleanText: contracts.cleanText,
credentialFailure: contracts.credentialFailure,
defaultAgentLabel: contracts.defaultAgentLabel,
emptyState: messageRecords.emptyState,
isTracedAgentTerminalMessage: runtimeContracts.isTracedAgentTerminalMessage,
+15 -1
View File
@@ -121,13 +121,27 @@ test('uses injectable jitter while keeping backoff and Retry-After bounded', ()
response: { headers: { 'Retry-After': '3' } },
}, { now: () => fakeNow })
assert.equal(limited.category, 'rate_limit')
// Provider throttling honors Retry-After beyond the generic process-retry
// cap, bounded by the rate-limit cap (60 s by default).
assert.equal(retryDecision(limited, {
attempt: 1,
maxAttempts: 4,
baseDelayMs: 250,
maxDelayMs: 2000,
jitter: raw => raw / 2,
}).delayMs, 2000)
}).delayMs, 3000)
const slow = normalizeFailure({
status: 429,
response: { headers: { 'Retry-After': '120' } },
}, { now: () => fakeNow })
assert.equal(retryDecision(slow, { attempt: 1, maxDelayMs: 2000 }).delayMs, 60000)
assert.equal(retryDecision(slow, {
attempt: 1, maxDelayMs: 2000, rateLimitMaxDelayMs: 5000,
}).delayMs, 5000)
const unhinted = normalizeFailure({ status: 429 }, { now: () => fakeNow })
assert.equal(retryDecision(unhinted, {
attempt: 1, baseDelayMs: 250, maxDelayMs: 2000, rateLimitMinDelayMs: 1500,
}).delayMs, 1500)
})
test('retries only transient categories and stops at the total attempt bound', () => {
@@ -0,0 +1,88 @@
const test = require('node:test')
const assert = require('node:assert/strict')
const {
authFailureCertainty,
isContextOverflow,
isNetworkFailure,
isProviderUnavailable,
isRateLimited,
retryAfterFromText,
} = require('../../src/runs/failure-signals.cjs')
test('explicit sign-in and API key statements are definitive authentication failures', () => {
for (const text of [
'Please log in first',
'Error: Not logged in · Please run /login',
'Invalid API key · Please run /login',
'[ERROR] expired access token',
'OAuth token has expired. Please obtain a new token.',
'HTTP 401: Unauthorized',
'API Error: 401 {"type":"error","error":{"type":"authentication_error","message":"invalid x-api-key"}}',
'Incorrect API key provided: sk-***',
'select an auth type',
'Provider rejected sk-redacted',
'请先登录',
'认证失败,请检查密钥',
]) {
assert.equal(authFailureCertainty(text), 'definitive', text)
}
})
test('refusals that do not prove missing credentials are only ambiguous', () => {
for (const text of [
'HTTP 403: Forbidden',
'API Error: 403 {"type":"error","error":{"type":"permission_error"}}',
'403 Forbidden',
'Error: Unauthorized',
]) {
assert.equal(authFailureCertainty(text), 'ambiguous', text)
}
})
test('unrelated diagnostics never look like authentication failures', () => {
for (const text of [
'',
'thread \'main\' panicked at src/session.rs: invalid utf-8',
'SyntaxError: Unexpected token } in JSON at position 4',
'Compiled 12 files; login page renders correctly',
'socket hang up',
'Failed to download documentation: HTTP 403 Forbidden',
'request failed: unauthorized resource in fixture',
]) {
assert.equal(authFailureCertainty(text), '', text)
}
})
test('transient provider conditions are recognized from CLI text', () => {
assert.equal(isRateLimited('Error: 429 Too Many Requests'), true)
assert.equal(isRateLimited('{"type":"overloaded_error"}'), true)
assert.equal(isRateLimited('Rate limit reached for gpt-5 in organization'), true)
assert.equal(isRateLimited('All 12 tests passed'), false)
assert.equal(isNetworkFailure('fetch failed: getaddrinfo ENOTFOUND api.example.com'), true)
assert.equal(isNetworkFailure('read ECONNRESET'), true)
assert.equal(isNetworkFailure('Wrote network.md'), false)
assert.equal(isProviderUnavailable('HTTP 503: Service Unavailable'), true)
assert.equal(isProviderUnavailable('{"type":"error","error":{"type":"api_error"}}'), true)
assert.equal(isProviderUnavailable('Processed 500 records'), false)
})
test('context overflow is detected across provider wordings', () => {
for (const text of [
'Prompt is too long: 210000 tokens > 200000 maximum',
'This model\'s maximum context length is 128000 tokens',
'context_length_exceeded',
'Input is too long for requested model.',
]) {
assert.equal(isContextOverflow(text), true, text)
}
assert.equal(isContextOverflow('The context of this change is the login flow'), false)
})
test('retry hints in CLI text become bounded delays', () => {
assert.equal(retryAfterFromText('429 Too Many Requests; retry after 20s'), 20000)
assert.equal(retryAfterFromText('Rate limited. Please try again in 1.5 seconds.'), 1500)
assert.equal(retryAfterFromText('retry-after: 750ms'), 750)
assert.equal(retryAfterFromText('quota resets in 2 minutes'), 120000)
assert.equal(retryAfterFromText('try again in 90 minutes'), 300000)
assert.equal(retryAfterFromText('no hint here'), null)
})
@@ -4563,11 +4563,56 @@ test('automatic dialogue honors bounded Retry-After backoff and recovers', async
assert.deepEqual(calls.map(call => call.agent.kind), [
'hermes', 'hermes', 'hermes', 'hermes', 'codex',
])
assert.deepEqual(delays, [4, 4, 4])
// Provider throttling honors the server's Retry-After instead of the
// generic process-retry cap, still bounded by the rate-limit cap.
assert.deepEqual(delays, [10, 10, 10])
assert.equal(new Set(calls.slice(0, 4).map(call => call.runOptions.operationId)).size, 1)
assert.equal(finished[0].status, 'completed')
})
test('automatic dialogue retries an ambiguous 403 and network errors without touching readiness', async (t) => {
const { directory, calls, options } = fixture()
t.after(() => fs.rmSync(directory, { recursive: true, force: true }))
const delays = []
const failures = [
agentRuntimeError('LOCAL_AGENT_PROCESS_FAILED', 'HTTP 403: Forbidden'),
agentRuntimeError('LOCAL_AGENT_PROCESS_FAILED', 'fetch failed: read ECONNRESET'),
]
options.retrySleep = async delayMs => { delays.push(delayMs) }
options.runAgent = async (agent, prompt, workdir, runOptions) => {
calls.push({ agent, prompt, workdir, runOptions })
if (agent.kind === 'hermes' && failures.length) throw failures.shift()
return {
text: `${agent.kind} agrees\n[[MELDWORK_CONSENSUS:agree]]`,
sessionRef: runOptions.sessionRef || `${agent.kind}-session`,
}
}
const workspace = new LocalWorkspace(options)
const finished = []
workspace.on('run-finished', result => finished.push(result))
await workspace.refreshAgents()
const readyBefore = workspace.snapshot().agents.find(agent => agent.kind === 'hermes').available
const group = workspace.createGroup({
name: 'Transient refusals', agentKinds: ['hermes', 'codex'], workdir: directory,
allowWrite: false,
})
workspace.addMessage(group.id, 'user', 'Recover from transient refusals')
workspace.startAuto({ groupId: group.id, maxRounds: 1 })
await workspace.activeRuns.get(group.id).promise
assert.deepEqual(calls.map(call => call.agent.kind), ['hermes', 'hermes', 'hermes', 'codex'])
assert.equal(delays.length, 2)
assert.equal(finished[0].status, 'completed')
const hermes = workspace.snapshot().agents.find(agent => agent.kind === 'hermes')
assert.equal(hermes.available, readyBefore)
assert.notEqual(hermes.credentialState, 'missing')
assert.deepEqual(workspace.getGroup(group.id).agentKinds, ['hermes', 'codex'])
assert.equal(workspace.snapshot().messages.some(message => (
message.system?.key === 'system.agentPausedAfterFailure'
)), false)
})
test('automatic dialogue retries one protocol failure without removing the Agent', async (t) => {
const { directory, calls, options } = fixture()
t.after(() => fs.rmSync(directory, { recursive: true, force: true }))
@@ -45,7 +45,6 @@ const EXPECTED_EXPORTS = [
'cleanProgressSteps',
'cleanRunMaxRounds',
'cleanText',
'credentialFailure',
'defaultAgentLabel',
'emptyState',
'isTracedAgentTerminalMessage',