mirror of
https://github.com/Ryder-Sun/Meldwork.git
synced 2026-11-07 06:56:03 -10:00
fix: tier authentication failures and retry transient provider errors
The merged branches disagreed on authentication failures: feature/1.0.5 marked every runtime auth failure 'unknown' so readiness never flipped, while main's22f811cclassified any auth-looking diagnostic as LOCAL_AGENT_AUTH_REQUIRED. Together with broad regexes (a bare "forbidden" anywhere, a credential regex matching "login" or "403"), proxy refusals stopped Agents while real sign-outs left them "ready". - New pure module runs/failure-signals.cjs classifies CLI text: explicit sign-in or API-key statements are definitive; line-leading 403/forbidden/unauthorized and permission_error are ambiguous; tool output that merely mentions 403 is neither. It also recognises rate limits (with Retry-After hints), network failures, provider 5xx and context overflow. - Definitive failures stay LOCAL_AGENT_AUTH_REQUIRED and now mark the credential 'missing' (the catalog clears it after the retry window or when native credentials verify). Ambiguous refusals become LOCAL_AGENT_ACCESS_DENIED, are retried once, and never touch readiness. - Process failures whose diagnostics show throttling, network loss or a provider outage are retried in that category. Rate limits honour Retry-After up to 60 s and wait at least 2 s without a hint. - A resumed session that no longer fits the model context is classified as LOCAL_AGENT_SESSION_INVALID so the existing fresh-session retry rebuilds it; Rust panics and JSON parse errors no longer discard a healthy session. - Automatic retries no longer re-run the whole Agent catalog refresh before each attempt, which flipped busy Agents to unavailable. Checks: failure-signals (new), failure-policy, cli-process-support, cli-adapters-agent-protocols, agent-runtime-contract and the auth/retry tests in local-workspace and local-workspace-auto pass on this tree, including three readiness tests that failed onbb55fad.
This commit is contained in:
1 parent
1033d086d3
commit
2da40db63d
12 files changed
+386
-43
No files matched your search
@@ -1,3 +1,4 @@
|
||||
const { authFailureCertainty } = require('../runs/failure-signals.cjs')
|
||||
const AGENT_OUTCOMES = Object.freeze([
|
||||
'accepted',
|
||||
'running',
|
||||
@@ -219,19 +220,25 @@ function terminalAuthenticationDiagnostic(value) {
|
||||
: ''
|
||||
}
|
||||
|
||||
// True only for explicit sign-in or API-key statements. Ambiguous refusals
|
||||
// (HTTP 403, proxy or region blocks, "provider rejected") are classified as
|
||||
// LOCAL_AGENT_ACCESS_DENIED so they are retried instead of signing the Agent out.
|
||||
function authConfigurationFailure(detail) {
|
||||
return String(detail || '').split(/\r?\n/).some((line) => {
|
||||
const text = line.trim().replace(/^(?:error|fatal)\s*:\s*/i, '')
|
||||
return Boolean(terminalAuthenticationDiagnostic(text))
|
||||
|| /^(?:(?:invalid|expired|missing|incorrect)\s+(?:api[ _-]?key|(?:access|refresh|auth)[ _-]?token|credentials?)\b|(?:authentication|authorization)\s+(?:failed|required)\b|(?:please\s+)?(?:log|sign)\s+in\b|not\s+logged\s+in\b|select an auth type\b|provider rejected\b)/i.test(text)
|
||||
|| /^(?:认证|鉴权|登录)(?:失败|过期)|^(?:令牌|凭据|密钥)(?:无效|缺失|已过期)|^请(?:先|重新)登录/.test(text)
|
||||
})
|
||||
return authFailureCertainty(detail) === 'definitive'
|
||||
}
|
||||
|
||||
function authFailureCode(detail) {
|
||||
const certainty = authFailureCertainty(detail)
|
||||
if (certainty === 'definitive') return 'LOCAL_AGENT_AUTH_REQUIRED'
|
||||
if (certainty === 'ambiguous') return 'LOCAL_AGENT_ACCESS_DENIED'
|
||||
return ''
|
||||
}
|
||||
|
||||
function failureCategory(code) {
|
||||
const value = String(code || '')
|
||||
if (value === 'LOCAL_AGENT_SESSION_INVALID') return 'session'
|
||||
if (value === 'LOCAL_AGENT_AUTH_REQUIRED') return 'authentication'
|
||||
if (value === 'LOCAL_AGENT_ACCESS_DENIED') return 'provider'
|
||||
if (value === 'LOCAL_AGENT_TIMEOUT') return 'timeout'
|
||||
if (value === 'LOCAL_AGENT_EXECUTION_STOPPED') return 'cancellation'
|
||||
if (value === 'LOCAL_AGENT_REFUSED') return 'refusal'
|
||||
@@ -317,6 +324,9 @@ function requireTerminalAgentResult(result) {
|
||||
// error, so the catalog can mark the Agent as needing a fresh login.
|
||||
if (failure.category !== 'authentication' && authConfigurationFailure(diagnostic)) {
|
||||
failure = agentRuntimeFailure('LOCAL_AGENT_AUTH_REQUIRED')
|
||||
} else if (failure.code === 'LOCAL_AGENT_PROCESS_FAILED'
|
||||
&& authFailureCertainty(normalized.diagnostic) === 'ambiguous') {
|
||||
failure = agentRuntimeFailure('LOCAL_AGENT_ACCESS_DENIED')
|
||||
}
|
||||
throw agentRuntimeError(
|
||||
failure.code,
|
||||
@@ -326,7 +336,10 @@ function requireTerminalAgentResult(result) {
|
||||
}
|
||||
const authenticationDiagnostic = terminalAuthenticationDiagnostic(normalized.text)
|
||||
if (authenticationDiagnostic) {
|
||||
throw agentRuntimeError('LOCAL_AGENT_AUTH_REQUIRED', authenticationDiagnostic)
|
||||
throw agentRuntimeError(
|
||||
authFailureCode(authenticationDiagnostic) || 'LOCAL_AGENT_ACCESS_DENIED',
|
||||
authenticationDiagnostic,
|
||||
)
|
||||
}
|
||||
return normalized
|
||||
}
|
||||
@@ -355,6 +368,7 @@ module.exports = {
|
||||
agentRuntimeError,
|
||||
agentRuntimeFailure,
|
||||
authConfigurationFailure,
|
||||
authFailureCode,
|
||||
isCodeReviewAgentKind,
|
||||
isReviewOnlyAgentKind,
|
||||
normalizeAgentResult,
|
||||
|
||||
@@ -4,8 +4,9 @@ const { NETWORK_ENV_KEYS, networkEnvironment } = require('./cli-network-environm
|
||||
const { redactChildSecrets } = require('./cli-runtime-events.cjs')
|
||||
const {
|
||||
agentRuntimeError,
|
||||
authConfigurationFailure,
|
||||
authFailureCode,
|
||||
} = require('../agent-runtime-contract.cjs')
|
||||
const { authFailureCertainty, isContextOverflow } = require('../../runs/failure-signals.cjs')
|
||||
const { validateOpenClawRuntimeGuard } = require('./openclaw-runtime.cjs')
|
||||
|
||||
const TERMINATE_GRACE_MS = 500
|
||||
@@ -34,8 +35,17 @@ function agentExecutionError(code, diagnostic = '') {
|
||||
|
||||
function invalidSessionFailure(detail) {
|
||||
const text = String(detail || '').trim()
|
||||
if (!text || authConfigurationFailure(text)
|
||||
if (!text || authFailureCertainty(text)
|
||||
|| /\b(?:api|access|refresh|auth|session)[ _-]?token\b/i.test(text)) return false
|
||||
// A resumed native session that no longer fits the model context cannot be
|
||||
// continued; rebuilding it with compact continuation context is the fix.
|
||||
if (isContextOverflow(text)) return true
|
||||
// Runtime crashes ("thread 'main' panicked ... invalid ...") and parser
|
||||
// errors mention threads or invalid input without saying anything about the
|
||||
// stored session. Discarding a healthy session for them loses context.
|
||||
if (/\bpanicked\b|\bSyntaxError\b|\binvalid JSON\b|\bUnexpected token\b|\bstack backtrace\b/i.test(text)) {
|
||||
return false
|
||||
}
|
||||
return /\b(?:no|unknown|invalid|expired|missing|stale)\s+(?:saved\s+)?(?:session|conversation|thread)\b|\b(?:session|conversation|thread)(?:\s+(?:id|key|reference))?\b[^\n]{0,100}\b(?:not found|does not exist|no longer exists|invalid|unknown|expired|has expired|was deleted|cannot be resumed)\b/i
|
||||
.test(text)
|
||||
}
|
||||
@@ -45,9 +55,7 @@ function failedAgentProcessError(detail, options = {}) {
|
||||
return agentExecutionError(
|
||||
options.sessionRef && invalidSessionFailure(detail)
|
||||
? 'LOCAL_AGENT_SESSION_INVALID'
|
||||
: authConfigurationFailure(detail)
|
||||
? 'LOCAL_AGENT_AUTH_REQUIRED'
|
||||
: 'LOCAL_AGENT_PROCESS_FAILED',
|
||||
: authFailureCode(detail) || 'LOCAL_AGENT_PROCESS_FAILED',
|
||||
detail,
|
||||
)
|
||||
}
|
||||
|
||||
@@ -1,3 +1,11 @@
|
||||
const {
|
||||
authFailureCertainty,
|
||||
isNetworkFailure,
|
||||
isProviderUnavailable,
|
||||
isRateLimited,
|
||||
retryAfterFromText,
|
||||
} = require('./failure-signals.cjs')
|
||||
|
||||
const FAILURE_CATEGORIES = Object.freeze([
|
||||
'authentication',
|
||||
'compatibility',
|
||||
@@ -121,23 +129,47 @@ function retryAfterMs(error, now) {
|
||||
const parsed = parseRetryAfter(headerValue(headers, 'retry-after'), now)
|
||||
if (parsed != null) return parsed
|
||||
}
|
||||
return null
|
||||
// Local CLIs surface provider throttling as text ("retry after 20s").
|
||||
return retryAfterFromText(error?.diagnostic || error?.message)
|
||||
}
|
||||
|
||||
// A local CLI process failure carries a code-derived category (usually
|
||||
// protocol). Its diagnostic often proves a transient provider condition, so
|
||||
// refine it before deciding whether to retry and how long to wait.
|
||||
function transientCategoryFromText(detail) {
|
||||
if (!detail) return ''
|
||||
if (isRateLimited(detail)) return 'rate_limit'
|
||||
if (isNetworkFailure(detail)) return 'network'
|
||||
if (isProviderUnavailable(detail)) return 'provider'
|
||||
return ''
|
||||
}
|
||||
|
||||
function classifiedCategory(error, code, status) {
|
||||
const explicit = String(error?.failure?.category || '')
|
||||
if (FAILURE_CATEGORY_SET.has(explicit)) return explicit
|
||||
const detail = String(error?.diagnostic || error?.message || '')
|
||||
if (FAILURE_CATEGORY_SET.has(explicit)) {
|
||||
if (['protocol', 'execution'].includes(explicit)) {
|
||||
const refined = transientCategoryFromText(error?.diagnostic || '')
|
||||
if (refined) return refined
|
||||
}
|
||||
return explicit
|
||||
}
|
||||
if (code === 'LOCAL_BUDGET_EXHAUSTED') return 'budget'
|
||||
if (/PERMISSION/.test(code)) return 'permission'
|
||||
if (code === 'LOCAL_AGENT_EXECUTION_STOPPED' || /CANCEL|ABORT/.test(code)) {
|
||||
return 'cancellation'
|
||||
}
|
||||
if (code === 'LOCAL_AGENT_SESSION_INVALID') return 'session'
|
||||
if (status === 401 || status === 403 || code === 'LOCAL_AGENT_AUTH_REQUIRED'
|
||||
|| /unauthori[sz]ed|forbidden|invalid token/i.test(detail)) return 'authentication'
|
||||
// Only explicit sign-in statements prove that credentials are gone. A 403,
|
||||
// a proxy refusal, or "unauthorized" inside unrelated output is retried as
|
||||
// a provider-side refusal instead of stopping the Agent for the run.
|
||||
const authCertainty = authFailureCertainty(detail)
|
||||
if (status === 401 || code === 'LOCAL_AGENT_AUTH_REQUIRED'
|
||||
|| authCertainty === 'definitive') return 'authentication'
|
||||
if (status === 403 || code === 'LOCAL_AGENT_ACCESS_DENIED'
|
||||
|| authCertainty === 'ambiguous') return 'provider'
|
||||
if (status === 429 || /RATE_LIMIT|TOO_MANY_REQUESTS/.test(code)
|
||||
|| /rate limit|too many requests/i.test(detail)) return 'rate_limit'
|
||||
|| isRateLimited(detail)) return 'rate_limit'
|
||||
if (NETWORK_CODES.has(code) || /NETWORK|CONNECTION/.test(code)
|
||||
|| /socket hang up|temporary failure|network error/i.test(detail)) return 'network'
|
||||
if (status != null && status >= 500) return 'provider'
|
||||
@@ -244,18 +276,30 @@ function retryDecision(input, options = {}) {
|
||||
if (safety?.sideEffectsPossible === true && safety.idempotencyMode !== 'durable') {
|
||||
return Object.freeze({ ...base, action: 'human_gate', exhausted: false })
|
||||
}
|
||||
const maxDelayMs = Math.max(1, boundedInteger(
|
||||
const configuredMaxDelayMs = Math.max(1, boundedInteger(
|
||||
options.maxDelayMs, 10 * 1000, 24 * 60 * 60 * 1000,
|
||||
))
|
||||
// Provider throttling needs a longer and server-directed wait than a crashed
|
||||
// process. Honor Retry-After up to the rate-limit cap instead of hammering
|
||||
// the provider with the generic sub-second backoff.
|
||||
const rateLimited = failure.category === 'rate_limit'
|
||||
const maxDelayMs = rateLimited
|
||||
? Math.max(configuredMaxDelayMs, boundedInteger(
|
||||
options.rateLimitMaxDelayMs, 60 * 1000, 24 * 60 * 60 * 1000,
|
||||
))
|
||||
: configuredMaxDelayMs
|
||||
const backoff = boundedBackoffDelay(attempt, { ...options, maxDelayMs })
|
||||
const requestedDelay = failure.retryAfterMs == null
|
||||
? 0
|
||||
: Math.min(maxDelayMs, boundedInteger(failure.retryAfterMs, 0, maxDelayMs))
|
||||
const minimumDelay = rateLimited && failure.retryAfterMs == null
|
||||
? Math.min(maxDelayMs, boundedInteger(options.rateLimitMinDelayMs, 0, maxDelayMs))
|
||||
: 0
|
||||
return Object.freeze({
|
||||
...base,
|
||||
action: 'retry',
|
||||
exhausted: false,
|
||||
delayMs: Math.max(backoff, requestedDelay),
|
||||
delayMs: Math.max(backoff, requestedDelay, minimumDelay),
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,129 @@
|
||||
// Pure text classifiers for local Agent failure diagnostics.
|
||||
//
|
||||
// Agent CLIs report failures as free text on stderr or inside structured error
|
||||
// results. Classification decides whether Meldwork retries, asks the user to
|
||||
// sign in again, or rebuilds a native session, so every rule here errs on the
|
||||
// side of "ambiguous": only explicit login and API-key statements count as a
|
||||
// definitive authentication failure. A bare 403, a proxy refusal, or a word
|
||||
// such as "unauthorized" inside unrelated output must never mark an installed
|
||||
// Agent as signed out.
|
||||
|
||||
const MAX_SIGNAL_TEXT_CHARS = 64 * 1024
|
||||
|
||||
const LOG_PREFIX = /^(?:\[[^\]]{1,40}\]\s*|(?:error|fatal|err|warning)\s*[:!-]\s*|[✗✘×!]\s*)+/i
|
||||
|
||||
const DEFINITIVE_AUTH_LINE = [
|
||||
/^(?:invalid|expired|missing|incorrect|revoked)\s+(?:api[ _-]?key|(?:access|refresh|auth|oauth|bearer)[ _-]?token|credentials?)\b/i,
|
||||
/^(?:authentication|authorization)\s+(?:failed|required)\b/i,
|
||||
/^(?:please\s+)?(?:log|sign)\s+in\b/i,
|
||||
/^(?:please\s+)?run\s+\S*\s*(?:\/)?login\b/i,
|
||||
/^not\s+(?:logged|signed)\s+in\b/i,
|
||||
/^you\s+are\s+not\s+(?:logged|signed)\s+in\b/i,
|
||||
/^select an auth type\b/i,
|
||||
/^oauth\s+token\s+(?:has\s+)?(?:expired|is\s+invalid|was\s+revoked)\b/i,
|
||||
/^HTTP\s+401\s*:/i,
|
||||
/^provider rejected\b/i,
|
||||
/^(?:认证|鉴权|登录)(?:失败|过期)/,
|
||||
/^(?:令牌|凭据|密钥)(?:无效|缺失|已过期)/,
|
||||
/^请(?:先|重新)登录/,
|
||||
/^未登录/,
|
||||
]
|
||||
|
||||
const DEFINITIVE_AUTH_ANYWHERE = /"type"\s*:\s*"authentication_error"|\binvalid_api_key\b|\bincorrect api key provided\b|\binvalid x-api-key\b/i
|
||||
|
||||
// Ambiguous refusals must lead a diagnostic line. Tool output such as
|
||||
// "Failed to download docs: HTTP 403 Forbidden" mentions a refusal of some
|
||||
// unrelated resource and stays a plain process failure.
|
||||
const AMBIGUOUS_AUTH_LINE = [
|
||||
/^HTTP\s+403\s*:/i,
|
||||
/^(?:403\s+)?forbidden\b/i,
|
||||
/^unauthori[sz]ed\b/i,
|
||||
/^access denied\b/i,
|
||||
/^invalid token\b/i,
|
||||
]
|
||||
|
||||
const AMBIGUOUS_AUTH_ANYWHERE = /"type"\s*:\s*"permission_error"/i
|
||||
|
||||
const RATE_LIMIT = /\b429\b|too many requests|rate[ _-]?limit(?:ed|ing)?\b|\boverloaded(?:_error)?\b|\bquota (?:exceeded|exhausted)\b|\bresource[ _-]?exhausted\b|\binsufficient_quota\b|\b529\b/i
|
||||
|
||||
const NETWORK = /\b(?:ECONNRESET|ECONNREFUSED|ECONNABORTED|ENOTFOUND|ETIMEDOUT|EAI_AGAIN|ENETUNREACH|ENETDOWN|EHOSTUNREACH)\b|socket hang up|fetch failed|network (?:error|is unreachable)|connection (?:reset|refused|closed|timed out)|getaddrinfo|tls handshake|ssl handshake|proxy (?:error|connection)/i
|
||||
|
||||
const PROVIDER_UNAVAILABLE = /\b(?:HTTP\s*)?50[0234]\b\s*(?::|-)?\s*(?:internal server error|bad gateway|service unavailable|gateway timeout)|\bservice unavailable\b|\bbad gateway\b|\bgateway timeout\b|\binternal server error\b|"type"\s*:\s*"api_error"|\bupstream connect error\b/i
|
||||
|
||||
const CONTEXT_OVERFLOW = /prompt is too long|context[_ ]length[_ ]exceeded|maximum context length|context window (?:exceeded|is full|limit)|input is too long|too many (?:input )?tokens|exceeds? the (?:maximum|model'?s?) context|context_window_exceeded|request too large for model/i
|
||||
|
||||
function signalText(value) {
|
||||
return String(value || '').slice(0, MAX_SIGNAL_TEXT_CHARS)
|
||||
}
|
||||
|
||||
function diagnosticLines(value) {
|
||||
return signalText(value)
|
||||
.split(/\r?\n/)
|
||||
.map(line => line.trim().replace(LOG_PREFIX, '').trim())
|
||||
.filter(Boolean)
|
||||
}
|
||||
|
||||
// Returns 'definitive', 'ambiguous', or ''.
|
||||
function authFailureCertainty(value) {
|
||||
const text = signalText(value)
|
||||
if (!text.trim()) return ''
|
||||
const lines = diagnosticLines(text)
|
||||
if (DEFINITIVE_AUTH_ANYWHERE.test(text)
|
||||
|| lines.some(line => DEFINITIVE_AUTH_LINE.some(pattern => pattern.test(line)))) {
|
||||
return 'definitive'
|
||||
}
|
||||
if (lines.some(line => AMBIGUOUS_AUTH_LINE.some(pattern => pattern.test(line)))
|
||||
|| AMBIGUOUS_AUTH_ANYWHERE.test(text)) {
|
||||
return 'ambiguous'
|
||||
}
|
||||
return ''
|
||||
}
|
||||
|
||||
function isRateLimited(value) {
|
||||
return RATE_LIMIT.test(signalText(value))
|
||||
}
|
||||
|
||||
function isNetworkFailure(value) {
|
||||
return NETWORK.test(signalText(value))
|
||||
}
|
||||
|
||||
function isProviderUnavailable(value) {
|
||||
return PROVIDER_UNAVAILABLE.test(signalText(value))
|
||||
}
|
||||
|
||||
function isContextOverflow(value) {
|
||||
return CONTEXT_OVERFLOW.test(signalText(value))
|
||||
}
|
||||
|
||||
const RETRY_AFTER_PATTERNS = [
|
||||
/retry[- ]after["']?\s*[:=]?\s*(\d+(?:\.\d+)?)\s*(ms|milliseconds?|s|secs?|seconds?|m|mins?|minutes?)?/i,
|
||||
/(?:try|retry) again in\s+(\d+(?:\.\d+)?)\s*(ms|milliseconds?|s|secs?|seconds?|m|mins?|minutes?)?/i,
|
||||
/(?:reset|resets) in\s+(\d+(?:\.\d+)?)\s*(ms|milliseconds?|s|secs?|seconds?|m|mins?|minutes?)/i,
|
||||
]
|
||||
|
||||
const MAX_RETRY_AFTER_MS = 5 * 60 * 1000
|
||||
|
||||
function retryAfterFromText(value) {
|
||||
const text = signalText(value)
|
||||
for (const pattern of RETRY_AFTER_PATTERNS) {
|
||||
const match = text.match(pattern)
|
||||
if (!match) continue
|
||||
const amount = Number(match[1])
|
||||
if (!Number.isFinite(amount) || amount < 0) continue
|
||||
const unit = String(match[2] || 's').toLowerCase()
|
||||
const multiplier = unit.startsWith('ms') || unit.startsWith('milli')
|
||||
? 1
|
||||
: unit.startsWith('m') ? 60 * 1000 : 1000
|
||||
return Math.min(MAX_RETRY_AFTER_MS, Math.round(amount * multiplier))
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
authFailureCertainty,
|
||||
isContextOverflow,
|
||||
isNetworkFailure,
|
||||
isProviderUnavailable,
|
||||
isRateLimited,
|
||||
retryAfterFromText,
|
||||
}
|
||||
@@ -24,7 +24,6 @@ const {
|
||||
attachmentType,
|
||||
cleanText,
|
||||
cleanProgressSteps,
|
||||
credentialFailure,
|
||||
normalizeAttachmentMetadata,
|
||||
parseAutoReply,
|
||||
settleWithin,
|
||||
@@ -1958,8 +1957,11 @@ class LocalWorkspaceAgentInvocation {
|
||||
: agentCancelled
|
||||
? agentStoppedError()
|
||||
: caughtError
|
||||
if (credentialFailure(error) && context.deferCredentialFailure !== true) {
|
||||
this.markRuntimeCredential(kind, 'unknown')
|
||||
// Only an explicit sign-in failure changes readiness. Network, quota and
|
||||
// provider refusals stay run-scoped so a healthy Agent remains usable.
|
||||
if (error?.code === 'LOCAL_AGENT_AUTH_REQUIRED'
|
||||
&& context.deferCredentialFailure !== true) {
|
||||
this.markRuntimeCredential(kind, 'missing')
|
||||
}
|
||||
const status = parentTimedOut
|
||||
? 'timeout'
|
||||
|
||||
@@ -3,7 +3,6 @@ const { MAX_MESSAGE_TEXT_CHARS } = require('./local-workspace-contracts.cjs')
|
||||
|
||||
const {
|
||||
cleanText,
|
||||
credentialFailure,
|
||||
normalizeKnowledgeBaseHint,
|
||||
normalizeSkillHint,
|
||||
} = require('./local-workspace-inputs.cjs')
|
||||
@@ -123,6 +122,11 @@ class LocalWorkspaceAutoRunner {
|
||||
this.retryMaxDelayMs = Number.isFinite(options.retryMaxDelayMs)
|
||||
? Math.max(this.retryBaseDelayMs, Math.floor(options.retryMaxDelayMs))
|
||||
: 2000
|
||||
// Provider throttling without a Retry-After hint waits at least this long
|
||||
// so automatic retries do not burn the remaining quota within a second.
|
||||
this.retryRateLimitMinDelayMs = Number.isFinite(options.retryRateLimitMinDelayMs)
|
||||
? Math.max(0, Math.floor(options.retryRateLimitMinDelayMs))
|
||||
: 2000
|
||||
this.sleep = options.retrySleep || abortableDelay
|
||||
}
|
||||
|
||||
@@ -306,14 +310,18 @@ class LocalWorkspaceAutoRunner {
|
||||
}
|
||||
// Session recovery is owned by LocalWorkspaceAgentInvocation so a stale
|
||||
// native Session gets exactly one fresh attempt across every run mode.
|
||||
// An ambiguous refusal (HTTP 403, proxy or region block) gets exactly
|
||||
// one more attempt: a transient gateway refusal recovers, while a real
|
||||
// permission problem fails fast without signing the Agent out.
|
||||
const maxAttempts = failure.category === 'timeout' && options.retryTimeouts !== true
|
||||
? 1
|
||||
: 4
|
||||
: failure.code === 'LOCAL_AGENT_ACCESS_DENIED' ? 2 : 4
|
||||
const decision = retryDecision(failure, {
|
||||
attempt,
|
||||
maxAttempts,
|
||||
baseDelayMs: this.retryBaseDelayMs,
|
||||
maxDelayMs: this.retryMaxDelayMs,
|
||||
rateLimitMinDelayMs: this.retryRateLimitMinDelayMs,
|
||||
safety,
|
||||
})
|
||||
if (decision.action === 'human_gate') {
|
||||
@@ -403,10 +411,9 @@ class LocalWorkspaceAutoRunner {
|
||||
finalOutcome: 'failed',
|
||||
...safety,
|
||||
})
|
||||
this.markRuntimeCredential(kind, 'unknown')
|
||||
try { await this.refreshAgents() } catch {
|
||||
if (controller.signal.aborted) throw new Error('LOCAL_AGENT_EXECUTION_STOPPED')
|
||||
}
|
||||
// Transient retries keep the Agent's readiness untouched. Refreshing
|
||||
// the whole catalog here re-ran discovery and credential probes while
|
||||
// the Agent was busy and could flip a healthy Agent to unavailable.
|
||||
await this.sleep(decision.delayMs, controller.signal)
|
||||
}
|
||||
}
|
||||
@@ -524,11 +531,11 @@ class LocalWorkspaceAutoRunner {
|
||||
continue
|
||||
}
|
||||
if (control) return { result: null, removed: false, control, error }
|
||||
if (!unauthorizedFailure(error)) {
|
||||
if (credentialFailure(error)) this.markRuntimeCredential(kind, 'unknown')
|
||||
throw error
|
||||
}
|
||||
this.markRuntimeCredential(kind, 'unknown')
|
||||
if (!unauthorizedFailure(error)) throw error
|
||||
// Only explicit sign-in failures reach this point. Mark the Agent as
|
||||
// needing a fresh login; the catalog clears it on the next refresh
|
||||
// after the retry window or as soon as native credentials verify.
|
||||
this.markRuntimeCredential(kind, 'missing')
|
||||
this.recordAttempt(group, controller, {
|
||||
agentKind: kind,
|
||||
phase,
|
||||
|
||||
@@ -34,11 +34,6 @@ function defaultAgentLabel(kind) {
|
||||
return AGENT_LABELS[kind] || String(kind || 'Agent')
|
||||
}
|
||||
|
||||
function credentialFailure(error) {
|
||||
return /api[ _-]?key|credential|auth(?:entication|orization)?|login|log in|unauthorized|forbidden|401|403|令牌|凭据|登录|认证/i
|
||||
.test(String(error?.message || error || ''))
|
||||
}
|
||||
|
||||
function cleanText(value, limit = MAX_MESSAGE_TEXT_CHARS) {
|
||||
return String(value || '').trim().slice(0, limit)
|
||||
}
|
||||
@@ -61,7 +56,6 @@ module.exports = {
|
||||
STABLE_USER_TURNS_PER_EDGE,
|
||||
cleanInline,
|
||||
cleanText,
|
||||
credentialFailure,
|
||||
defaultAgentLabel,
|
||||
isSupportedAgentKind,
|
||||
}
|
||||
@@ -40,7 +40,6 @@ module.exports = {
|
||||
cleanProgressSteps: runtimeContracts.cleanProgressSteps,
|
||||
cleanRunMaxRounds: runtimeContracts.cleanRunMaxRounds,
|
||||
cleanText: contracts.cleanText,
|
||||
credentialFailure: contracts.credentialFailure,
|
||||
defaultAgentLabel: contracts.defaultAgentLabel,
|
||||
emptyState: messageRecords.emptyState,
|
||||
isTracedAgentTerminalMessage: runtimeContracts.isTracedAgentTerminalMessage,
|
||||
|
||||
@@ -121,13 +121,27 @@ test('uses injectable jitter while keeping backoff and Retry-After bounded', ()
|
||||
response: { headers: { 'Retry-After': '3' } },
|
||||
}, { now: () => fakeNow })
|
||||
assert.equal(limited.category, 'rate_limit')
|
||||
// Provider throttling honors Retry-After beyond the generic process-retry
|
||||
// cap, bounded by the rate-limit cap (60 s by default).
|
||||
assert.equal(retryDecision(limited, {
|
||||
attempt: 1,
|
||||
maxAttempts: 4,
|
||||
baseDelayMs: 250,
|
||||
maxDelayMs: 2000,
|
||||
jitter: raw => raw / 2,
|
||||
}).delayMs, 2000)
|
||||
}).delayMs, 3000)
|
||||
const slow = normalizeFailure({
|
||||
status: 429,
|
||||
response: { headers: { 'Retry-After': '120' } },
|
||||
}, { now: () => fakeNow })
|
||||
assert.equal(retryDecision(slow, { attempt: 1, maxDelayMs: 2000 }).delayMs, 60000)
|
||||
assert.equal(retryDecision(slow, {
|
||||
attempt: 1, maxDelayMs: 2000, rateLimitMaxDelayMs: 5000,
|
||||
}).delayMs, 5000)
|
||||
const unhinted = normalizeFailure({ status: 429 }, { now: () => fakeNow })
|
||||
assert.equal(retryDecision(unhinted, {
|
||||
attempt: 1, baseDelayMs: 250, maxDelayMs: 2000, rateLimitMinDelayMs: 1500,
|
||||
}).delayMs, 1500)
|
||||
})
|
||||
|
||||
test('retries only transient categories and stops at the total attempt bound', () => {
|
||||
|
||||
@@ -0,0 +1,88 @@
|
||||
const test = require('node:test')
|
||||
const assert = require('node:assert/strict')
|
||||
const {
|
||||
authFailureCertainty,
|
||||
isContextOverflow,
|
||||
isNetworkFailure,
|
||||
isProviderUnavailable,
|
||||
isRateLimited,
|
||||
retryAfterFromText,
|
||||
} = require('../../src/runs/failure-signals.cjs')
|
||||
|
||||
test('explicit sign-in and API key statements are definitive authentication failures', () => {
|
||||
for (const text of [
|
||||
'Please log in first',
|
||||
'Error: Not logged in · Please run /login',
|
||||
'Invalid API key · Please run /login',
|
||||
'[ERROR] expired access token',
|
||||
'OAuth token has expired. Please obtain a new token.',
|
||||
'HTTP 401: Unauthorized',
|
||||
'API Error: 401 {"type":"error","error":{"type":"authentication_error","message":"invalid x-api-key"}}',
|
||||
'Incorrect API key provided: sk-***',
|
||||
'select an auth type',
|
||||
'Provider rejected sk-redacted',
|
||||
'请先登录',
|
||||
'认证失败,请检查密钥',
|
||||
]) {
|
||||
assert.equal(authFailureCertainty(text), 'definitive', text)
|
||||
}
|
||||
})
|
||||
|
||||
test('refusals that do not prove missing credentials are only ambiguous', () => {
|
||||
for (const text of [
|
||||
'HTTP 403: Forbidden',
|
||||
'API Error: 403 {"type":"error","error":{"type":"permission_error"}}',
|
||||
'403 Forbidden',
|
||||
'Error: Unauthorized',
|
||||
]) {
|
||||
assert.equal(authFailureCertainty(text), 'ambiguous', text)
|
||||
}
|
||||
})
|
||||
|
||||
test('unrelated diagnostics never look like authentication failures', () => {
|
||||
for (const text of [
|
||||
'',
|
||||
'thread \'main\' panicked at src/session.rs: invalid utf-8',
|
||||
'SyntaxError: Unexpected token } in JSON at position 4',
|
||||
'Compiled 12 files; login page renders correctly',
|
||||
'socket hang up',
|
||||
'Failed to download documentation: HTTP 403 Forbidden',
|
||||
'request failed: unauthorized resource in fixture',
|
||||
]) {
|
||||
assert.equal(authFailureCertainty(text), '', text)
|
||||
}
|
||||
})
|
||||
|
||||
test('transient provider conditions are recognized from CLI text', () => {
|
||||
assert.equal(isRateLimited('Error: 429 Too Many Requests'), true)
|
||||
assert.equal(isRateLimited('{"type":"overloaded_error"}'), true)
|
||||
assert.equal(isRateLimited('Rate limit reached for gpt-5 in organization'), true)
|
||||
assert.equal(isRateLimited('All 12 tests passed'), false)
|
||||
assert.equal(isNetworkFailure('fetch failed: getaddrinfo ENOTFOUND api.example.com'), true)
|
||||
assert.equal(isNetworkFailure('read ECONNRESET'), true)
|
||||
assert.equal(isNetworkFailure('Wrote network.md'), false)
|
||||
assert.equal(isProviderUnavailable('HTTP 503: Service Unavailable'), true)
|
||||
assert.equal(isProviderUnavailable('{"type":"error","error":{"type":"api_error"}}'), true)
|
||||
assert.equal(isProviderUnavailable('Processed 500 records'), false)
|
||||
})
|
||||
|
||||
test('context overflow is detected across provider wordings', () => {
|
||||
for (const text of [
|
||||
'Prompt is too long: 210000 tokens > 200000 maximum',
|
||||
'This model\'s maximum context length is 128000 tokens',
|
||||
'context_length_exceeded',
|
||||
'Input is too long for requested model.',
|
||||
]) {
|
||||
assert.equal(isContextOverflow(text), true, text)
|
||||
}
|
||||
assert.equal(isContextOverflow('The context of this change is the login flow'), false)
|
||||
})
|
||||
|
||||
test('retry hints in CLI text become bounded delays', () => {
|
||||
assert.equal(retryAfterFromText('429 Too Many Requests; retry after 20s'), 20000)
|
||||
assert.equal(retryAfterFromText('Rate limited. Please try again in 1.5 seconds.'), 1500)
|
||||
assert.equal(retryAfterFromText('retry-after: 750ms'), 750)
|
||||
assert.equal(retryAfterFromText('quota resets in 2 minutes'), 120000)
|
||||
assert.equal(retryAfterFromText('try again in 90 minutes'), 300000)
|
||||
assert.equal(retryAfterFromText('no hint here'), null)
|
||||
})
|
||||
@@ -4563,11 +4563,56 @@ test('automatic dialogue honors bounded Retry-After backoff and recovers', async
|
||||
assert.deepEqual(calls.map(call => call.agent.kind), [
|
||||
'hermes', 'hermes', 'hermes', 'hermes', 'codex',
|
||||
])
|
||||
assert.deepEqual(delays, [4, 4, 4])
|
||||
// Provider throttling honors the server's Retry-After instead of the
|
||||
// generic process-retry cap, still bounded by the rate-limit cap.
|
||||
assert.deepEqual(delays, [10, 10, 10])
|
||||
assert.equal(new Set(calls.slice(0, 4).map(call => call.runOptions.operationId)).size, 1)
|
||||
assert.equal(finished[0].status, 'completed')
|
||||
})
|
||||
|
||||
test('automatic dialogue retries an ambiguous 403 and network errors without touching readiness', async (t) => {
|
||||
const { directory, calls, options } = fixture()
|
||||
t.after(() => fs.rmSync(directory, { recursive: true, force: true }))
|
||||
const delays = []
|
||||
const failures = [
|
||||
agentRuntimeError('LOCAL_AGENT_PROCESS_FAILED', 'HTTP 403: Forbidden'),
|
||||
agentRuntimeError('LOCAL_AGENT_PROCESS_FAILED', 'fetch failed: read ECONNRESET'),
|
||||
]
|
||||
options.retrySleep = async delayMs => { delays.push(delayMs) }
|
||||
options.runAgent = async (agent, prompt, workdir, runOptions) => {
|
||||
calls.push({ agent, prompt, workdir, runOptions })
|
||||
if (agent.kind === 'hermes' && failures.length) throw failures.shift()
|
||||
return {
|
||||
text: `${agent.kind} agrees\n[[MELDWORK_CONSENSUS:agree]]`,
|
||||
sessionRef: runOptions.sessionRef || `${agent.kind}-session`,
|
||||
}
|
||||
}
|
||||
const workspace = new LocalWorkspace(options)
|
||||
const finished = []
|
||||
workspace.on('run-finished', result => finished.push(result))
|
||||
await workspace.refreshAgents()
|
||||
const readyBefore = workspace.snapshot().agents.find(agent => agent.kind === 'hermes').available
|
||||
const group = workspace.createGroup({
|
||||
name: 'Transient refusals', agentKinds: ['hermes', 'codex'], workdir: directory,
|
||||
allowWrite: false,
|
||||
})
|
||||
workspace.addMessage(group.id, 'user', 'Recover from transient refusals')
|
||||
|
||||
workspace.startAuto({ groupId: group.id, maxRounds: 1 })
|
||||
await workspace.activeRuns.get(group.id).promise
|
||||
|
||||
assert.deepEqual(calls.map(call => call.agent.kind), ['hermes', 'hermes', 'hermes', 'codex'])
|
||||
assert.equal(delays.length, 2)
|
||||
assert.equal(finished[0].status, 'completed')
|
||||
const hermes = workspace.snapshot().agents.find(agent => agent.kind === 'hermes')
|
||||
assert.equal(hermes.available, readyBefore)
|
||||
assert.notEqual(hermes.credentialState, 'missing')
|
||||
assert.deepEqual(workspace.getGroup(group.id).agentKinds, ['hermes', 'codex'])
|
||||
assert.equal(workspace.snapshot().messages.some(message => (
|
||||
message.system?.key === 'system.agentPausedAfterFailure'
|
||||
)), false)
|
||||
})
|
||||
|
||||
test('automatic dialogue retries one protocol failure without removing the Agent', async (t) => {
|
||||
const { directory, calls, options } = fixture()
|
||||
t.after(() => fs.rmSync(directory, { recursive: true, force: true }))
|
||||
|
||||
@@ -45,7 +45,6 @@ const EXPECTED_EXPORTS = [
|
||||
'cleanProgressSteps',
|
||||
'cleanRunMaxRounds',
|
||||
'cleanText',
|
||||
'credentialFailure',
|
||||
'defaultAgentLabel',
|
||||
'emptyState',
|
||||
'isTracedAgentTerminalMessage',
|
||||
|
||||
Reference in new issue
Block a user