diff --git a/desktop/src/agents/agent-runtime-contract.cjs b/desktop/src/agents/agent-runtime-contract.cjs index 0710dc8..0d51946 100644 --- a/desktop/src/agents/agent-runtime-contract.cjs +++ b/desktop/src/agents/agent-runtime-contract.cjs @@ -1,3 +1,4 @@ +const { authFailureCertainty } = require('../runs/failure-signals.cjs') const AGENT_OUTCOMES = Object.freeze([ 'accepted', 'running', @@ -219,19 +220,25 @@ function terminalAuthenticationDiagnostic(value) { : '' } +// True only for explicit sign-in or API-key statements. Ambiguous refusals +// (HTTP 403, proxy or region blocks, "provider rejected") are classified as +// LOCAL_AGENT_ACCESS_DENIED so they are retried instead of signing the Agent out. function authConfigurationFailure(detail) { - return String(detail || '').split(/\r?\n/).some((line) => { - const text = line.trim().replace(/^(?:error|fatal)\s*:\s*/i, '') - return Boolean(terminalAuthenticationDiagnostic(text)) - || /^(?:(?:invalid|expired|missing|incorrect)\s+(?:api[ _-]?key|(?:access|refresh|auth)[ _-]?token|credentials?)\b|(?:authentication|authorization)\s+(?:failed|required)\b|(?:please\s+)?(?:log|sign)\s+in\b|not\s+logged\s+in\b|select an auth type\b|provider rejected\b)/i.test(text) - || /^(?:认证|鉴权|登录)(?:失败|过期)|^(?:令牌|凭据|密钥)(?:无效|缺失|已过期)|^请(?:先|重新)登录/.test(text) - }) + return authFailureCertainty(detail) === 'definitive' +} + +function authFailureCode(detail) { + const certainty = authFailureCertainty(detail) + if (certainty === 'definitive') return 'LOCAL_AGENT_AUTH_REQUIRED' + if (certainty === 'ambiguous') return 'LOCAL_AGENT_ACCESS_DENIED' + return '' } function failureCategory(code) { const value = String(code || '') if (value === 'LOCAL_AGENT_SESSION_INVALID') return 'session' if (value === 'LOCAL_AGENT_AUTH_REQUIRED') return 'authentication' + if (value === 'LOCAL_AGENT_ACCESS_DENIED') return 'provider' if (value === 'LOCAL_AGENT_TIMEOUT') return 'timeout' if (value === 'LOCAL_AGENT_EXECUTION_STOPPED') return 'cancellation' if (value === 'LOCAL_AGENT_REFUSED') return 'refusal' @@ -317,6 +324,9 @@ function requireTerminalAgentResult(result) { // error, so the catalog can mark the Agent as needing a fresh login. if (failure.category !== 'authentication' && authConfigurationFailure(diagnostic)) { failure = agentRuntimeFailure('LOCAL_AGENT_AUTH_REQUIRED') + } else if (failure.code === 'LOCAL_AGENT_PROCESS_FAILED' + && authFailureCertainty(normalized.diagnostic) === 'ambiguous') { + failure = agentRuntimeFailure('LOCAL_AGENT_ACCESS_DENIED') } throw agentRuntimeError( failure.code, @@ -326,7 +336,10 @@ function requireTerminalAgentResult(result) { } const authenticationDiagnostic = terminalAuthenticationDiagnostic(normalized.text) if (authenticationDiagnostic) { - throw agentRuntimeError('LOCAL_AGENT_AUTH_REQUIRED', authenticationDiagnostic) + throw agentRuntimeError( + authFailureCode(authenticationDiagnostic) || 'LOCAL_AGENT_ACCESS_DENIED', + authenticationDiagnostic, + ) } return normalized } @@ -355,6 +368,7 @@ module.exports = { agentRuntimeError, agentRuntimeFailure, authConfigurationFailure, + authFailureCode, isCodeReviewAgentKind, isReviewOnlyAgentKind, normalizeAgentResult, diff --git a/desktop/src/agents/cli/cli-process-support.cjs b/desktop/src/agents/cli/cli-process-support.cjs index 15a61c3..15ef010 100644 --- a/desktop/src/agents/cli/cli-process-support.cjs +++ b/desktop/src/agents/cli/cli-process-support.cjs @@ -4,8 +4,9 @@ const { NETWORK_ENV_KEYS, networkEnvironment } = require('./cli-network-environm const { redactChildSecrets } = require('./cli-runtime-events.cjs') const { agentRuntimeError, - authConfigurationFailure, + authFailureCode, } = require('../agent-runtime-contract.cjs') +const { authFailureCertainty, isContextOverflow } = require('../../runs/failure-signals.cjs') const { validateOpenClawRuntimeGuard } = require('./openclaw-runtime.cjs') const TERMINATE_GRACE_MS = 500 @@ -34,8 +35,17 @@ function agentExecutionError(code, diagnostic = '') { function invalidSessionFailure(detail) { const text = String(detail || '').trim() - if (!text || authConfigurationFailure(text) + if (!text || authFailureCertainty(text) || /\b(?:api|access|refresh|auth|session)[ _-]?token\b/i.test(text)) return false + // A resumed native session that no longer fits the model context cannot be + // continued; rebuilding it with compact continuation context is the fix. + if (isContextOverflow(text)) return true + // Runtime crashes ("thread 'main' panicked ... invalid ...") and parser + // errors mention threads or invalid input without saying anything about the + // stored session. Discarding a healthy session for them loses context. + if (/\bpanicked\b|\bSyntaxError\b|\binvalid JSON\b|\bUnexpected token\b|\bstack backtrace\b/i.test(text)) { + return false + } return /\b(?:no|unknown|invalid|expired|missing|stale)\s+(?:saved\s+)?(?:session|conversation|thread)\b|\b(?:session|conversation|thread)(?:\s+(?:id|key|reference))?\b[^\n]{0,100}\b(?:not found|does not exist|no longer exists|invalid|unknown|expired|has expired|was deleted|cannot be resumed)\b/i .test(text) } @@ -45,9 +55,7 @@ function failedAgentProcessError(detail, options = {}) { return agentExecutionError( options.sessionRef && invalidSessionFailure(detail) ? 'LOCAL_AGENT_SESSION_INVALID' - : authConfigurationFailure(detail) - ? 'LOCAL_AGENT_AUTH_REQUIRED' - : 'LOCAL_AGENT_PROCESS_FAILED', + : authFailureCode(detail) || 'LOCAL_AGENT_PROCESS_FAILED', detail, ) } diff --git a/desktop/src/runs/failure-policy.cjs b/desktop/src/runs/failure-policy.cjs index 8bf772f..8c94fa6 100644 --- a/desktop/src/runs/failure-policy.cjs +++ b/desktop/src/runs/failure-policy.cjs @@ -1,3 +1,11 @@ +const { + authFailureCertainty, + isNetworkFailure, + isProviderUnavailable, + isRateLimited, + retryAfterFromText, +} = require('./failure-signals.cjs') + const FAILURE_CATEGORIES = Object.freeze([ 'authentication', 'compatibility', @@ -121,23 +129,47 @@ function retryAfterMs(error, now) { const parsed = parseRetryAfter(headerValue(headers, 'retry-after'), now) if (parsed != null) return parsed } - return null + // Local CLIs surface provider throttling as text ("retry after 20s"). + return retryAfterFromText(error?.diagnostic || error?.message) +} + +// A local CLI process failure carries a code-derived category (usually +// protocol). Its diagnostic often proves a transient provider condition, so +// refine it before deciding whether to retry and how long to wait. +function transientCategoryFromText(detail) { + if (!detail) return '' + if (isRateLimited(detail)) return 'rate_limit' + if (isNetworkFailure(detail)) return 'network' + if (isProviderUnavailable(detail)) return 'provider' + return '' } function classifiedCategory(error, code, status) { const explicit = String(error?.failure?.category || '') - if (FAILURE_CATEGORY_SET.has(explicit)) return explicit const detail = String(error?.diagnostic || error?.message || '') + if (FAILURE_CATEGORY_SET.has(explicit)) { + if (['protocol', 'execution'].includes(explicit)) { + const refined = transientCategoryFromText(error?.diagnostic || '') + if (refined) return refined + } + return explicit + } if (code === 'LOCAL_BUDGET_EXHAUSTED') return 'budget' if (/PERMISSION/.test(code)) return 'permission' if (code === 'LOCAL_AGENT_EXECUTION_STOPPED' || /CANCEL|ABORT/.test(code)) { return 'cancellation' } if (code === 'LOCAL_AGENT_SESSION_INVALID') return 'session' - if (status === 401 || status === 403 || code === 'LOCAL_AGENT_AUTH_REQUIRED' - || /unauthori[sz]ed|forbidden|invalid token/i.test(detail)) return 'authentication' + // Only explicit sign-in statements prove that credentials are gone. A 403, + // a proxy refusal, or "unauthorized" inside unrelated output is retried as + // a provider-side refusal instead of stopping the Agent for the run. + const authCertainty = authFailureCertainty(detail) + if (status === 401 || code === 'LOCAL_AGENT_AUTH_REQUIRED' + || authCertainty === 'definitive') return 'authentication' + if (status === 403 || code === 'LOCAL_AGENT_ACCESS_DENIED' + || authCertainty === 'ambiguous') return 'provider' if (status === 429 || /RATE_LIMIT|TOO_MANY_REQUESTS/.test(code) - || /rate limit|too many requests/i.test(detail)) return 'rate_limit' + || isRateLimited(detail)) return 'rate_limit' if (NETWORK_CODES.has(code) || /NETWORK|CONNECTION/.test(code) || /socket hang up|temporary failure|network error/i.test(detail)) return 'network' if (status != null && status >= 500) return 'provider' @@ -244,18 +276,30 @@ function retryDecision(input, options = {}) { if (safety?.sideEffectsPossible === true && safety.idempotencyMode !== 'durable') { return Object.freeze({ ...base, action: 'human_gate', exhausted: false }) } - const maxDelayMs = Math.max(1, boundedInteger( + const configuredMaxDelayMs = Math.max(1, boundedInteger( options.maxDelayMs, 10 * 1000, 24 * 60 * 60 * 1000, )) + // Provider throttling needs a longer and server-directed wait than a crashed + // process. Honor Retry-After up to the rate-limit cap instead of hammering + // the provider with the generic sub-second backoff. + const rateLimited = failure.category === 'rate_limit' + const maxDelayMs = rateLimited + ? Math.max(configuredMaxDelayMs, boundedInteger( + options.rateLimitMaxDelayMs, 60 * 1000, 24 * 60 * 60 * 1000, + )) + : configuredMaxDelayMs const backoff = boundedBackoffDelay(attempt, { ...options, maxDelayMs }) const requestedDelay = failure.retryAfterMs == null ? 0 : Math.min(maxDelayMs, boundedInteger(failure.retryAfterMs, 0, maxDelayMs)) + const minimumDelay = rateLimited && failure.retryAfterMs == null + ? Math.min(maxDelayMs, boundedInteger(options.rateLimitMinDelayMs, 0, maxDelayMs)) + : 0 return Object.freeze({ ...base, action: 'retry', exhausted: false, - delayMs: Math.max(backoff, requestedDelay), + delayMs: Math.max(backoff, requestedDelay, minimumDelay), }) } diff --git a/desktop/src/runs/failure-signals.cjs b/desktop/src/runs/failure-signals.cjs new file mode 100644 index 0000000..2d4466c --- /dev/null +++ b/desktop/src/runs/failure-signals.cjs @@ -0,0 +1,129 @@ +// Pure text classifiers for local Agent failure diagnostics. +// +// Agent CLIs report failures as free text on stderr or inside structured error +// results. Classification decides whether Meldwork retries, asks the user to +// sign in again, or rebuilds a native session, so every rule here errs on the +// side of "ambiguous": only explicit login and API-key statements count as a +// definitive authentication failure. A bare 403, a proxy refusal, or a word +// such as "unauthorized" inside unrelated output must never mark an installed +// Agent as signed out. + +const MAX_SIGNAL_TEXT_CHARS = 64 * 1024 + +const LOG_PREFIX = /^(?:\[[^\]]{1,40}\]\s*|(?:error|fatal|err|warning)\s*[:!-]\s*|[✗✘×!]\s*)+/i + +const DEFINITIVE_AUTH_LINE = [ + /^(?:invalid|expired|missing|incorrect|revoked)\s+(?:api[ _-]?key|(?:access|refresh|auth|oauth|bearer)[ _-]?token|credentials?)\b/i, + /^(?:authentication|authorization)\s+(?:failed|required)\b/i, + /^(?:please\s+)?(?:log|sign)\s+in\b/i, + /^(?:please\s+)?run\s+\S*\s*(?:\/)?login\b/i, + /^not\s+(?:logged|signed)\s+in\b/i, + /^you\s+are\s+not\s+(?:logged|signed)\s+in\b/i, + /^select an auth type\b/i, + /^oauth\s+token\s+(?:has\s+)?(?:expired|is\s+invalid|was\s+revoked)\b/i, + /^HTTP\s+401\s*:/i, + /^provider rejected\b/i, + /^(?:认证|鉴权|登录)(?:失败|过期)/, + /^(?:令牌|凭据|密钥)(?:无效|缺失|已过期)/, + /^请(?:先|重新)登录/, + /^未登录/, +] + +const DEFINITIVE_AUTH_ANYWHERE = /"type"\s*:\s*"authentication_error"|\binvalid_api_key\b|\bincorrect api key provided\b|\binvalid x-api-key\b/i + +// Ambiguous refusals must lead a diagnostic line. Tool output such as +// "Failed to download docs: HTTP 403 Forbidden" mentions a refusal of some +// unrelated resource and stays a plain process failure. +const AMBIGUOUS_AUTH_LINE = [ + /^HTTP\s+403\s*:/i, + /^(?:403\s+)?forbidden\b/i, + /^unauthori[sz]ed\b/i, + /^access denied\b/i, + /^invalid token\b/i, +] + +const AMBIGUOUS_AUTH_ANYWHERE = /"type"\s*:\s*"permission_error"/i + +const RATE_LIMIT = /\b429\b|too many requests|rate[ _-]?limit(?:ed|ing)?\b|\boverloaded(?:_error)?\b|\bquota (?:exceeded|exhausted)\b|\bresource[ _-]?exhausted\b|\binsufficient_quota\b|\b529\b/i + +const NETWORK = /\b(?:ECONNRESET|ECONNREFUSED|ECONNABORTED|ENOTFOUND|ETIMEDOUT|EAI_AGAIN|ENETUNREACH|ENETDOWN|EHOSTUNREACH)\b|socket hang up|fetch failed|network (?:error|is unreachable)|connection (?:reset|refused|closed|timed out)|getaddrinfo|tls handshake|ssl handshake|proxy (?:error|connection)/i + +const PROVIDER_UNAVAILABLE = /\b(?:HTTP\s*)?50[0234]\b\s*(?::|-)?\s*(?:internal server error|bad gateway|service unavailable|gateway timeout)|\bservice unavailable\b|\bbad gateway\b|\bgateway timeout\b|\binternal server error\b|"type"\s*:\s*"api_error"|\bupstream connect error\b/i + +const CONTEXT_OVERFLOW = /prompt is too long|context[_ ]length[_ ]exceeded|maximum context length|context window (?:exceeded|is full|limit)|input is too long|too many (?:input )?tokens|exceeds? the (?:maximum|model'?s?) context|context_window_exceeded|request too large for model/i + +function signalText(value) { + return String(value || '').slice(0, MAX_SIGNAL_TEXT_CHARS) +} + +function diagnosticLines(value) { + return signalText(value) + .split(/\r?\n/) + .map(line => line.trim().replace(LOG_PREFIX, '').trim()) + .filter(Boolean) +} + +// Returns 'definitive', 'ambiguous', or ''. +function authFailureCertainty(value) { + const text = signalText(value) + if (!text.trim()) return '' + const lines = diagnosticLines(text) + if (DEFINITIVE_AUTH_ANYWHERE.test(text) + || lines.some(line => DEFINITIVE_AUTH_LINE.some(pattern => pattern.test(line)))) { + return 'definitive' + } + if (lines.some(line => AMBIGUOUS_AUTH_LINE.some(pattern => pattern.test(line))) + || AMBIGUOUS_AUTH_ANYWHERE.test(text)) { + return 'ambiguous' + } + return '' +} + +function isRateLimited(value) { + return RATE_LIMIT.test(signalText(value)) +} + +function isNetworkFailure(value) { + return NETWORK.test(signalText(value)) +} + +function isProviderUnavailable(value) { + return PROVIDER_UNAVAILABLE.test(signalText(value)) +} + +function isContextOverflow(value) { + return CONTEXT_OVERFLOW.test(signalText(value)) +} + +const RETRY_AFTER_PATTERNS = [ + /retry[- ]after["']?\s*[:=]?\s*(\d+(?:\.\d+)?)\s*(ms|milliseconds?|s|secs?|seconds?|m|mins?|minutes?)?/i, + /(?:try|retry) again in\s+(\d+(?:\.\d+)?)\s*(ms|milliseconds?|s|secs?|seconds?|m|mins?|minutes?)?/i, + /(?:reset|resets) in\s+(\d+(?:\.\d+)?)\s*(ms|milliseconds?|s|secs?|seconds?|m|mins?|minutes?)/i, +] + +const MAX_RETRY_AFTER_MS = 5 * 60 * 1000 + +function retryAfterFromText(value) { + const text = signalText(value) + for (const pattern of RETRY_AFTER_PATTERNS) { + const match = text.match(pattern) + if (!match) continue + const amount = Number(match[1]) + if (!Number.isFinite(amount) || amount < 0) continue + const unit = String(match[2] || 's').toLowerCase() + const multiplier = unit.startsWith('ms') || unit.startsWith('milli') + ? 1 + : unit.startsWith('m') ? 60 * 1000 : 1000 + return Math.min(MAX_RETRY_AFTER_MS, Math.round(amount * multiplier)) + } + return null +} + +module.exports = { + authFailureCertainty, + isContextOverflow, + isNetworkFailure, + isProviderUnavailable, + isRateLimited, + retryAfterFromText, +} diff --git a/desktop/src/workspace/local-workspace-agent-invocation.cjs b/desktop/src/workspace/local-workspace-agent-invocation.cjs index 2b3adab..affc4ca 100644 --- a/desktop/src/workspace/local-workspace-agent-invocation.cjs +++ b/desktop/src/workspace/local-workspace-agent-invocation.cjs @@ -24,7 +24,6 @@ const { attachmentType, cleanText, cleanProgressSteps, - credentialFailure, normalizeAttachmentMetadata, parseAutoReply, settleWithin, @@ -1958,8 +1957,11 @@ class LocalWorkspaceAgentInvocation { : agentCancelled ? agentStoppedError() : caughtError - if (credentialFailure(error) && context.deferCredentialFailure !== true) { - this.markRuntimeCredential(kind, 'unknown') + // Only an explicit sign-in failure changes readiness. Network, quota and + // provider refusals stay run-scoped so a healthy Agent remains usable. + if (error?.code === 'LOCAL_AGENT_AUTH_REQUIRED' + && context.deferCredentialFailure !== true) { + this.markRuntimeCredential(kind, 'missing') } const status = parentTimedOut ? 'timeout' diff --git a/desktop/src/workspace/local-workspace-auto-runner.cjs b/desktop/src/workspace/local-workspace-auto-runner.cjs index 2e52a9c..1887242 100644 --- a/desktop/src/workspace/local-workspace-auto-runner.cjs +++ b/desktop/src/workspace/local-workspace-auto-runner.cjs @@ -3,7 +3,6 @@ const { MAX_MESSAGE_TEXT_CHARS } = require('./local-workspace-contracts.cjs') const { cleanText, - credentialFailure, normalizeKnowledgeBaseHint, normalizeSkillHint, } = require('./local-workspace-inputs.cjs') @@ -123,6 +122,11 @@ class LocalWorkspaceAutoRunner { this.retryMaxDelayMs = Number.isFinite(options.retryMaxDelayMs) ? Math.max(this.retryBaseDelayMs, Math.floor(options.retryMaxDelayMs)) : 2000 + // Provider throttling without a Retry-After hint waits at least this long + // so automatic retries do not burn the remaining quota within a second. + this.retryRateLimitMinDelayMs = Number.isFinite(options.retryRateLimitMinDelayMs) + ? Math.max(0, Math.floor(options.retryRateLimitMinDelayMs)) + : 2000 this.sleep = options.retrySleep || abortableDelay } @@ -306,14 +310,18 @@ class LocalWorkspaceAutoRunner { } // Session recovery is owned by LocalWorkspaceAgentInvocation so a stale // native Session gets exactly one fresh attempt across every run mode. + // An ambiguous refusal (HTTP 403, proxy or region block) gets exactly + // one more attempt: a transient gateway refusal recovers, while a real + // permission problem fails fast without signing the Agent out. const maxAttempts = failure.category === 'timeout' && options.retryTimeouts !== true ? 1 - : 4 + : failure.code === 'LOCAL_AGENT_ACCESS_DENIED' ? 2 : 4 const decision = retryDecision(failure, { attempt, maxAttempts, baseDelayMs: this.retryBaseDelayMs, maxDelayMs: this.retryMaxDelayMs, + rateLimitMinDelayMs: this.retryRateLimitMinDelayMs, safety, }) if (decision.action === 'human_gate') { @@ -403,10 +411,9 @@ class LocalWorkspaceAutoRunner { finalOutcome: 'failed', ...safety, }) - this.markRuntimeCredential(kind, 'unknown') - try { await this.refreshAgents() } catch { - if (controller.signal.aborted) throw new Error('LOCAL_AGENT_EXECUTION_STOPPED') - } + // Transient retries keep the Agent's readiness untouched. Refreshing + // the whole catalog here re-ran discovery and credential probes while + // the Agent was busy and could flip a healthy Agent to unavailable. await this.sleep(decision.delayMs, controller.signal) } } @@ -524,11 +531,11 @@ class LocalWorkspaceAutoRunner { continue } if (control) return { result: null, removed: false, control, error } - if (!unauthorizedFailure(error)) { - if (credentialFailure(error)) this.markRuntimeCredential(kind, 'unknown') - throw error - } - this.markRuntimeCredential(kind, 'unknown') + if (!unauthorizedFailure(error)) throw error + // Only explicit sign-in failures reach this point. Mark the Agent as + // needing a fresh login; the catalog clears it on the next refresh + // after the retry window or as soon as native credentials verify. + this.markRuntimeCredential(kind, 'missing') this.recordAttempt(group, controller, { agentKind: kind, phase, diff --git a/desktop/src/workspace/local-workspace-contracts.cjs b/desktop/src/workspace/local-workspace-contracts.cjs index 9687ecc..0cbf7f1 100644 --- a/desktop/src/workspace/local-workspace-contracts.cjs +++ b/desktop/src/workspace/local-workspace-contracts.cjs @@ -34,11 +34,6 @@ function defaultAgentLabel(kind) { return AGENT_LABELS[kind] || String(kind || 'Agent') } -function credentialFailure(error) { - return /api[ _-]?key|credential|auth(?:entication|orization)?|login|log in|unauthorized|forbidden|401|403|令牌|凭据|登录|认证/i - .test(String(error?.message || error || '')) -} - function cleanText(value, limit = MAX_MESSAGE_TEXT_CHARS) { return String(value || '').trim().slice(0, limit) } @@ -61,7 +56,6 @@ module.exports = { STABLE_USER_TURNS_PER_EDGE, cleanInline, cleanText, - credentialFailure, defaultAgentLabel, isSupportedAgentKind, } diff --git a/desktop/src/workspace/local-workspace-inputs.cjs b/desktop/src/workspace/local-workspace-inputs.cjs index bea248d..2aa983c 100644 --- a/desktop/src/workspace/local-workspace-inputs.cjs +++ b/desktop/src/workspace/local-workspace-inputs.cjs @@ -40,7 +40,6 @@ module.exports = { cleanProgressSteps: runtimeContracts.cleanProgressSteps, cleanRunMaxRounds: runtimeContracts.cleanRunMaxRounds, cleanText: contracts.cleanText, - credentialFailure: contracts.credentialFailure, defaultAgentLabel: contracts.defaultAgentLabel, emptyState: messageRecords.emptyState, isTracedAgentTerminalMessage: runtimeContracts.isTracedAgentTerminalMessage, diff --git a/desktop/test/runs/failure-policy.test.cjs b/desktop/test/runs/failure-policy.test.cjs index b649137..38b8f09 100644 --- a/desktop/test/runs/failure-policy.test.cjs +++ b/desktop/test/runs/failure-policy.test.cjs @@ -121,13 +121,27 @@ test('uses injectable jitter while keeping backoff and Retry-After bounded', () response: { headers: { 'Retry-After': '3' } }, }, { now: () => fakeNow }) assert.equal(limited.category, 'rate_limit') + // Provider throttling honors Retry-After beyond the generic process-retry + // cap, bounded by the rate-limit cap (60 s by default). assert.equal(retryDecision(limited, { attempt: 1, maxAttempts: 4, baseDelayMs: 250, maxDelayMs: 2000, jitter: raw => raw / 2, - }).delayMs, 2000) + }).delayMs, 3000) + const slow = normalizeFailure({ + status: 429, + response: { headers: { 'Retry-After': '120' } }, + }, { now: () => fakeNow }) + assert.equal(retryDecision(slow, { attempt: 1, maxDelayMs: 2000 }).delayMs, 60000) + assert.equal(retryDecision(slow, { + attempt: 1, maxDelayMs: 2000, rateLimitMaxDelayMs: 5000, + }).delayMs, 5000) + const unhinted = normalizeFailure({ status: 429 }, { now: () => fakeNow }) + assert.equal(retryDecision(unhinted, { + attempt: 1, baseDelayMs: 250, maxDelayMs: 2000, rateLimitMinDelayMs: 1500, + }).delayMs, 1500) }) test('retries only transient categories and stops at the total attempt bound', () => { diff --git a/desktop/test/runs/failure-signals.test.cjs b/desktop/test/runs/failure-signals.test.cjs new file mode 100644 index 0000000..317f99d --- /dev/null +++ b/desktop/test/runs/failure-signals.test.cjs @@ -0,0 +1,88 @@ +const test = require('node:test') +const assert = require('node:assert/strict') +const { + authFailureCertainty, + isContextOverflow, + isNetworkFailure, + isProviderUnavailable, + isRateLimited, + retryAfterFromText, +} = require('../../src/runs/failure-signals.cjs') + +test('explicit sign-in and API key statements are definitive authentication failures', () => { + for (const text of [ + 'Please log in first', + 'Error: Not logged in · Please run /login', + 'Invalid API key · Please run /login', + '[ERROR] expired access token', + 'OAuth token has expired. Please obtain a new token.', + 'HTTP 401: Unauthorized', + 'API Error: 401 {"type":"error","error":{"type":"authentication_error","message":"invalid x-api-key"}}', + 'Incorrect API key provided: sk-***', + 'select an auth type', + 'Provider rejected sk-redacted', + '请先登录', + '认证失败，请检查密钥', + ]) { + assert.equal(authFailureCertainty(text), 'definitive', text) + } +}) + +test('refusals that do not prove missing credentials are only ambiguous', () => { + for (const text of [ + 'HTTP 403: Forbidden', + 'API Error: 403 {"type":"error","error":{"type":"permission_error"}}', + '403 Forbidden', + 'Error: Unauthorized', + ]) { + assert.equal(authFailureCertainty(text), 'ambiguous', text) + } +}) + +test('unrelated diagnostics never look like authentication failures', () => { + for (const text of [ + '', + 'thread \'main\' panicked at src/session.rs: invalid utf-8', + 'SyntaxError: Unexpected token } in JSON at position 4', + 'Compiled 12 files; login page renders correctly', + 'socket hang up', + 'Failed to download documentation: HTTP 403 Forbidden', + 'request failed: unauthorized resource in fixture', + ]) { + assert.equal(authFailureCertainty(text), '', text) + } +}) + +test('transient provider conditions are recognized from CLI text', () => { + assert.equal(isRateLimited('Error: 429 Too Many Requests'), true) + assert.equal(isRateLimited('{"type":"overloaded_error"}'), true) + assert.equal(isRateLimited('Rate limit reached for gpt-5 in organization'), true) + assert.equal(isRateLimited('All 12 tests passed'), false) + assert.equal(isNetworkFailure('fetch failed: getaddrinfo ENOTFOUND api.example.com'), true) + assert.equal(isNetworkFailure('read ECONNRESET'), true) + assert.equal(isNetworkFailure('Wrote network.md'), false) + assert.equal(isProviderUnavailable('HTTP 503: Service Unavailable'), true) + assert.equal(isProviderUnavailable('{"type":"error","error":{"type":"api_error"}}'), true) + assert.equal(isProviderUnavailable('Processed 500 records'), false) +}) + +test('context overflow is detected across provider wordings', () => { + for (const text of [ + 'Prompt is too long: 210000 tokens > 200000 maximum', + 'This model\'s maximum context length is 128000 tokens', + 'context_length_exceeded', + 'Input is too long for requested model.', + ]) { + assert.equal(isContextOverflow(text), true, text) + } + assert.equal(isContextOverflow('The context of this change is the login flow'), false) +}) + +test('retry hints in CLI text become bounded delays', () => { + assert.equal(retryAfterFromText('429 Too Many Requests; retry after 20s'), 20000) + assert.equal(retryAfterFromText('Rate limited. Please try again in 1.5 seconds.'), 1500) + assert.equal(retryAfterFromText('retry-after: 750ms'), 750) + assert.equal(retryAfterFromText('quota resets in 2 minutes'), 120000) + assert.equal(retryAfterFromText('try again in 90 minutes'), 300000) + assert.equal(retryAfterFromText('no hint here'), null) +}) diff --git a/desktop/test/workspace/local-workspace-auto.test.cjs b/desktop/test/workspace/local-workspace-auto.test.cjs index 1ca7ec0..90e1183 100644 --- a/desktop/test/workspace/local-workspace-auto.test.cjs +++ b/desktop/test/workspace/local-workspace-auto.test.cjs @@ -4563,11 +4563,56 @@ test('automatic dialogue honors bounded Retry-After backoff and recovers', async assert.deepEqual(calls.map(call => call.agent.kind), [ 'hermes', 'hermes', 'hermes', 'hermes', 'codex', ]) - assert.deepEqual(delays, [4, 4, 4]) + // Provider throttling honors the server's Retry-After instead of the + // generic process-retry cap, still bounded by the rate-limit cap. + assert.deepEqual(delays, [10, 10, 10]) assert.equal(new Set(calls.slice(0, 4).map(call => call.runOptions.operationId)).size, 1) assert.equal(finished[0].status, 'completed') }) +test('automatic dialogue retries an ambiguous 403 and network errors without touching readiness', async (t) => { + const { directory, calls, options } = fixture() + t.after(() => fs.rmSync(directory, { recursive: true, force: true })) + const delays = [] + const failures = [ + agentRuntimeError('LOCAL_AGENT_PROCESS_FAILED', 'HTTP 403: Forbidden'), + agentRuntimeError('LOCAL_AGENT_PROCESS_FAILED', 'fetch failed: read ECONNRESET'), + ] + options.retrySleep = async delayMs => { delays.push(delayMs) } + options.runAgent = async (agent, prompt, workdir, runOptions) => { + calls.push({ agent, prompt, workdir, runOptions }) + if (agent.kind === 'hermes' && failures.length) throw failures.shift() + return { + text: `${agent.kind} agrees\n[[MELDWORK_CONSENSUS:agree]]`, + sessionRef: runOptions.sessionRef || `${agent.kind}-session`, + } + } + const workspace = new LocalWorkspace(options) + const finished = [] + workspace.on('run-finished', result => finished.push(result)) + await workspace.refreshAgents() + const readyBefore = workspace.snapshot().agents.find(agent => agent.kind === 'hermes').available + const group = workspace.createGroup({ + name: 'Transient refusals', agentKinds: ['hermes', 'codex'], workdir: directory, + allowWrite: false, + }) + workspace.addMessage(group.id, 'user', 'Recover from transient refusals') + + workspace.startAuto({ groupId: group.id, maxRounds: 1 }) + await workspace.activeRuns.get(group.id).promise + + assert.deepEqual(calls.map(call => call.agent.kind), ['hermes', 'hermes', 'hermes', 'codex']) + assert.equal(delays.length, 2) + assert.equal(finished[0].status, 'completed') + const hermes = workspace.snapshot().agents.find(agent => agent.kind === 'hermes') + assert.equal(hermes.available, readyBefore) + assert.notEqual(hermes.credentialState, 'missing') + assert.deepEqual(workspace.getGroup(group.id).agentKinds, ['hermes', 'codex']) + assert.equal(workspace.snapshot().messages.some(message => ( + message.system?.key === 'system.agentPausedAfterFailure' + )), false) +}) + test('automatic dialogue retries one protocol failure without removing the Agent', async (t) => { const { directory, calls, options } = fixture() t.after(() => fs.rmSync(directory, { recursive: true, force: true })) diff --git a/desktop/test/workspace/local-workspace-inputs-api.test.cjs b/desktop/test/workspace/local-workspace-inputs-api.test.cjs index d661731..2f2a60d 100644 --- a/desktop/test/workspace/local-workspace-inputs-api.test.cjs +++ b/desktop/test/workspace/local-workspace-inputs-api.test.cjs @@ -45,7 +45,6 @@ const EXPECTED_EXPORTS = [ 'cleanProgressSteps', 'cleanRunMaxRounds', 'cleanText', - 'credentialFailure', 'defaultAgentLabel', 'emptyState', 'isTracedAgentTerminalMessage',