mirror of
https://github.com/Ryder-Sun/Meldwork.git
synced 2026-11-07 06:56:03 -10:00
fix: keep group members after Agent failures and recover emptied groups
An authentication or compatibility failure in an automatic discussion
called removeAgent, which rewrote and persisted group.agentKinds. One
proxy 403 or expired OAuth refresh made the Agent disappear from every
later turn of that group, and when the last member was removed the group
could no longer load: normalizeLoadedGroup rejects an empty member list,
so the next start silently dropped the group together with all of its
messages.
- The auto runner now pauses a failed Agent only for the current run.
Group membership is never edited; the run-scoped exclusion already
persisted in orchestration activeKinds keeps resume behaviour intact.
- The notice says the Agent is paused for this discussion, with the
failure reason, and stays in the group (system.agentPausedAfterFailure,
zh/en). The removal notice text is kept so history still renders.
- removeAgent refuses to remove a group's last member.
- Loading rebuilds an emptied group's members from the Agents that
replied in it instead of dropping the group and its history.
Checks: node --test local-workspace-state.test.cjs (3 pass, including a
new recovery test) and the membership tests in
local-workspace-auto.test.cjs. Two tests there that also expect a
'missing' credential state stay failing until the next commit (they
already failed on bb55fad).
This commit is contained in:
1 parent
4f31b77503
commit
1033d086d3
9 files changed
+99
-37
No files matched your search
@@ -95,7 +95,6 @@ class LocalWorkspaceAutoRunner {
|
||||
this.recordAgentFailure = options.recordAgentFailure
|
||||
this.recordAgentInterruption = options.recordAgentInterruption
|
||||
this.addMessage = options.addMessage
|
||||
this.removeAgent = typeof options.removeAgent === 'function' ? options.removeAgent : null
|
||||
this.commitV4AgentMessage = typeof options.commitV4AgentMessage === 'function'
|
||||
? options.commitV4AgentMessage
|
||||
: null
|
||||
@@ -133,39 +132,43 @@ class LocalWorkspaceAutoRunner {
|
||||
this.emitChanged()
|
||||
}
|
||||
|
||||
removeFailedAgent(group, controller, kind, threadRootId) {
|
||||
if (!this.removeAgent || group.conversationType === 'direct' || !group.agentKinds.includes(kind)) {
|
||||
return false
|
||||
}
|
||||
const removed = this.removeAgent(group.id, kind)
|
||||
if (!removed) return false
|
||||
pauseFailedAgentForRun(group, controller, kind, threadRootId, error = null) {
|
||||
// Failures are scoped to the current run. Group membership is user
|
||||
// configuration: editing it after a provider or login hiccup made the
|
||||
// Agent disappear from every later conversation turn, and removing the
|
||||
// last member made the whole group (and its history) unloadable.
|
||||
if (group.conversationType === 'direct' || !group.agentKinds.includes(kind)) return false
|
||||
if (!controller.failedKinds.includes(kind)) controller.failedKinds.push(kind)
|
||||
const alreadyReported = this.state().messages.some(message => (
|
||||
message.threadRootId === threadRootId
|
||||
&& message.agentKind === kind
|
||||
&& message.system?.key === 'system.agentRemovedAfterFailure'
|
||||
&& message.system?.key === 'system.agentPausedAfterFailure'
|
||||
&& message.system?.params?.runId === controller.runId
|
||||
))
|
||||
if (!alreadyReported) {
|
||||
const agent = this.agentLabel(kind)
|
||||
const reason = normalizeFailure(error).code || 'LOCAL_AGENT_PROCESS_FAILED'
|
||||
this.addMessage(
|
||||
group.id,
|
||||
'system',
|
||||
`${agent} was removed from the group after a terminal failure. The discussion continued with the remaining Agents.`,
|
||||
`${agent} is paused for the rest of this discussion (${reason}). It stays in the group and will be tried again on the next message.`,
|
||||
kind,
|
||||
threadRootId,
|
||||
{ key: 'system.agentRemovedAfterFailure', params: { agent } },
|
||||
{
|
||||
key: 'system.agentPausedAfterFailure',
|
||||
params: { agent, reason, runId: controller.runId },
|
||||
},
|
||||
)
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
shouldRemoveFailedAgent(error) {
|
||||
shouldPauseFailedAgent(error) {
|
||||
const category = normalizeFailure(error).category
|
||||
// Runtime, provider, rate-limit, protocol, and timeout failures are
|
||||
// isolated to this run. A local CLI may recover on the next turn, so its
|
||||
// configured group membership must remain intact. Only failures proving
|
||||
// that the installed Agent cannot currently authenticate or run are
|
||||
// removed from the persistent group.
|
||||
// Runtime, provider, rate-limit, protocol, and timeout failures stay
|
||||
// silent run-scoped exclusions; their own failure message explains them.
|
||||
// Authentication and compatibility failures get an explicit notice
|
||||
// because the user usually has to act (sign in or update the CLI).
|
||||
return ['authentication', 'compatibility'].includes(category)
|
||||
}
|
||||
|
||||
@@ -4941,10 +4944,10 @@ class LocalWorkspaceAutoRunner {
|
||||
const removed = []
|
||||
for (const kind of failedKinds) {
|
||||
const failure = (result?.failures || []).find(item => item.kind === kind)?.error
|
||||
// A local CLI timeout is a run-scoped availability problem. Keep the
|
||||
// configured group membership intact so a later turn can retry it.
|
||||
const didRemove = this.shouldRemoveFailedAgent(failure)
|
||||
? this.removeFailedAgent(group, controller, kind, threadRootId)
|
||||
// Every failure is a run-scoped exclusion. Group membership is never
|
||||
// edited, so a later turn can use the Agent again.
|
||||
const didRemove = this.shouldPauseFailedAgent(failure)
|
||||
? this.pauseFailedAgentForRun(group, controller, kind, threadRootId, failure)
|
||||
: false
|
||||
// The failed slot is still isolated from this run's active queue even
|
||||
// when its configured group membership is retained.
|
||||
@@ -5801,10 +5804,10 @@ class LocalWorkspaceAutoRunner {
|
||||
controller.failedKinds.push(executionKind)
|
||||
}
|
||||
terminalFailureOccurred = true
|
||||
// Preserve timed-out CLIs in the configured group. They are removed
|
||||
// only from this run's active queue and can be retried next turn.
|
||||
if (this.shouldRemoveFailedAgent(error)) {
|
||||
this.removeFailedAgent(group, controller, executionKind, threadRootId)
|
||||
// Failed CLIs leave only this run's active queue; the configured
|
||||
// group keeps them so the next turn can retry.
|
||||
if (this.shouldPauseFailedAgent(error)) {
|
||||
this.pauseFailedAgentForRun(group, controller, executionKind, threadRootId, error)
|
||||
}
|
||||
activeKinds = activeKinds.filter(activeKind => activeKind !== executionKind)
|
||||
}
|
||||
|
||||
@@ -139,6 +139,9 @@ class LocalWorkspaceConversations {
|
||||
removeAgent(groupId, kind) {
|
||||
const group = this.getGroup(groupId)
|
||||
if (group.conversationType === 'direct' || !group.agentKinds.includes(kind)) return false
|
||||
// A group without members cannot be loaded again, which would discard
|
||||
// its whole history. Membership edits must always keep one Agent.
|
||||
if (group.agentKinds.length <= 1) return false
|
||||
return this.commit(() => {
|
||||
group.agentKinds = group.agentKinds.filter(agentKind => agentKind !== kind)
|
||||
group.updatedAt = this.now()
|
||||
|
||||
@@ -171,12 +171,17 @@ function knowledgeBaseHintsPrompt(hints) {
|
||||
].join('\n')
|
||||
}
|
||||
|
||||
function normalizeLoadedGroup(input) {
|
||||
function normalizeLoadedGroup(input, options = {}) {
|
||||
if (!input || typeof input !== 'object' || Array.isArray(input)) return null
|
||||
const id = cleanText(input.id, 100)
|
||||
const agentKinds = [...new Set((Array.isArray(input.agentKinds) ? input.agentKinds : [])
|
||||
const normalizeKinds = values => [...new Set((Array.isArray(values) ? values : [])
|
||||
.map(kind => cleanInline(kind, 40))
|
||||
.filter(isSupportedAgentKind))]
|
||||
let agentKinds = normalizeKinds(input.agentKinds)
|
||||
// Earlier builds could persistently remove every member of a group after
|
||||
// repeated failures. Dropping such a group would also drop its messages,
|
||||
// so rebuild the membership from the Agents that replied in it instead.
|
||||
if (!agentKinds.length) agentKinds = normalizeKinds(options.recoveredAgentKinds)
|
||||
if (!id || !agentKinds.length) return null
|
||||
|
||||
const group = {
|
||||
|
||||
@@ -184,8 +184,17 @@ function loadWorkspaceState(storagePath) {
|
||||
}
|
||||
|
||||
try {
|
||||
const repliedAgentKinds = new Map()
|
||||
for (const message of Array.isArray(parsed.messages) ? parsed.messages : []) {
|
||||
if (message?.role !== 'agent' || typeof message.groupId !== 'string') continue
|
||||
const kinds = repliedAgentKinds.get(message.groupId) || []
|
||||
if (!kinds.includes(message.agentKind)) kinds.push(message.agentKind)
|
||||
repliedAgentKinds.set(message.groupId, kinds)
|
||||
}
|
||||
const groups = parsed.groups
|
||||
.map(normalizeLoadedGroup)
|
||||
.map(group => normalizeLoadedGroup(group, {
|
||||
recoveredAgentKinds: repliedAgentKinds.get(group?.id) || [],
|
||||
}))
|
||||
.filter(Boolean)
|
||||
const groupIds = new Set(groups.map(group => group.id))
|
||||
const messages = parsed.messages
|
||||
|
||||
@@ -329,7 +329,6 @@ class LocalWorkspace extends EventEmitter {
|
||||
recordAgentFailure: (...args) => this.recordAgentFailure(...args),
|
||||
recordAgentInterruption: (...args) => this.recordAgentInterruption(...args),
|
||||
addMessage: (...args) => this.addMessage(...args),
|
||||
removeAgent: (...args) => this.conversations.removeAgent(...args),
|
||||
commitV4AgentMessage: input => this.commitV4AgentMessage(input),
|
||||
emitChanged: () => this.emitChanged(),
|
||||
finishRun: (...args) => this.finishRun(...args),
|
||||
|
||||
@@ -4362,7 +4362,10 @@ test('automatic HTTP 401 completes without a retry delay or recovery handoff', a
|
||||
|
||||
assert.deepEqual(calls.map(call => call.agent.kind), ['hermes', 'codex'])
|
||||
assert.deepEqual(delays, [])
|
||||
assert.deepEqual(workspace.getGroup(group.id).agentKinds, ['codex'])
|
||||
assert.deepEqual(workspace.getGroup(group.id).agentKinds, ['hermes', 'codex'])
|
||||
assert.equal(workspace.snapshot().messages.some(message => (
|
||||
message.agentKind === 'hermes' && message.system?.key === 'system.agentPausedAfterFailure'
|
||||
)), true)
|
||||
assert.equal(calls.some(call => call.prompt.includes('Harness recovery task')), false)
|
||||
assert.equal(workspace.snapshot().messages.some(message => (
|
||||
message.system?.key === 'system.autoTimeout'
|
||||
@@ -4707,7 +4710,7 @@ test('automatic dialogue exhausts bounded transient retries before continuing',
|
||||
)), false)
|
||||
})
|
||||
|
||||
test('automatic dialogue fails HTTP 401 once, removes the Agent, and continues healthy slots', async (t) => {
|
||||
test('automatic dialogue fails HTTP 401 once, pauses the Agent for the run, and continues healthy slots', async (t) => {
|
||||
const { directory, calls, options } = fixture()
|
||||
t.after(() => fs.rmSync(directory, { recursive: true, force: true }))
|
||||
let hermesAttempts = 0
|
||||
@@ -4736,7 +4739,10 @@ test('automatic dialogue fails HTTP 401 once, removes the Agent, and continues h
|
||||
assert.deepEqual(calls.map(call => call.agent.kind), [
|
||||
'hermes', 'codex',
|
||||
])
|
||||
assert.deepEqual(workspace.getGroup(group.id).agentKinds, ['codex'])
|
||||
assert.deepEqual(workspace.getGroup(group.id).agentKinds, ['hermes', 'codex'])
|
||||
assert.equal(workspace.snapshot().messages.some(message => (
|
||||
message.agentKind === 'hermes' && message.system?.key === 'system.agentPausedAfterFailure'
|
||||
)), true)
|
||||
assert.equal(workspace.snapshot().messages.some(message => (
|
||||
message.system?.key === 'system.agentCallFailed'
|
||||
)), true)
|
||||
@@ -4838,7 +4844,10 @@ test('automatic dialogue does not send an authentication recovery handoff to ano
|
||||
'hermes', 'codex',
|
||||
])
|
||||
assert.doesNotMatch(calls[1].prompt, /Harness recovery task|Invalid token/)
|
||||
assert.deepEqual(workspace.getGroup(group.id).agentKinds, ['codex'])
|
||||
assert.deepEqual(workspace.getGroup(group.id).agentKinds, ['hermes', 'codex'])
|
||||
assert.equal(workspace.snapshot().messages.some(message => (
|
||||
message.agentKind === 'hermes' && message.system?.key === 'system.agentPausedAfterFailure'
|
||||
)), true)
|
||||
assert.equal(finished[0].status, 'partial')
|
||||
})
|
||||
|
||||
@@ -4882,7 +4891,7 @@ test('automatic dialogue keeps a failed authentication slot out of later group c
|
||||
assert.doesNotMatch(persistedContext, /Harness recovery task|repaired auth/)
|
||||
})
|
||||
|
||||
test('automatic dialogue persists one failed 401 attempt and removes the Agent', async (t) => {
|
||||
test('automatic dialogue persists one failed 401 attempt and pauses the Agent for the run', async (t) => {
|
||||
const { directory, calls, options } = fixture()
|
||||
t.after(() => fs.rmSync(directory, { recursive: true, force: true }))
|
||||
const ledgerPath = path.join(directory, 'run-ledger.json')
|
||||
@@ -4915,7 +4924,10 @@ test('automatic dialogue persists one failed 401 attempt and removes the Agent',
|
||||
assert.deepEqual(calls.map(call => call.agent.kind), [
|
||||
'hermes', 'codex', 'workbuddy',
|
||||
])
|
||||
assert.deepEqual(workspace.getGroup(group.id).agentKinds, ['codex', 'workbuddy'])
|
||||
assert.deepEqual(workspace.getGroup(group.id).agentKinds, ['hermes', 'codex', 'workbuddy'])
|
||||
assert.equal(workspace.snapshot().messages.some(message => (
|
||||
message.agentKind === 'hermes' && message.system?.key === 'system.agentPausedAfterFailure'
|
||||
)), true)
|
||||
assert.equal(
|
||||
workspace.state.sessions[workspace.sessionKey(group.id, 'hermes')],
|
||||
'unusable-hermes-session',
|
||||
@@ -4930,7 +4942,7 @@ test('automatic dialogue persists one failed 401 attempt and removes the Agent',
|
||||
assert.equal(failure.system.params.reason, 'HTTP 401; authentication failed; Agent retained')
|
||||
assert.equal(workspace.snapshot().messages.some(message => (
|
||||
message.agentKind === 'hermes' && message.system?.key === 'system.agentRemovedAfterFailure'
|
||||
)), true)
|
||||
)), false)
|
||||
assert.equal(finished.length, 1)
|
||||
assert.equal(finished[0].status, 'partial')
|
||||
assert.equal(finished[0].failedKinds.includes('hermes'), true)
|
||||
@@ -4953,7 +4965,7 @@ test('automatic dialogue persists one failed 401 attempt and removes the Agent',
|
||||
)
|
||||
})
|
||||
|
||||
test('automatic dialogue treats HTTP 403 as terminal and removes the Agent', async (t) => {
|
||||
test('automatic dialogue treats an adapter-classified 403 sign-in failure as terminal and pauses the Agent', async (t) => {
|
||||
const { directory, calls, options } = fixture()
|
||||
t.after(() => fs.rmSync(directory, { recursive: true, force: true }))
|
||||
options.runAgent = async (agent, prompt, workdir, runOptions) => {
|
||||
@@ -4979,7 +4991,10 @@ test('automatic dialogue treats HTTP 403 as terminal and removes the Agent', asy
|
||||
assert.deepEqual(calls.map(call => call.agent.kind), [
|
||||
'hermes', 'codex',
|
||||
])
|
||||
assert.deepEqual(workspace.getGroup(group.id).agentKinds, ['codex'])
|
||||
assert.deepEqual(workspace.getGroup(group.id).agentKinds, ['hermes', 'codex'])
|
||||
assert.equal(workspace.snapshot().messages.some(message => (
|
||||
message.agentKind === 'hermes' && message.system?.key === 'system.agentPausedAfterFailure'
|
||||
)), true)
|
||||
assert.equal(
|
||||
workspace.snapshot().agents.find(agent => agent.kind === 'hermes').credentialState,
|
||||
'missing',
|
||||
|
||||
@@ -61,3 +61,29 @@ test('distinguishes missing, corrupt, unreadable, unsupported, and valid workspa
|
||||
assert.equal(loadWorkspaceState(storagePath).status, 'ready')
|
||||
assert.equal(loadWorkspaceState(storagePath).trusted, true)
|
||||
})
|
||||
|
||||
test('a group emptied by an earlier build is recovered from its replies instead of dropped', (t) => {
|
||||
const storagePath = fixture(t)
|
||||
fs.writeFileSync(storagePath, JSON.stringify({
|
||||
version: 3,
|
||||
groups: [
|
||||
{ id: 'emptied', name: 'Emptied', agentKinds: [], workdir: '/tmp' },
|
||||
{ id: 'silent', name: 'Never replied', agentKinds: [], workdir: '/tmp' },
|
||||
],
|
||||
messages: [
|
||||
{ id: 'u1', groupId: 'emptied', role: 'user', content: 'Plan it' },
|
||||
{ id: 'a1', groupId: 'emptied', role: 'agent', agentKind: 'hermes', content: 'Hermes reply' },
|
||||
{ id: 'a2', groupId: 'emptied', role: 'agent', agentKind: 'codex', content: 'Codex reply' },
|
||||
{ id: 'a3', groupId: 'emptied', role: 'agent', agentKind: 'hermes', content: 'Hermes again' },
|
||||
{ id: 'u2', groupId: 'silent', role: 'user', content: 'Nobody answered' },
|
||||
],
|
||||
sessions: {},
|
||||
}))
|
||||
|
||||
const { state, trusted } = loadWorkspaceState(storagePath)
|
||||
assert.equal(trusted, true)
|
||||
assert.deepEqual(state.groups.map(group => [group.id, group.agentKinds]), [
|
||||
['emptied', ['hermes', 'codex']],
|
||||
])
|
||||
assert.deepEqual(state.messages.map(message => message.id), ['u1', 'a1', 'a2', 'a3'])
|
||||
})
|
||||
@@ -693,6 +693,7 @@ export default {
|
||||
'system.autoTaskDecisionMissing': 'Discussion stopped without a valid task completion decision.',
|
||||
'system.autoTaskBlocked': 'Task blocked: {reason}',
|
||||
'system.autoTaskNeedsHuman': 'Human decision needed: {reason}',
|
||||
'system.agentPausedAfterFailure': '{agent} is paused for the rest of this discussion: {reason} It stays in the group and will be tried again on the next message.',
|
||||
'system.agentRemovedAfterFailure': '{agent} was removed from the group after a terminal failure. The discussion continued with the remaining Agents.',
|
||||
'system.autoTimeout': 'Automatic discussion stopped after a runtime timeout before consensus.',
|
||||
'system.runCircuitBreaker': 'The mandatory circuit breaker stopped this run after {maxAttempts} Agent attempts.',
|
||||
|
||||
@@ -693,6 +693,7 @@ export default {
|
||||
'system.autoTaskDecisionMissing': '讨论已停止,尚无有效的任务完成判断。',
|
||||
'system.autoTaskBlocked': '任务受阻:{reason}',
|
||||
'system.autoTaskNeedsHuman': '需要人类决定:{reason}',
|
||||
'system.agentPausedAfterFailure': '{agent} 在本次讨论中暂停:{reason} 它仍在群聊中,下一条消息会再次尝试。',
|
||||
'system.agentRemovedAfterFailure': '{agent} 发生终止性故障,已自动移出群聊;其余 Agent 将继续讨论。',
|
||||
'system.autoTimeout': '自动讨论在达成共识前因运行超时而停止。',
|
||||
'system.runCircuitBreaker': '不可关闭的熔断器已在 {maxAttempts} 次 Agent 尝试后停止本次运行。',
|
||||
|
||||
Reference in new issue
Block a user