Address review: keep goals on failure, coerce model list fields

- explore dequeues its goal only after the engine call succeeds
- model-supplied list fields go through as_list/str_list, so null or
  string values can't crash a run or turn into per-character tests
- run_ab_eval rejects blank problems before any model call and
  randomizes baseline/treated solver call order
- middleware merges into a leading system message with block content

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
Claude-Session: https://claude.ai/code/session_01YbuVfbQEpaitM5TMjBMzkF

museengine/agent.py

@@ -25,7 +25,7 @@ from typing import Any, Dict, List, Optional
 
 from .engine import MuseEngine
 from .governance import GovernanceError, Governor, ProposedAction
-from .llm import complete_json
+from .llm import complete_json, str_list
 from .memory import MemoryStore
 
 _REFLECT = """TASK: REFLECT
@@ -145,8 +145,10 @@ class MuseAgent:
     # -- actions --------------------------------------------------------------
 
     def _do_explore(self, action: ProposedAction) -> Dict[str, Any]:
-        problem = self._pop_goal()
+        problem = action.payload["problem"]
         report = self.engine.imagine(problem)
+        # Only dequeue once the work succeeded, so a failed call keeps the goal.
+        self._pop_goal()
         return {
             "problem": problem,
             "mode": report.mode,
@@ -165,7 +167,7 @@ class MuseAgent:
             self.engine.llm,
             _REFLECT.format(episodes="\n".join(f"- {e.problem} -> {e.summary}" for e in recent)),
         )
-        lessons = [str(x) for x in data.get("lessons", []) if str(x).strip()]
+        lessons = str_list(data.get("lessons"))
         for lesson in lessons:
             self.memory.add_note(lesson, [e.id for e in recent])
         self.memory.set_state("reflected_count", len(episodes))

museengine/engine.py

@@ -25,7 +25,7 @@ from dataclasses import asdict, dataclass, field
 from typing import Any, Dict, List, Optional, Sequence
 
 from .archetypes import Archetype, index_by_id, load_default_archetypes
-from .llm import LLM, CountingLLM, clamp01, complete_json
+from .llm import LLM, CountingLLM, as_list, clamp01, complete_json, str_list
 from .memory import Episode, MemoryStore
 
 SURVIVES = "survives"
@@ -287,13 +287,13 @@ class MuseEngine:
         tensions = [
             {"axis": str(t.get("axis", "")), "description": str(t.get("description", "")),
              "weight": clamp01(t.get("weight"), 0.5)}
-            for t in data.get("tensions", []) if isinstance(t, dict)
+            for t in as_list(data.get("tensions")) if isinstance(t, dict)
         ]
         return ProblemStructure(
             domain=str(data.get("domain", "unknown")),
             tensions=tensions,
-            constraints=[str(c) for c in data.get("constraints", [])],
-            failure_modes=[str(f) for f in data.get("failure_modes", [])],
+            constraints=str_list(data.get("constraints")),
+            failure_modes=str_list(data.get("failure_modes")),
         )
 
     def _map(
@@ -316,7 +316,7 @@ class MuseEngine:
         )
         out: List[Analogy] = []
         seen = set()
-        for item in data.get("analogies", []):
+        for item in as_list(data.get("analogies")):
             if not isinstance(item, dict):
                 continue
             arch = self.by_id.get(str(item.get("archetype_id", "")))
@@ -356,8 +356,8 @@ class MuseEngine:
             id=f"c{idx + 1}_{analogy.archetype_id}",
             title=str(data.get("title", analogy.archetype_name)),
             thesis=str(data.get("thesis", "")),
-            mechanism=[str(m) for m in data.get("mechanism", [])],
-            tests=[str(t) for t in data.get("tests", [])],
+            mechanism=str_list(data.get("mechanism")),
+            tests=str_list(data.get("tests")),
             source_archetype=analogy.archetype_id,
         )
 
@@ -374,7 +374,7 @@ class MuseEngine:
         )
         cand.novelty = clamp01(data.get("novelty"))
         cand.rigor = clamp01(data.get("rigor"))
-        cand.fatal_flaws = [str(f) for f in data.get("fatal_flaws", [])]
+        cand.fatal_flaws = str_list(data.get("fatal_flaws"))
         cand.critique = str(data.get("reasoning", ""))
         verdict = str(data.get("verdict", "")).strip().lower()
         # Anything other than an explicit "survives" counts as refuted, and a

museengine/evaluation.py

@@ -116,12 +116,19 @@ def run_ab_eval(
     seed: int = 0,
     on_pair: Optional[Callable[[PairResult], None]] = None,
 ) -> EvalResult:
+    problems = [p.strip() for p in problems]
+    if not problems or not all(problems):
+        raise ValueError("problems must be a non-empty list of non-empty strings")
     rng = random.Random(seed)
     pairs = []
     for problem in problems:
-        baseline = solver(problem)
         report = engine.imagine(problem)
-        treated = solver(f"{report.scaffold()}\n\n{problem}")
+        prompts = {"baseline": problem, "treated": f"{report.scaffold()}\n\n{problem}"}
+        # Randomize call order in case the solver carries state between calls.
+        order = ["baseline", "treated"]
+        rng.shuffle(order)
+        answers = {label: solver(prompts[label]) for label in order}
+        baseline, treated = answers["baseline"], answers["treated"]
         pair = judge_pair(judge, problem, baseline, treated, rng)
         pairs.append(pair)
         if on_pair:

museengine/llm.py

@@ -9,7 +9,7 @@ from __future__ import annotations
 
 import json
 import re
-from typing import Any, Callable, Dict, Optional
+from typing import Any, Callable, Dict, List, Optional
 
 LLM = Callable[[str], str]
 
@@ -75,6 +75,16 @@ def complete_json(llm: LLM, prompt: str, retries: int = 1) -> Dict[str, Any]:
     raise LLMError(f"model did not return valid JSON: {last_error}")
 
 
+def as_list(value: Any) -> List[Any]:
+    """A model-supplied field as a list; anything that is not a list becomes []."""
+    return value if isinstance(value, list) else []
+
+
+def str_list(value: Any) -> List[str]:
+    """A model-supplied field as a list of non-empty strings."""
+    return [str(v) for v in as_list(value) if str(v).strip()]
+
+
 def clamp01(value: Any, default: float = 0.0) -> float:
     try:
         return max(0.0, min(1.0, float(value)))

museengine/middleware.py

@@ -58,7 +58,11 @@ def _latest_user_text(messages: Messages) -> str:
 
 
 def _inject_system(messages: Messages, scaffold: str) -> Messages:
-    if messages and messages[0].get("role") == "system" and isinstance(messages[0].get("content"), str):
-        first = dict(messages[0], content=f"{messages[0]['content']}\n\n{scaffold}")
-        return [first, *messages[1:]]
+    if messages and messages[0].get("role") == "system":
+        content = messages[0].get("content")
+        if isinstance(content, str):
+            return [dict(messages[0], content=f"{content}\n\n{scaffold}"), *messages[1:]]
+        if isinstance(content, list):
+            blocks = [*content, {"type": "text", "text": scaffold}]
+            return [dict(messages[0], content=blocks), *messages[1:]]
     return [{"role": "system", "content": scaffold}, *messages]

tests/test_agent_and_eval.py

@@ -147,3 +147,22 @@ def test_run_ab_eval_uses_scaffold():
     assert (result.wins, result.losses, result.ties) == (2, 0, 0)
     assert len(solver_prompts) == 4
     assert "scaffold won 2" in result.summary()
+
+
+def test_failed_explore_keeps_goal():
+    def boom(prompt):
+        raise RuntimeError("model down")
+
+    agent, memory = make_agent(boom)
+    agent.add_goal("keep me")
+    with pytest.raises(RuntimeError):
+        agent.run()
+    assert agent.goals() == ["keep me"]
+    assert memory.audit_log()[-1]["decision"] == "failed"
+
+
+def test_run_ab_eval_rejects_blank_problems_before_calling_models():
+    calls = []
+    with pytest.raises(ValueError):
+        run_ab_eval(["ok", "  "], lambda p: calls.append(p) or "", lambda p: "{}", MuseEngine())
+    assert calls == []

tests/test_engine.py

@@ -145,3 +145,26 @@ def test_anthropic_adapter_with_stub_client():
     assert anthropic_llm(client=client)("hello") == "hi"
     assert calls[0]["model"] == "claude-opus-5-5"
     assert calls[0]["messages"] == [{"role": "user", "content": "hello"}]
+
+
+def test_malformed_list_fields_are_ignored(fake_llm):
+    fake_llm.responses["ANALYZE"] = {"domain": "x", "tensions": None, "constraints": "one string"}
+    fake_llm.responses["GENERATE"] = dict(fake_llm.responses["GENERATE"], tests="not a list")
+    fake_llm.responses["CRITIQUE"] = {"novelty": 1, "rigor": 1, "verdict": "survives", "fatal_flaws": None}
+    report = MuseEngine(llm=fake_llm).imagine("Design a backpressure mesh")
+    assert report.structure.tensions == [] and report.structure.constraints == []
+    assert all(c.tests == [] and c.verdict == "refuted" for c in report.candidates)
+
+
+def test_middleware_merges_block_system_message():
+    seen = {}
+
+    @MuseMiddleware(MuseEngine())
+    def agent(messages):
+        seen["messages"] = messages
+
+    agent([{"role": "system", "content": [{"type": "text", "text": "be brief"}]},
+           {"role": "user", "content": "slime mold routing"}])
+    assert len(seen["messages"]) == 2
+    blocks = seen["messages"][0]["content"]
+    assert blocks[0]["text"] == "be brief" and "<muse_scaffold>" in blocks[1]["text"]