From b122a556570198af48942c22f847e07588870c5f Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 23 Oct 2026 16:15:50 +0000 Subject: [PATCH] Address review: keep goals on failure, coerce model list fields - explore dequeues its goal only after the engine call succeeds - model-supplied list fields go through as_list/str_list, so null or string values can't crash a run or turn into per-character tests - run_ab_eval rejects blank problems before any model call and randomizes baseline/treated solver call order - middleware merges into a leading system message with block content Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01YbuVfbQEpaitM5TMjBMzkF --- museengine/agent.py | 8 +++++--- museengine/engine.py | 16 ++++++++-------- museengine/evaluation.py | 11 +++++++++-- museengine/llm.py | 12 +++++++++++- museengine/middleware.py | 10 +++++++--- tests/test_agent_and_eval.py | 19 +++++++++++++++++++ tests/test_engine.py | 23 +++++++++++++++++++++++ 7 files changed, 82 insertions(+), 17 deletions(-) diff --git a/museengine/agent.py b/museengine/agent.py index b6c7613..939d430 100644 --- a/museengine/agent.py +++ b/museengine/agent.py @@ -25,7 +25,7 @@ from typing import Any, Dict, List, Optional from .engine import MuseEngine from .governance import GovernanceError, Governor, ProposedAction -from .llm import complete_json +from .llm import complete_json, str_list from .memory import MemoryStore _REFLECT = """TASK: REFLECT @@ -145,8 +145,10 @@ class MuseAgent: # -- actions -------------------------------------------------------------- def _do_explore(self, action: ProposedAction) -> Dict[str, Any]: - problem = self._pop_goal() + problem = action.payload["problem"] report = self.engine.imagine(problem) + # Only dequeue once the work succeeded, so a failed call keeps the goal. + self._pop_goal() return { "problem": problem, "mode": report.mode, @@ -165,7 +167,7 @@ class MuseAgent: self.engine.llm, _REFLECT.format(episodes="\n".join(f"- {e.problem} -> {e.summary}" for e in recent)), ) - lessons = [str(x) for x in data.get("lessons", []) if str(x).strip()] + lessons = str_list(data.get("lessons")) for lesson in lessons: self.memory.add_note(lesson, [e.id for e in recent]) self.memory.set_state("reflected_count", len(episodes)) diff --git a/museengine/engine.py b/museengine/engine.py index 5c97b9f..f69853e 100644 --- a/museengine/engine.py +++ b/museengine/engine.py @@ -25,7 +25,7 @@ from dataclasses import asdict, dataclass, field from typing import Any, Dict, List, Optional, Sequence from .archetypes import Archetype, index_by_id, load_default_archetypes -from .llm import LLM, CountingLLM, clamp01, complete_json +from .llm import LLM, CountingLLM, as_list, clamp01, complete_json, str_list from .memory import Episode, MemoryStore SURVIVES = "survives" @@ -287,13 +287,13 @@ class MuseEngine: tensions = [ {"axis": str(t.get("axis", "")), "description": str(t.get("description", "")), "weight": clamp01(t.get("weight"), 0.5)} - for t in data.get("tensions", []) if isinstance(t, dict) + for t in as_list(data.get("tensions")) if isinstance(t, dict) ] return ProblemStructure( domain=str(data.get("domain", "unknown")), tensions=tensions, - constraints=[str(c) for c in data.get("constraints", [])], - failure_modes=[str(f) for f in data.get("failure_modes", [])], + constraints=str_list(data.get("constraints")), + failure_modes=str_list(data.get("failure_modes")), ) def _map( @@ -316,7 +316,7 @@ class MuseEngine: ) out: List[Analogy] = [] seen = set() - for item in data.get("analogies", []): + for item in as_list(data.get("analogies")): if not isinstance(item, dict): continue arch = self.by_id.get(str(item.get("archetype_id", ""))) @@ -356,8 +356,8 @@ class MuseEngine: id=f"c{idx + 1}_{analogy.archetype_id}", title=str(data.get("title", analogy.archetype_name)), thesis=str(data.get("thesis", "")), - mechanism=[str(m) for m in data.get("mechanism", [])], - tests=[str(t) for t in data.get("tests", [])], + mechanism=str_list(data.get("mechanism")), + tests=str_list(data.get("tests")), source_archetype=analogy.archetype_id, ) @@ -374,7 +374,7 @@ class MuseEngine: ) cand.novelty = clamp01(data.get("novelty")) cand.rigor = clamp01(data.get("rigor")) - cand.fatal_flaws = [str(f) for f in data.get("fatal_flaws", [])] + cand.fatal_flaws = str_list(data.get("fatal_flaws")) cand.critique = str(data.get("reasoning", "")) verdict = str(data.get("verdict", "")).strip().lower() # Anything other than an explicit "survives" counts as refuted, and a diff --git a/museengine/evaluation.py b/museengine/evaluation.py index ac2c2df..24e3ddb 100644 --- a/museengine/evaluation.py +++ b/museengine/evaluation.py @@ -116,12 +116,19 @@ def run_ab_eval( seed: int = 0, on_pair: Optional[Callable[[PairResult], None]] = None, ) -> EvalResult: + problems = [p.strip() for p in problems] + if not problems or not all(problems): + raise ValueError("problems must be a non-empty list of non-empty strings") rng = random.Random(seed) pairs = [] for problem in problems: - baseline = solver(problem) report = engine.imagine(problem) - treated = solver(f"{report.scaffold()}\n\n{problem}") + prompts = {"baseline": problem, "treated": f"{report.scaffold()}\n\n{problem}"} + # Randomize call order in case the solver carries state between calls. + order = ["baseline", "treated"] + rng.shuffle(order) + answers = {label: solver(prompts[label]) for label in order} + baseline, treated = answers["baseline"], answers["treated"] pair = judge_pair(judge, problem, baseline, treated, rng) pairs.append(pair) if on_pair: diff --git a/museengine/llm.py b/museengine/llm.py index 4a14b2d..cf650ab 100644 --- a/museengine/llm.py +++ b/museengine/llm.py @@ -9,7 +9,7 @@ from __future__ import annotations import json import re -from typing import Any, Callable, Dict, Optional +from typing import Any, Callable, Dict, List, Optional LLM = Callable[[str], str] @@ -75,6 +75,16 @@ def complete_json(llm: LLM, prompt: str, retries: int = 1) -> Dict[str, Any]: raise LLMError(f"model did not return valid JSON: {last_error}") +def as_list(value: Any) -> List[Any]: + """A model-supplied field as a list; anything that is not a list becomes [].""" + return value if isinstance(value, list) else [] + + +def str_list(value: Any) -> List[str]: + """A model-supplied field as a list of non-empty strings.""" + return [str(v) for v in as_list(value) if str(v).strip()] + + def clamp01(value: Any, default: float = 0.0) -> float: try: return max(0.0, min(1.0, float(value))) diff --git a/museengine/middleware.py b/museengine/middleware.py index e6fc0a3..754f971 100644 --- a/museengine/middleware.py +++ b/museengine/middleware.py @@ -58,7 +58,11 @@ def _latest_user_text(messages: Messages) -> str: def _inject_system(messages: Messages, scaffold: str) -> Messages: - if messages and messages[0].get("role") == "system" and isinstance(messages[0].get("content"), str): - first = dict(messages[0], content=f"{messages[0]['content']}\n\n{scaffold}") - return [first, *messages[1:]] + if messages and messages[0].get("role") == "system": + content = messages[0].get("content") + if isinstance(content, str): + return [dict(messages[0], content=f"{content}\n\n{scaffold}"), *messages[1:]] + if isinstance(content, list): + blocks = [*content, {"type": "text", "text": scaffold}] + return [dict(messages[0], content=blocks), *messages[1:]] return [{"role": "system", "content": scaffold}, *messages] diff --git a/tests/test_agent_and_eval.py b/tests/test_agent_and_eval.py index 2864e99..d293d3c 100644 --- a/tests/test_agent_and_eval.py +++ b/tests/test_agent_and_eval.py @@ -147,3 +147,22 @@ def test_run_ab_eval_uses_scaffold(): assert (result.wins, result.losses, result.ties) == (2, 0, 0) assert len(solver_prompts) == 4 assert "scaffold won 2" in result.summary() + + +def test_failed_explore_keeps_goal(): + def boom(prompt): + raise RuntimeError("model down") + + agent, memory = make_agent(boom) + agent.add_goal("keep me") + with pytest.raises(RuntimeError): + agent.run() + assert agent.goals() == ["keep me"] + assert memory.audit_log()[-1]["decision"] == "failed" + + +def test_run_ab_eval_rejects_blank_problems_before_calling_models(): + calls = [] + with pytest.raises(ValueError): + run_ab_eval(["ok", " "], lambda p: calls.append(p) or "", lambda p: "{}", MuseEngine()) + assert calls == [] diff --git a/tests/test_engine.py b/tests/test_engine.py index e9b2192..1eed5fa 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -145,3 +145,26 @@ def test_anthropic_adapter_with_stub_client(): assert anthropic_llm(client=client)("hello") == "hi" assert calls[0]["model"] == "claude-opus-5-5" assert calls[0]["messages"] == [{"role": "user", "content": "hello"}] + + +def test_malformed_list_fields_are_ignored(fake_llm): + fake_llm.responses["ANALYZE"] = {"domain": "x", "tensions": None, "constraints": "one string"} + fake_llm.responses["GENERATE"] = dict(fake_llm.responses["GENERATE"], tests="not a list") + fake_llm.responses["CRITIQUE"] = {"novelty": 1, "rigor": 1, "verdict": "survives", "fatal_flaws": None} + report = MuseEngine(llm=fake_llm).imagine("Design a backpressure mesh") + assert report.structure.tensions == [] and report.structure.constraints == [] + assert all(c.tests == [] and c.verdict == "refuted" for c in report.candidates) + + +def test_middleware_merges_block_system_message(): + seen = {} + + @MuseMiddleware(MuseEngine()) + def agent(messages): + seen["messages"] = messages + + agent([{"role": "system", "content": [{"type": "text", "text": "be brief"}]}, + {"role": "user", "content": "slime mold routing"}]) + assert len(seen["messages"]) == 2 + blocks = seen["messages"][0]["content"] + assert blocks[0]["text"] == "be brief" and "" in blocks[1]["text"]