fix: ReliabilityEvaluator snapshots after lazy session-history restore

Resumed sessions restore saved chat_history on the first chat(), after the
pre-run snapshot, so old tool calls were counted as current-run calls.
Trigger the idempotent _init_db_session/_init_session_store loaders before
snapshotting. Addresses Greptile P1 on #6346.

Co-Authored-By: Claude Opus 5.5 <[email protected]>

src/praisonai-agents/praisonaiagents/background/runner.py

@@ -179,7 +179,7 @@ class BackgroundRunner:
                     return result
 
             except asyncio.TimeoutError:
-                task.fail(f"Task timed out after {timeout}s")
+                task.fail(f"Task timed out after {timeout or self.config.default_timeout}s")
                 logger.warning(f"Background task timed out: {task.name}")
 
                 _safe_on_complete(on_complete, task)

src/praisonai-agents/praisonaiagents/background/task.py

@@ -137,11 +137,15 @@ class BackgroundTask:
         
         try:
             result = await asyncio.wait_for(self._future, timeout=timeout)
-            return result
         except asyncio.TimeoutError:
             raise
         except Exception as e:
             raise RuntimeError(str(e))
+        # The runner records failures on the task and resolves the future
+        # to None, so surface the documented RuntimeError here.
+        if self.status == TaskStatus.FAILED:
+            raise RuntimeError(self.error)
+        return result
     
     def should_cancel(self) -> bool:
         """Check if cancellation was requested."""

src/praisonai-agents/praisonaiagents/eval/reliability.py

@@ -4,6 +4,7 @@ Reliability evaluator for PraisonAI Agents.
 Evaluates agent reliability by verifying expected tool calls are made.
 """
 
+import asyncio
 from praisonaiagents._logging import get_logger
 from typing import Dict, List, Optional, Set, Any, TYPE_CHECKING
 
@@ -55,12 +56,30 @@ class ReliabilityEvaluator(BaseEvaluator):
         if agent is None:
             raise ValueError("'agent' must be provided")
     
-    def _extract_tool_calls(self, response: Any) -> Set[str]:
+    def _history_snapshot(self) -> list:
+        # Resumed sessions restore saved history lazily on the first chat();
+        # load it now (idempotent) so it lands in the snapshot, not the run.
+        for loader in ('_init_db_session', '_init_session_store'):
+            fn = getattr(self.agent, loader, None)
+            if callable(fn):
+                try:
+                    fn()
+                except Exception as e:
+                    logger.debug(f"{loader} before snapshot failed: {e}")
+        try:
+            return list(getattr(self.agent, 'chat_history', None) or [])
+        except TypeError:
+            return []
+
+    def _extract_tool_calls(self, response: Any, prior_history: Optional[list] = None) -> Set[str]:
         """
         Extract tool names from agent response.
         
         Args:
             response: Agent response object
+            prior_history: chat_history snapshot taken before the run; only
+                messages not present in it (by identity) are inspected, so
+                trimmed/rewritten histories are handled correctly
             
         Returns:
             Set of tool names that were called
@@ -75,7 +94,12 @@ class ReliabilityEvaluator(BaseEvaluator):
                     tool_calls.add(tc.function.name)
         
         if hasattr(self.agent, 'chat_history'):
-            for msg in self.agent.chat_history:
+            # Identity, not index: survives history trimming/rewrites. The
+            # snapshot list keeps prior objects alive, so ids stay unique.
+            seen = {id(m) for m in (prior_history or [])}
+            for msg in list(self.agent.chat_history or []):
+                if id(msg) in seen:
+                    continue
                 if isinstance(msg, dict):
                     if msg.get('role') == 'assistant' and 'tool_calls' in msg:
                         for tc in msg.get('tool_calls', []):
@@ -106,6 +130,7 @@ class ReliabilityEvaluator(BaseEvaluator):
         )
         
         try:
+            prior_history = self._history_snapshot()
             if hasattr(self.agent, 'chat'):
                 response = self.agent.chat(self.input_text)
             elif hasattr(self.agent, 'start'):
@@ -113,7 +138,7 @@ class ReliabilityEvaluator(BaseEvaluator):
             else:
                 raise ValueError("Agent must have 'chat' or 'start' method")
             
-            actual_tools = self._extract_tool_calls(response)
+            actual_tools = self._extract_tool_calls(response, prior_history)
             
             if self.verbose:
                 logger.info(f"Expected tools: {self.expected_tools}")
@@ -139,7 +164,9 @@ class ReliabilityEvaluator(BaseEvaluator):
             
         except Exception as e:
             logger.error(f"Error during reliability evaluation: {e}")
-            for tool in self.expected_tools:
+            # A crashed run is never a pass: record a failure even when only
+            # forbidden_tools (or nothing) was configured.
+            for tool in (self.expected_tools or {"<agent_run>"}):
                 result.tool_results.append(ToolCallResult(
                     tool_name=tool,
                     expected=True,
@@ -147,6 +174,14 @@ class ReliabilityEvaluator(BaseEvaluator):
                     passed=False,
                     metadata={"error": str(e)}
                 ))
+            for tool in self.forbidden_tools:
+                result.tool_results.append(ToolCallResult(
+                    tool_name=tool,
+                    expected=False,
+                    actual=False,
+                    passed=False,
+                    metadata={"error": str(e), "inconclusive": True}
+                ))
         
         self.after_run(result)
         
@@ -166,7 +201,11 @@ class ReliabilityEvaluator(BaseEvaluator):
             ReliabilityResult with tool call verification results
         """
         await self.async_before_run()
-        result = self.run(print_summary=False)
+        from ..trace.context_events import copy_context_to_callable
+        loop = asyncio.get_running_loop()
+        result = await loop.run_in_executor(
+            None, copy_context_to_callable(lambda: self.run(print_summary=False))
+        )
         await self.async_after_run(result)
         
         if print_summary:

src/praisonai-agents/tests/unit/test_issue_6319_background_reliability.py

@@ -0,0 +1,102 @@
+"""Regression tests for #6319 (part b): failed BackgroundTask.wait() and
+ReliabilityEvaluator stale/crashed runs."""
+import asyncio
+
+import pytest
+
+from praisonaiagents.background import BackgroundRunner
+from praisonaiagents.eval.reliability import ReliabilityEvaluator
+
+
+def test_background_wait_raises_on_failure_and_timeout():
+    async def main():
+        r = BackgroundRunner()
+
+        def boom():
+            raise ValueError("kaboom")
+
+        async def slow():
+            await asyncio.sleep(5)
+
+        t = await r.submit(boom, name="boom")
+        with pytest.raises(RuntimeError, match="kaboom"):
+            await t.wait()
+        t2 = await r.submit(slow, name="slow", timeout=0.1)
+        with pytest.raises(RuntimeError, match="timed out after 0.1s"):
+            await t2.wait()
+        t3 = await r.submit(lambda: 42, name="ok")
+        assert await t3.wait() == 42
+
+    asyncio.run(main())
+
+
+class _Stub:
+    def __init__(self):
+        self.chat_history = [{"role": "tool", "name": "search"}]
+
+    def chat(self, q):
+        self.chat_history.append({"role": "assistant", "content": "x"})
+        return "x"
+
+
+class _Crash:
+    chat_history = []
+
+    def chat(self, q):
+        raise RuntimeError("crash")
+
+
+class _Rewriter:
+    """chat() replaces history with an equal-length new history."""
+
+    def __init__(self):
+        self.chat_history = [{"role": "user", "content": str(i)} for i in range(2)]
+
+    def chat(self, q):
+        self.chat_history = [{"role": "user", "content": q},
+                             {"role": "tool", "name": "rm"}]
+        return "x"
+
+
+def test_reliability_ignores_prior_history():
+    assert not ReliabilityEvaluator(agent=_Stub(), input_text="q",
+                                    expected_tools=["search"]).run().passed
+    assert ReliabilityEvaluator(agent=_Stub(), input_text="q",
+                                forbidden_tools=["search"]).run().passed
+
+
+def test_reliability_crash_never_passes():
+    assert not ReliabilityEvaluator(agent=_Crash(), input_text="q",
+                                    forbidden_tools=["rm"]).run().passed
+    assert not asyncio.run(
+        ReliabilityEvaluator(agent=_Crash(), input_text="q").run_async()).passed
+
+
+def test_reliability_rewritten_history_detects_new_calls():
+    assert not ReliabilityEvaluator(agent=_Rewriter(), input_text="q",
+                                    forbidden_tools=["rm"]).run().passed
+
+
+class _Resumed:
+    """Saved history is restored lazily on the first chat() (resumed session)."""
+
+    def __init__(self):
+        self.chat_history = []
+        self._loaded = False
+
+    def _init_session_store(self):
+        if not self._loaded:
+            self._loaded = True
+            self.chat_history.append({"role": "tool", "name": "search"})
+
+    def chat(self, q):
+        self._init_session_store()
+        self.chat_history.append({"role": "assistant", "content": "x"})
+        return "x"
+
+
+def test_reliability_ignores_lazily_restored_session_history():
+    assert not ReliabilityEvaluator(agent=_Resumed(), input_text="q",
+                                    expected_tools=["search"]).run().passed
+    assert ReliabilityEvaluator(agent=_Resumed(), input_text="q",
+                                forbidden_tools=["search"]).run().passed