mirror of
https://github.com/MervinPraison/PraisonAI.git
synced 2026-11-08 01:08:39 -10:00
Compare commits
146
Commits
No files matched your search
@@ -34,10 +34,24 @@ jobs:
|
||||
id: config
|
||||
env:
|
||||
HAS_APP_CREDENTIALS: ${{ secrets.PRAISONAI_APP_ID != '' && secrets.PRAISONAI_APP_PRIVATE_KEY != '' }}
|
||||
run: echo "configured=$HAS_APP_CREDENTIALS" >> "$GITHUB_OUTPUT"
|
||||
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
|
||||
run: |
|
||||
echo "configured=$HAS_APP_CREDENTIALS" >> "$GITHUB_OUTPUT"
|
||||
openai_ready=true
|
||||
if [ -z "$OPENAI_API_KEY" ] || [ "$OPENAI_API_KEY" = "not-needed" ]; then
|
||||
openai_ready=false
|
||||
if [ "$HAS_APP_CREDENTIALS" = "true" ]; then
|
||||
echo "::notice::Skipping PraisonAI PR review: OPENAI_API_KEY is missing or a placeholder."
|
||||
fi
|
||||
fi
|
||||
if [ "$HAS_APP_CREDENTIALS" = "true" ] && [ "$openai_ready" = "true" ]; then
|
||||
echo "ready=true" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "ready=false" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
- name: Determine checkout ref
|
||||
if: steps.config.outputs.configured == 'true'
|
||||
if: steps.config.outputs.ready == 'true'
|
||||
id: dest
|
||||
env:
|
||||
ISSUE_NUMBER: ${{ github.event.issue.number }}
|
||||
@@ -53,7 +67,7 @@ jobs:
|
||||
fi
|
||||
|
||||
- name: Checkout Repository
|
||||
if: steps.config.outputs.configured == 'true'
|
||||
if: steps.config.outputs.ready == 'true'
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
persist-credentials: false
|
||||
@@ -61,7 +75,7 @@ jobs:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Generate GitHub App Token
|
||||
if: steps.config.outputs.configured == 'true'
|
||||
if: steps.config.outputs.ready == 'true'
|
||||
id: generate_token
|
||||
uses: tibdex/github-app-token@v2
|
||||
with:
|
||||
@@ -69,17 +83,17 @@ jobs:
|
||||
private_key: ${{ secrets.PRAISONAI_APP_PRIVATE_KEY }}
|
||||
|
||||
- name: Set up Python
|
||||
if: steps.config.outputs.configured == 'true'
|
||||
if: steps.config.outputs.ready == 'true'
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.11'
|
||||
|
||||
- name: Install PraisonAI
|
||||
if: steps.config.outputs.configured == 'true'
|
||||
if: steps.config.outputs.ready == 'true'
|
||||
run: pip install "praisonai[all]"
|
||||
|
||||
- name: Run PraisonAI PR Review
|
||||
if: steps.config.outputs.configured == 'true'
|
||||
if: steps.config.outputs.ready == 'true'
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ steps.generate_token.outputs.token }}
|
||||
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
|
||||
|
||||
@@ -773,7 +773,9 @@ jobs:
|
||||
env:
|
||||
# Bump deliberately: this is the version everything in this job was
|
||||
# verified against. Latest at time of writing is v0.33.3.
|
||||
OLLAMA_VERSION: v0.33.2
|
||||
# No leading "v": install.sh passes ?version=$OLLAMA_VERSION and ollama.com
|
||||
# redirects to GitHub tag v$version (vv0.33.2 → 404 if set to v0.33.2).
|
||||
OLLAMA_VERSION: "0.33.2"
|
||||
# 522 MB, capabilities: completion, tools, thinking.
|
||||
OLLAMA_TEST_MODEL: qwen3:0.6b
|
||||
|
||||
|
||||
@@ -18,76 +18,76 @@ This report compares **Python SDK feature categories** against **Python document
|
||||
| Category | Features | Docs | Lines |
|
||||
|----------|----------|------|-------|
|
||||
| ✅ AGUI | 1 | 2 | 511 |
|
||||
| ✅ Agent | 22 | 59 | 18997 |
|
||||
| ✅ Agent | 22 | 59 | 19076 |
|
||||
| ✅ Agent-to-Agent (A2A) | 1 | 7 | 2996 |
|
||||
| ✅ Agent-to-User (A2U) | 1 | 3 | 701 |
|
||||
| ✅ Approval | 1 | 8 | 3941 |
|
||||
| ✅ Approval | 1 | 8 | 3969 |
|
||||
| ✅ Audio | 2 | 12 | 1000 |
|
||||
| ✅ Auto Generation | 5 | 12 | 3879 |
|
||||
| ✅ Autonomy | 3 | 7 | 2794 |
|
||||
| ✅ Bots | 7 | 38 | 15473 |
|
||||
| ✅ Auto Generation | 5 | 12 | 3941 |
|
||||
| ✅ Autonomy | 3 | 7 | 2805 |
|
||||
| ✅ Bots | 7 | 38 | 15809 |
|
||||
| ✅ Budget | 1 | 1 | 287 |
|
||||
| ✅ CLI | 5 | 124 | 38956 |
|
||||
| ✅ CLI | 5 | 124 | 39191 |
|
||||
| ✅ Chunking | 2 | 2 | 435 |
|
||||
| ✅ Citations | 2 | 1 | 202 |
|
||||
| ✅ Code Execution | 2 | 14 | 5022 |
|
||||
| ✅ Conditions | 1 | 3 | 1386 |
|
||||
| ✅ Configuration | 3 | 8 | 3341 |
|
||||
| ✅ Context Management | 16 | 36 | 14289 |
|
||||
| ✅ Context Management | 16 | 36 | 14384 |
|
||||
| ✅ Deep Research | 8 | 2 | 591 |
|
||||
| ✅ Display | 6 | 3 | 844 |
|
||||
| ✅ Embeddings | 6 | 23 | 2051 |
|
||||
| ✅ Evaluation | 1 | 9 | 3537 |
|
||||
| ✅ Events | 1 | 2 | 780 |
|
||||
| ✅ Execution | 3 | 4 | 1693 |
|
||||
| ✅ Failover | 2 | 1 | 480 |
|
||||
| ✅ Evaluation | 1 | 9 | 3637 |
|
||||
| ✅ Events | 1 | 2 | 782 |
|
||||
| ✅ Execution | 3 | 4 | 1694 |
|
||||
| ✅ Failover | 2 | 1 | 537 |
|
||||
| ✅ Files | 2 | 7 | 2376 |
|
||||
| ✅ Flow | 1 | 3 | 811 |
|
||||
| ✅ Gateway | 7 | 90 | 34764 |
|
||||
| ✅ Guardrails | 5 | 4 | 2529 |
|
||||
| ✅ Gateway | 7 | 97 | 37597 |
|
||||
| ✅ Guardrails | 5 | 4 | 2642 |
|
||||
| ✅ Handoffs | 11 | 6 | 2748 |
|
||||
| ✅ Hooks | 2 | 9 | 5011 |
|
||||
| ✅ Hooks | 2 | 9 | 5230 |
|
||||
| ✅ Image | 1 | 11 | 1295 |
|
||||
| ✅ Knowledge | 4 | 20 | 6188 |
|
||||
| ✅ LLM | 3 | 15 | 5946 |
|
||||
| ✅ Loops | 4 | 5 | 1475 |
|
||||
| ✅ MCP | 1 | 64 | 15068 |
|
||||
| ✅ Memory | 6 | 21 | 8188 |
|
||||
| ✅ Knowledge | 4 | 20 | 6260 |
|
||||
| ✅ LLM | 3 | 15 | 5958 |
|
||||
| ✅ Loops | 4 | 5 | 1478 |
|
||||
| ✅ MCP | 1 | 64 | 15116 |
|
||||
| ✅ Memory | 6 | 21 | 8198 |
|
||||
| ✅ OCR | 2 | 1 | 237 |
|
||||
| ✅ Observability | 2 | 23 | 2859 |
|
||||
| ✅ Optimizer | 1 | 2 | 792 |
|
||||
| ✅ Output | 3 | 5 | 1289 |
|
||||
| ✅ Parallel Execution | 3 | 2 | 528 |
|
||||
| ✅ Planning | 6 | 7 | 1825 |
|
||||
| ✅ Plugins | 8 | 5 | 3590 |
|
||||
| ✅ Planning | 6 | 7 | 1827 |
|
||||
| ✅ Plugins | 8 | 5 | 4108 |
|
||||
| ✅ Prompts | 2 | 9 | 2043 |
|
||||
| ✅ Providers | 1 | 58 | 8734 |
|
||||
| ✅ Providers | 1 | 58 | 8751 |
|
||||
| ✅ Query | 1 | 2 | 736 |
|
||||
| ✅ RAG | 5 | 15 | 3152 |
|
||||
| ✅ Realtime | 2 | 5 | 668 |
|
||||
| ✅ Reflection | 3 | 3 | 732 |
|
||||
| ✅ Retrieval | 2 | 5 | 1046 |
|
||||
| ✅ Routing | 1 | 2 | 430 |
|
||||
| ✅ Sandbox | 5 | 7 | 3712 |
|
||||
| ✅ Security | 1 | 3 | 3607 |
|
||||
| ✅ Sessions | 4 | 21 | 8382 |
|
||||
| ✅ Skills | 6 | 16 | 5988 |
|
||||
| ✅ Sandbox | 5 | 7 | 3720 |
|
||||
| ✅ Security | 1 | 3 | 3642 |
|
||||
| ✅ Sessions | 4 | 21 | 8388 |
|
||||
| ✅ Skills | 6 | 16 | 6045 |
|
||||
| ✅ Tasks | 2 | 8 | 3266 |
|
||||
| ✅ Telemetry | 1 | 2 | 660 |
|
||||
| ✅ Templates | 1 | 9 | 1859 |
|
||||
| ✅ Tools | 12 | 145 | 44758 |
|
||||
| ✅ Tools | 12 | 147 | 45378 |
|
||||
| ✅ Tracing | 3 | 2 | 139 |
|
||||
| ✅ Vector Store | 1 | 12 | 1219 |
|
||||
| ✅ Video | 2 | 10 | 1303 |
|
||||
| ✅ Vision | 2 | 1 | 329 |
|
||||
| ✅ Web | 3 | 10 | 2815 |
|
||||
| ✅ Workflows | 5 | 21 | 8862 |
|
||||
| ✅ Web | 3 | 10 | 2822 |
|
||||
| ✅ Workflows | 5 | 21 | 8880 |
|
||||
|
||||
## Documentation Without Features
|
||||
|
||||
These docs exist but don't match any implemented feature category:
|
||||
|
||||
- ℹ️ Database (43 docs, 7657 lines)
|
||||
- ℹ️ Database (44 docs, 8206 lines)
|
||||
- ℹ️ Documents (1 docs, 810 lines)
|
||||
- ℹ️ Teams (1 docs, 214 lines)
|
||||
|
||||
|
||||
@@ -183,7 +183,9 @@ banner. Key fragments:
|
||||
timeout-minutes: 15
|
||||
env:
|
||||
# Bump deliberately; the cache key and what we test both follow this.
|
||||
OLLAMA_VERSION: v0.33.2
|
||||
# No leading "v": install.sh passes ?version=$OLLAMA_VERSION and ollama.com
|
||||
# redirects to GitHub tag v$version (vv0.33.2 → 404 if set to v0.33.2).
|
||||
OLLAMA_VERSION: "0.33.2"
|
||||
OLLAMA_TEST_MODEL: qwen3:0.6b
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
@@ -204,9 +206,12 @@ banner. Key fragments:
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
curl -fsSL -o /tmp/ollama.tgz \
|
||||
"https://github.com/ollama/ollama/releases/download/${OLLAMA_VERSION}/ollama-linux-amd64.tgz"
|
||||
sudo tar -C /usr -xzf /tmp/ollama.tgz
|
||||
# The official installer, VERSION-PINNED via OLLAMA_VERSION (which it
|
||||
# honours). Since v0.33 the linux bundle ships as a .tar.zst with GPU
|
||||
# runtimes bundled in, so a direct release-asset download is not a
|
||||
# workable alternative; the installer handles the zst/tgz split.
|
||||
sudo apt-get update -qq && sudo apt-get install -y -qq zstd
|
||||
curl -fsSL https://ollama.com/install.sh | sh
|
||||
ollama --version
|
||||
- name: Start Ollama and pull model
|
||||
shell: bash
|
||||
|
||||
@@ -7,10 +7,7 @@ from praisonaiagents._logging import get_logger
|
||||
import asyncio
|
||||
import contextlib
|
||||
import threading
|
||||
import concurrent.futures
|
||||
import random
|
||||
import re
|
||||
from typing import List, Optional, Any, Dict, Union, Literal, TYPE_CHECKING, Callable, Generator
|
||||
from typing import List, Optional, Any, Dict, Union, Literal, TYPE_CHECKING, Callable
|
||||
from collections import OrderedDict
|
||||
import inspect
|
||||
|
||||
@@ -7126,6 +7123,25 @@ Answer:"""
|
||||
"""The most recent rejection message a guardrail sent back to the model."""
|
||||
return getattr(self, "_last_guardrail_error", None)
|
||||
|
||||
def _guardrail_blocked_message(self, error=None):
|
||||
"""Fixed, user-facing string returned when an output guardrail blocks a run.
|
||||
|
||||
A blocked response must never surface the rejected (potentially unsafe)
|
||||
model text, but it must also never be ``None`` - callers expect a
|
||||
``str`` and would otherwise crash on ``.strip()`` (issue #5596, FR-001).
|
||||
|
||||
The returned string is intentionally **fixed** and carries no diagnostic
|
||||
detail: a guardrail's own rejection reason (especially an LLM validator's
|
||||
free-text) can echo the blocked output, so appending it would leak the
|
||||
very content the guardrail rejected. The reason is logged instead, where
|
||||
it stays auditable without reaching the caller. The rejection is also
|
||||
exposed programmatically via :attr:`last_guardrail_error`.
|
||||
"""
|
||||
reason = str(error) if error else getattr(self, "_last_guardrail_error", None)
|
||||
if reason:
|
||||
logging.info(f"Agent {self.name}: guardrail block reason: {reason}")
|
||||
return "I'm sorry, but I can't share that response because it did not pass the safety guardrail."
|
||||
|
||||
def _process_guardrail(self, task_output):
|
||||
"""Process the guardrail validation for a task output.
|
||||
|
||||
|
||||
@@ -27,27 +27,21 @@ def run_async_in_sync_context(coro):
|
||||
tool loop, otherwise a bare un-awaited coroutine would be handed to the
|
||||
model as the tool result and the tool body would never run (silent data
|
||||
loss).
|
||||
|
||||
Delegates to the single canonical bridge
|
||||
(:func:`utils.async_bridge.run_coroutine_from_any_context`) so all sync→async
|
||||
call sites share one implementation: the caller's ``contextvars`` (trace /
|
||||
session / approval context) are carried into the coroutine, and the running
|
||||
loop is handled on a dedicated worker thread rather than raising.
|
||||
|
||||
``timeout=None`` is passed deliberately: this helper backs the sync
|
||||
tool-calling path, where a tool's own timeout policy (or lack of one) already
|
||||
governs how long a tool may run. Imposing the bridge's default 5-minute
|
||||
deadline here would silently cancel long-running tools that previously had no
|
||||
limit — a backward-incompatible regression — so the bridge adds none.
|
||||
"""
|
||||
try:
|
||||
asyncio.get_running_loop()
|
||||
except RuntimeError:
|
||||
# No event loop - safe to use asyncio.run()
|
||||
return asyncio.run(coro)
|
||||
|
||||
# Event loop exists - avoid deadlock by running in dedicated thread
|
||||
import concurrent.futures
|
||||
|
||||
def run_in_thread():
|
||||
new_loop = asyncio.new_event_loop()
|
||||
asyncio.set_event_loop(new_loop)
|
||||
try:
|
||||
return new_loop.run_until_complete(coro)
|
||||
finally:
|
||||
new_loop.close()
|
||||
|
||||
with concurrent.futures.ThreadPoolExecutor() as executor:
|
||||
future = executor.submit(run_in_thread)
|
||||
return future.result(timeout=300) # 5 minute timeout
|
||||
from ..utils.async_bridge import run_coroutine_from_any_context
|
||||
return run_coroutine_from_any_context(coro, timeout=None)
|
||||
|
||||
|
||||
class DualLock:
|
||||
@@ -306,13 +300,13 @@ class AsyncSafeState:
|
||||
|
||||
def __enter__(self):
|
||||
"""Support for synchronous context manager protocol (backward compatibility)."""
|
||||
self._lock._thread_lock.acquire()
|
||||
self._sync_guard = self._lock.sync()
|
||||
self._sync_guard.__enter__()
|
||||
return self.value
|
||||
|
||||
def __exit__(self, exc_type, exc_val, exc_tb):
|
||||
"""Support for synchronous context manager protocol (backward compatibility)."""
|
||||
self._lock._thread_lock.release()
|
||||
return None
|
||||
return self._sync_guard.__exit__(exc_type, exc_val, exc_tb)
|
||||
|
||||
async def __aenter__(self):
|
||||
"""Support for asynchronous context manager protocol."""
|
||||
|
||||
@@ -751,6 +751,43 @@ Your Goal: {self.goal}"""
|
||||
|
||||
return None
|
||||
|
||||
def _coerce_structured_output(self, response_text, schema_model):
|
||||
"""Parse a string response into the requested Pydantic model.
|
||||
|
||||
When ``output_pydantic``/``output_json`` is a Pydantic model, callers
|
||||
expect a validated instance rather than a raw string. This tolerates
|
||||
markdown code fences around the JSON payload.
|
||||
|
||||
Returns the parsed model instance on success, or the original
|
||||
``response_text`` unchanged if parsing is not possible (preserves
|
||||
backward compatibility for non-Pydantic schemas or malformed output).
|
||||
"""
|
||||
if not schema_model or not isinstance(response_text, str):
|
||||
return response_text
|
||||
if not hasattr(schema_model, 'model_validate_json'):
|
||||
return response_text
|
||||
|
||||
candidate = response_text.strip()
|
||||
try:
|
||||
return schema_model.model_validate_json(candidate)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Strip a surrounding markdown code fence (```json ... ```) if present
|
||||
fenced = re.search(r"```(?:json)?\s*(.*?)\s*```", candidate, re.DOTALL)
|
||||
if fenced:
|
||||
try:
|
||||
return schema_model.model_validate_json(fenced.group(1).strip())
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
logging.warning(
|
||||
f"Agent {self.name}: output_pydantic={getattr(schema_model, '__name__', schema_model)} "
|
||||
f"was requested but the response could not be parsed into the model; "
|
||||
f"returning raw string."
|
||||
)
|
||||
return response_text
|
||||
|
||||
def _supports_native_structured_output(self):
|
||||
"""Check if current model supports native structured output via response_format.
|
||||
|
||||
@@ -3562,6 +3599,13 @@ Your Goal: {self.goal}"""
|
||||
# Apply guardrail validation for custom LLM response
|
||||
try:
|
||||
validated_response = self._apply_guardrail_with_retry(response_text, prompt, temperature, tools, task_name, task_description, task_id, cancel_token=cancel_token, messages=processed_history)
|
||||
# Coerce into the requested Pydantic model so callers
|
||||
# get a validated instance, not a raw JSON string.
|
||||
# Only output_pydantic changes the return type; output_json
|
||||
# must stay a string for backward-compatible downstream
|
||||
# JSON parsing (task parsing, workflow results).
|
||||
if output_pydantic:
|
||||
validated_response = self._coerce_structured_output(validated_response, output_pydantic)
|
||||
# Execute callback and display after validation
|
||||
self._execute_callback_and_display(prompt, validated_response, time.time() - start_time, task_name, task_description, task_id)
|
||||
return self._trigger_after_agent_hook(prompt, validated_response, start_time)
|
||||
@@ -3569,7 +3613,7 @@ Your Goal: {self.goal}"""
|
||||
logging.error(f"Agent {self.name}: Guardrail validation failed for custom LLM: {e}")
|
||||
# Rollback chat history on guardrail failure
|
||||
self._rollback_chat_history_to(chat_history_length)
|
||||
return None
|
||||
return self._guardrail_blocked_message(e)
|
||||
except ToolExecutionError:
|
||||
raise
|
||||
except Exception as e:
|
||||
@@ -3693,6 +3737,11 @@ Your Goal: {self.goal}"""
|
||||
# Apply guardrail validation even for JSON output
|
||||
try:
|
||||
validated_response = self._apply_guardrail_with_retry(response_text, original_prompt, temperature, tools, task_name, task_description, task_id, cancel_token=cancel_token, messages=messages)
|
||||
# Coerce into the requested Pydantic model so callers
|
||||
# get a validated instance, not a raw JSON string.
|
||||
# output_json intentionally stays a string (backward compat).
|
||||
if output_pydantic:
|
||||
validated_response = self._coerce_structured_output(validated_response, output_pydantic)
|
||||
# Execute callback after validation
|
||||
self._execute_callback_and_display(original_prompt, validated_response, time.time() - start_time, task_name, task_description, task_id)
|
||||
return self._trigger_after_agent_hook(original_prompt, validated_response, start_time)
|
||||
@@ -3700,7 +3749,7 @@ Your Goal: {self.goal}"""
|
||||
logging.error(f"Agent {self.name}: Guardrail validation failed for JSON output: {e}")
|
||||
# Rollback chat history on guardrail failure
|
||||
self._rollback_chat_history_to(chat_history_length)
|
||||
return None
|
||||
return self._guardrail_blocked_message(e)
|
||||
|
||||
if not self.self_reflect:
|
||||
# User message already added before LLM call via _build_messages
|
||||
@@ -3721,7 +3770,7 @@ Your Goal: {self.goal}"""
|
||||
logging.error(f"Agent {self.name}: Guardrail validation failed for reasoning content: {e}")
|
||||
# Rollback chat history on guardrail failure
|
||||
self._rollback_chat_history_to(chat_history_length)
|
||||
return None
|
||||
return self._guardrail_blocked_message(e)
|
||||
# Apply guardrail to regular response
|
||||
try:
|
||||
validated_response = self._apply_guardrail_with_retry(response_text, original_prompt, temperature, tools, task_name, task_description, task_id, cancel_token=cancel_token, messages=messages)
|
||||
@@ -3732,7 +3781,7 @@ Your Goal: {self.goal}"""
|
||||
logging.error(f"Agent {self.name}: Guardrail validation failed: {e}")
|
||||
# Rollback chat history on guardrail failure
|
||||
self._rollback_chat_history_to(chat_history_length)
|
||||
return None
|
||||
return self._guardrail_blocked_message(e)
|
||||
|
||||
reflection_prompt = f"""
|
||||
Reflect on your previous response: '{response_text}'.
|
||||
@@ -3800,7 +3849,7 @@ Output MUST be JSON with 'reflection' and 'satisfactory'.
|
||||
# Rollback chat history on guardrail failure
|
||||
self._rollback_chat_history_to(chat_history_length)
|
||||
self._end_run(None, "error", {"error": str(e)})
|
||||
return None
|
||||
return self._guardrail_blocked_message(e)
|
||||
|
||||
# Check if we've hit max reflections
|
||||
if reflection_count >= self.max_reflect - 1:
|
||||
@@ -3818,7 +3867,7 @@ Output MUST be JSON with 'reflection' and 'satisfactory'.
|
||||
logging.error(f"Agent {self.name}: Guardrail validation failed after max reflections: {e}")
|
||||
# Rollback chat history on guardrail failure
|
||||
self._rollback_chat_history_to(chat_history_length)
|
||||
return None
|
||||
return self._guardrail_blocked_message(e)
|
||||
|
||||
# If not satisfactory and not at max reflections, continue with regeneration
|
||||
logging.debug(f"{self.name} reflection count {reflection_count + 1}, continuing reflection process")
|
||||
@@ -4228,6 +4277,11 @@ Output MUST be JSON with 'reflection' and 'satisfactory'.
|
||||
# Apply guardrail validation for custom LLM response
|
||||
try:
|
||||
validated_response = await self._aapply_guardrail_with_retry(response_text, prompt, temperature, tools, task_name, task_description, task_id, messages=effective_history)
|
||||
# Coerce into the requested Pydantic model so callers
|
||||
# get a validated instance, not a raw JSON string.
|
||||
# output_json intentionally stays a string (backward compat).
|
||||
if output_pydantic:
|
||||
validated_response = self._coerce_structured_output(validated_response, output_pydantic)
|
||||
# Execute callback after validation
|
||||
self._execute_callback_and_display(normalized_content, validated_response, time.time() - start_time, task_name, task_description, task_id)
|
||||
return await self._atrigger_after_agent_hook(prompt, validated_response, start_time)
|
||||
@@ -4235,7 +4289,7 @@ Output MUST be JSON with 'reflection' and 'satisfactory'.
|
||||
logging.error(f"Agent {self.name}: Guardrail validation failed for custom LLM: {e}")
|
||||
# Rollback chat history on guardrail failure
|
||||
self._rollback_chat_history_to(chat_history_length)
|
||||
return None
|
||||
return self._guardrail_blocked_message(e)
|
||||
except ToolExecutionError:
|
||||
raise
|
||||
except Exception as e:
|
||||
@@ -4373,6 +4427,11 @@ Output MUST be JSON with 'reflection' and 'satisfactory'.
|
||||
# Apply guardrail validation even for JSON output
|
||||
try:
|
||||
validated_response = await self._aapply_guardrail_with_retry(response_text, original_prompt, temperature, tools, task_name, task_description, task_id, messages=messages)
|
||||
# Coerce into the requested Pydantic model so callers
|
||||
# get a validated instance, not a raw JSON string.
|
||||
# output_json intentionally stays a string (backward compat).
|
||||
if output_pydantic:
|
||||
validated_response = self._coerce_structured_output(validated_response, output_pydantic)
|
||||
# Execute callback after validation
|
||||
self._execute_callback_and_display(original_prompt, validated_response, time.time() - start_time, task_name, task_description, task_id)
|
||||
return await self._atrigger_after_agent_hook(original_prompt, validated_response, start_time)
|
||||
@@ -4380,7 +4439,7 @@ Output MUST be JSON with 'reflection' and 'satisfactory'.
|
||||
logging.error(f"Agent {self.name}: Guardrail validation failed for JSON output: {e}")
|
||||
# Rollback chat history on guardrail failure
|
||||
self._rollback_chat_history_to(chat_history_length)
|
||||
return None
|
||||
return self._guardrail_blocked_message(e)
|
||||
|
||||
# For regular responses (no self-reflection)
|
||||
if not self.self_reflect:
|
||||
@@ -4403,7 +4462,7 @@ Output MUST be JSON with 'reflection' and 'satisfactory'.
|
||||
logging.error(f"Agent {self.name}: Guardrail validation failed for reasoning content: {e}")
|
||||
# Rollback chat history on guardrail failure
|
||||
self._rollback_chat_history_to(chat_history_length)
|
||||
return None
|
||||
return self._guardrail_blocked_message(e)
|
||||
else:
|
||||
# Apply guardrail to regular response content
|
||||
try:
|
||||
@@ -4415,7 +4474,7 @@ Output MUST be JSON with 'reflection' and 'satisfactory'.
|
||||
logging.error(f"Agent {self.name}: Guardrail validation failed: {e}")
|
||||
# Rollback chat history on guardrail failure
|
||||
self._rollback_chat_history_to(chat_history_length)
|
||||
return None
|
||||
return self._guardrail_blocked_message(e)
|
||||
|
||||
# If self-reflection is enabled, implement reflection logic
|
||||
if self.self_reflect:
|
||||
@@ -4455,7 +4514,7 @@ Output MUST be JSON with 'reflection' and 'satisfactory'.
|
||||
logging.error(f"Agent {self.name}: Guardrail validation failed: {e}")
|
||||
# Rollback chat history on guardrail failure
|
||||
self._rollback_chat_history_to(chat_history_length)
|
||||
return None
|
||||
return self._guardrail_blocked_message(e)
|
||||
|
||||
_areflection_kwargs = {}
|
||||
if temperature is not None:
|
||||
@@ -4489,7 +4548,7 @@ Output MUST be JSON with 'reflection' and 'satisfactory'.
|
||||
logging.error(f"Agent {self.name}: Guardrail validation failed after reflection: {e}")
|
||||
# Rollback chat history on guardrail failure
|
||||
self._rollback_chat_history_to(chat_history_length)
|
||||
return None
|
||||
return self._guardrail_blocked_message(e)
|
||||
|
||||
# Check if we've hit max reflections
|
||||
if reflection_count >= self.max_reflect - 1:
|
||||
@@ -4508,7 +4567,7 @@ Output MUST be JSON with 'reflection' and 'satisfactory'.
|
||||
logging.error(f"Agent {self.name}: Guardrail validation failed after max reflections: {e}")
|
||||
# Rollback chat history on guardrail failure
|
||||
self._rollback_chat_history_to(chat_history_length)
|
||||
return None
|
||||
return self._guardrail_blocked_message(e)
|
||||
|
||||
# Regenerate response based on reflection
|
||||
regenerate_messages = reflection_messages + [
|
||||
@@ -4558,7 +4617,7 @@ Output MUST be JSON with 'reflection' and 'satisfactory'.
|
||||
logging.error(f"Agent {self.name}: Guardrail validation failed after reflection error: {guard_e}")
|
||||
# Rollback chat history on guardrail failure
|
||||
self._rollback_chat_history_to(chat_history_length)
|
||||
return None
|
||||
return self._guardrail_blocked_message(guard_e)
|
||||
continue
|
||||
|
||||
# This should never be reached due to the returns above
|
||||
@@ -4608,6 +4667,11 @@ Output MUST be JSON with 'reflection' and 'satisfactory'.
|
||||
if get_logger().getEffectiveLevel() == logging.DEBUG:
|
||||
total_time = time.time() - start_time
|
||||
logging.debug(f"Agent.achat completed in {total_time:.2f} seconds")
|
||||
# Coerce into the requested Pydantic model so callers
|
||||
# get a validated instance, not a raw JSON string.
|
||||
# output_json intentionally stays a string (backward compat).
|
||||
if output_pydantic:
|
||||
response_text = self._coerce_structured_output(response_text, output_pydantic)
|
||||
# Execute callback after JSON/Pydantic completion
|
||||
self._execute_callback_and_display(original_prompt, response_text, time.time() - start_time, task_name, task_description, task_id)
|
||||
return await self._atrigger_after_agent_hook(original_prompt, response_text, start_time)
|
||||
@@ -4722,7 +4786,7 @@ Output MUST be JSON with 'reflection' and 'satisfactory'.
|
||||
logging.error(f"Agent {self.name}: Guardrail validation failed for OpenAI client: {e}")
|
||||
# Rollback chat history on guardrail failure
|
||||
self._rollback_chat_history_to(chat_history_length)
|
||||
return None
|
||||
return self._guardrail_blocked_message(e)
|
||||
except ToolExecutionError:
|
||||
raise
|
||||
except Exception as e:
|
||||
@@ -5728,6 +5792,28 @@ Output MUST be JSON with 'reflection' and 'satisfactory'.
|
||||
|
||||
return llm_summarize_async
|
||||
|
||||
def _resolve_compaction_max_tokens(self, execution_cfg):
|
||||
"""
|
||||
Resolve the compaction trigger budget.
|
||||
|
||||
An explicit ``ExecutionConfig.max_context_tokens`` always wins. When it
|
||||
is unset, size the trigger to the active model's context window (reusing
|
||||
the model-aware ``budgeter`` the in-loop path already uses) instead of a
|
||||
flat default, so large-context models are not compacted prematurely.
|
||||
Falls back to a static default only for the offline / unknown-model case.
|
||||
"""
|
||||
_explicit = getattr(execution_cfg, 'max_context_tokens', None)
|
||||
if _explicit:
|
||||
return _explicit
|
||||
try:
|
||||
from ..context.budgeter import get_model_limit, get_output_reserve
|
||||
model_name = self.llm if isinstance(self.llm, str) else "gpt-4o-mini"
|
||||
limit = get_model_limit(model_name)
|
||||
reserve = get_output_reserve(model_name)
|
||||
return max(1, int(limit * 0.8) - reserve)
|
||||
except Exception:
|
||||
return 8000
|
||||
|
||||
def _apply_context_compaction(self, messages, hook_event_class):
|
||||
"""
|
||||
Apply context compaction to messages if enabled (sync version).
|
||||
@@ -5747,7 +5833,7 @@ Output MUST be JSON with 'reflection' and 'satisfactory'.
|
||||
from ..compaction import ContextCompactor
|
||||
from ..compaction.strategy import CompactionStrategy
|
||||
|
||||
_max_tok = getattr(_execution_cfg, 'max_context_tokens', None) or 8000
|
||||
_max_tok = self._resolve_compaction_max_tokens(_execution_cfg)
|
||||
_strategy = getattr(_execution_cfg, 'compaction_strategy', None) or CompactionStrategy.TRUNCATE
|
||||
|
||||
# Create LLM summarization function if strategy is LLM_SUMMARIZE
|
||||
@@ -5779,17 +5865,20 @@ Output MUST be JSON with 'reflection' and 'satisfactory'.
|
||||
|
||||
# Perform compaction
|
||||
if _strategy == CompactionStrategy.LLM_SUMMARIZE and _llm_fn:
|
||||
import asyncio
|
||||
try:
|
||||
# Run async compaction in event loop
|
||||
compacted_msgs, _cr = asyncio.run(_compactor.compact_async(messages))
|
||||
except RuntimeError:
|
||||
# If already in async context, fall back to sync (naive) compaction
|
||||
logging.warning(
|
||||
f"[compaction] {self.name}: LLM_SUMMARIZE fell back to naive summarization "
|
||||
f"(asyncio.run not available in sync context)"
|
||||
)
|
||||
compacted_msgs, _cr = _compactor.compact(messages)
|
||||
# Drive the async summariser through the shared bridge so it runs
|
||||
# correctly whether or not a loop is already running (FastAPI /
|
||||
# Jupyter / a bot handler). The previous code caught bare
|
||||
# RuntimeError from ``asyncio.run`` ("event loop already
|
||||
# running") and silently downgraded LLM summarisation to naive
|
||||
# truncation whenever a loop was live; the bridge removes that
|
||||
# spurious fallback so the real summariser always runs. Provider
|
||||
# errors inside the summariser remain best-effort (see
|
||||
# ``_llm_summarize_async``, which falls back to a naive summary on
|
||||
# failure) — compaction never hard-fails the turn.
|
||||
from ..utils.async_bridge import run_coroutine_from_any_context
|
||||
compacted_msgs, _cr = run_coroutine_from_any_context(
|
||||
_compactor.compact_async(messages)
|
||||
)
|
||||
else:
|
||||
compacted_msgs, _cr = _compactor.compact(messages)
|
||||
|
||||
@@ -5834,7 +5923,7 @@ Output MUST be JSON with 'reflection' and 'satisfactory'.
|
||||
from ..compaction import ContextCompactor
|
||||
from ..compaction.strategy import CompactionStrategy
|
||||
|
||||
_max_tok = getattr(_execution_cfg, 'max_context_tokens', None) or 8000
|
||||
_max_tok = self._resolve_compaction_max_tokens(_execution_cfg)
|
||||
_strategy = getattr(_execution_cfg, 'compaction_strategy', None) or CompactionStrategy.TRUNCATE
|
||||
|
||||
# Create LLM summarization function if strategy is LLM_SUMMARIZE
|
||||
|
||||
@@ -764,6 +764,50 @@ class MemoryMixin:
|
||||
except Exception as e:
|
||||
logging.debug(f"Auto-memory extraction failed: {e}")
|
||||
|
||||
def _persist_memory_turn(self, user_message: str, assistant_response: str):
|
||||
"""Persist a raw conversation turn to the memory store.
|
||||
|
||||
Called after each response when ``_memory_instance`` is set but
|
||||
``auto_memory`` extraction is off. Stores the turn to short-term memory
|
||||
so a later Agent sharing the same store (e.g. ``memory={"user_id": uid}``)
|
||||
recalls it via ``get_memory_context()``. Best-effort: never raises.
|
||||
|
||||
Async-safe: a memory store write is blocking file/DB I/O. When this runs
|
||||
inside a live event loop (the async ``achat``/``astart`` after-agent path),
|
||||
the write is offloaded to a worker thread so it never stalls the loop
|
||||
(AGENTS.md §4.5). On the sync path (no running loop) it writes inline.
|
||||
"""
|
||||
memory = getattr(self, "_memory_instance", None)
|
||||
if memory is None or not assistant_response:
|
||||
return
|
||||
store = getattr(memory, "store_short_term", None) or getattr(memory, "add_short_term", None)
|
||||
if store is None:
|
||||
return
|
||||
text = f"User: {user_message}\nAssistant: {assistant_response}"
|
||||
metadata = {"agent_id": getattr(self, "agent_id", getattr(self, "name", None))}
|
||||
|
||||
def _do_store():
|
||||
try:
|
||||
store(text, metadata=metadata)
|
||||
except Exception as e:
|
||||
logging.debug(f"Memory turn persistence failed: {e}")
|
||||
|
||||
try:
|
||||
import asyncio
|
||||
asyncio.get_running_loop()
|
||||
except RuntimeError:
|
||||
# No running loop (sync chat/run/start path): write inline.
|
||||
_do_store()
|
||||
return
|
||||
# Inside a live loop: offload the blocking write so it cannot stall the
|
||||
# event loop. Fire-and-forget — persistence is best-effort.
|
||||
try:
|
||||
import asyncio
|
||||
asyncio.get_running_loop().run_in_executor(None, _do_store)
|
||||
except Exception as e:
|
||||
logging.debug(f"Memory turn persistence scheduling failed: {e}")
|
||||
_do_store()
|
||||
|
||||
def _process_auto_learning(self):
|
||||
"""Process auto-learning extraction after agent response.
|
||||
|
||||
|
||||
@@ -1904,6 +1904,11 @@ class ToolExecutionMixin:
|
||||
``_atrigger_after_agent_hook`` without any async-safety change.
|
||||
"""
|
||||
from ..hooks import HookEvent, AfterAgentInput
|
||||
# AfterAgentInput.response is typed/serialized as a string (its
|
||||
# to_dict truncates via ``response[:500]``). Structured-output turns
|
||||
# (output_pydantic) return a Pydantic instance, so coerce to str here
|
||||
# to keep string-only hooks (e.g. command hooks) from crashing while
|
||||
# the public chat() return value stays the model.
|
||||
return AfterAgentInput(
|
||||
session_id=getattr(self, '_session_id', 'default'),
|
||||
cwd=os.getcwd(),
|
||||
@@ -1911,7 +1916,7 @@ class ToolExecutionMixin:
|
||||
timestamp=str(time.time()),
|
||||
agent_name=self.name,
|
||||
prompt=prompt if isinstance(prompt, str) else str(prompt),
|
||||
response=response or "",
|
||||
response=response if isinstance(response, str) else (str(response) if response else ""),
|
||||
tools_used=tools_used or [],
|
||||
total_tokens=0,
|
||||
execution_time_ms=(time.time() - start_time) * 1000
|
||||
@@ -1928,7 +1933,16 @@ class ToolExecutionMixin:
|
||||
# Auto-memory extraction (opt-in via MemoryConfig(auto_memory=True))
|
||||
if response:
|
||||
prompt_str = prompt if isinstance(prompt, str) else str(prompt)
|
||||
self._process_auto_memory(prompt_str, str(response))
|
||||
if getattr(self, "_auto_memory", False):
|
||||
self._process_auto_memory(prompt_str, str(response))
|
||||
else:
|
||||
# When memory is enabled but auto_memory extraction is off, still
|
||||
# persist the raw turn so a later Agent sharing the same memory
|
||||
# (e.g. memory={"user_id": uid}) can recall it. Without this the
|
||||
# sync chat/run/start path wrote nothing and cross-session recall
|
||||
# silently returned empty (issue #5595). Mirrors the write already
|
||||
# done on the async unified path.
|
||||
self._persist_memory_turn(prompt_str, str(response))
|
||||
|
||||
# Auto-learning extraction (opt-in via LearnConfig(mode=LearnMode.AGENTIC))
|
||||
self._process_auto_learning()
|
||||
|
||||
@@ -2382,6 +2382,20 @@ class AgentTeam(SpawnAnnounceProtocol):
|
||||
|
||||
def run_all_tasks(self):
|
||||
"""Synchronous version of run_all_tasks method"""
|
||||
# The sync entry point (.start()/.run()) drives tasks one at a time and
|
||||
# cannot batch async_execution=True tasks via asyncio.gather the way
|
||||
# arun_all_tasks does. Warn once so the parallel fan-out feature the
|
||||
# constructor docstring/ValueError advertise doesn't silently degrade to
|
||||
# sequential execution -- point the caller at the async entry point.
|
||||
if self.process in ("workflow", "sequential") and any(
|
||||
getattr(t, 'async_execution', False) for t in self.tasks.values()
|
||||
):
|
||||
logger.warning(
|
||||
"One or more tasks have async_execution=True, but the synchronous "
|
||||
"entry point (.start()/.run()) runs them sequentially -- use "
|
||||
"`await team.astart()` for parallel fan-out."
|
||||
)
|
||||
|
||||
process = Process(
|
||||
tasks=self.tasks,
|
||||
agents=self.agents,
|
||||
|
||||
@@ -7,9 +7,16 @@ another tool's harness, and results can be diffed/aggregated across tools.
|
||||
|
||||
Mapping (1:1 with EvalPort's ``Suite`` / ``ResultSet`` concepts):
|
||||
|
||||
EvalPackage <-> EvalPort Suite (name/description/version + cases + thresholds)
|
||||
EvalCase <-> EvalPort test case (input / expected_output / grader refs)
|
||||
EvalReport -> EvalPort ResultSet (aggregate + per-case pass/score/latency)
|
||||
EvalPackage <-> EvalPort Suite (id/name/description + test_cases + graders)
|
||||
EvalCase <-> EvalPort TestCase (id / input / expected_output / grader refs)
|
||||
EvalReport -> EvalPort ResultSet (summary + per-case grader_results)
|
||||
|
||||
Per-criterion scores map to native EvalPort ``grader_results`` (one per criterion,
|
||||
matching the criterion graders referenced by the Suite). PraisonAI-only fields
|
||||
with no EvalPort slot (package version, thresholds, seed, trajectory records, a
|
||||
nonpositive ``timeout_seconds`` that the spec's ``timeout_ms`` cannot represent,
|
||||
...) are kept under the ``praisonai`` key of the relevant ``metadata`` object so
|
||||
round trips stay lossless.
|
||||
|
||||
Design: this module is intentionally **dependency-free**. It emits and consumes
|
||||
plain spec-shaped dicts (JSON), so the core SDK does not take on ``evalport-sdk``
|
||||
@@ -27,53 +34,90 @@ Example:
|
||||
>>> pkg2.name == pkg.name
|
||||
True
|
||||
"""
|
||||
from typing import Any, Dict, List
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from .package import EvalCase, EvalPackage, EvalReport
|
||||
|
||||
EVALPORT_SPEC_VERSION = "1.0"
|
||||
# EvalPort spec version the emitted documents conform to (semver, per spec).
|
||||
EVALPORT_SPEC_VERSION = "1.0.0-rc.5"
|
||||
|
||||
# Grader that carries PraisonAI's own per-case verdict (EvalResult.passed/score).
|
||||
# EvalPort requires every test case to reference at least one grader.
|
||||
VERDICT_GRADER_ID = "praisonai_verdict"
|
||||
_HANDLER = "praisonaiagents.eval"
|
||||
_META_KEY = "praisonai"
|
||||
|
||||
|
||||
def _grader_def(grader_id: str, description: str) -> Dict[str, Any]:
|
||||
return {
|
||||
"id": grader_id,
|
||||
"type": "custom",
|
||||
"params": {"handler": _HANDLER},
|
||||
"description": description,
|
||||
}
|
||||
|
||||
|
||||
def _case_to_evalport(case: EvalCase) -> Dict[str, Any]:
|
||||
"""Map an ``EvalCase`` to an EvalPort test-case dict.
|
||||
"""Map an ``EvalCase`` to an EvalPort TestCase dict.
|
||||
|
||||
The native ``timeout_seconds`` is emitted as a dedicated top-level field so
|
||||
it never collides with a user-supplied ``metadata["timeout_seconds"]`` key,
|
||||
keeping round trips lossless for both.
|
||||
``criteria`` become grader references (defined at suite level); a case
|
||||
without criteria references the verdict grader. ``timeout_seconds`` maps to
|
||||
the spec's ``timeout_ms``; user ``metadata`` is passed through untouched.
|
||||
"""
|
||||
# A case's own criteria are its graders; the verdict grader is always
|
||||
# referenced too so PraisonAI's pass/score verdict has a home in the
|
||||
# ResultSet (spec requires >= 1 grader per case).
|
||||
graders = [VERDICT_GRADER_ID] + [c for c in case.criteria if c != VERDICT_GRADER_ID]
|
||||
item: Dict[str, Any] = {
|
||||
"id": case.name,
|
||||
"input": case.input,
|
||||
"graders": list(dict.fromkeys(graders)),
|
||||
}
|
||||
if case.expected is not None:
|
||||
item["expected_output"] = case.expected
|
||||
if case.criteria:
|
||||
item["graders"] = list(case.criteria)
|
||||
if case.timeout_seconds is not None:
|
||||
item["timeout_seconds"] = case.timeout_seconds
|
||||
metadata = dict(case.metadata or {})
|
||||
if metadata:
|
||||
item["metadata"] = metadata
|
||||
if case.metadata:
|
||||
item["metadata"] = dict(case.metadata)
|
||||
if case.timeout_seconds is not None and case.timeout_seconds > 0:
|
||||
item["timeout_ms"] = max(1, int(round(case.timeout_seconds * 1000)))
|
||||
elif case.timeout_seconds is not None:
|
||||
# Nonpositive timeout is not spec-representable (timeout_ms >= 1); keep
|
||||
# the raw value under the reserved namespace so import restores it
|
||||
# instead of silently applying the 30s default.
|
||||
meta = item.setdefault("metadata", {})
|
||||
native = dict(meta.get(_META_KEY) or {})
|
||||
native["timeout_seconds"] = case.timeout_seconds
|
||||
meta[_META_KEY] = native
|
||||
return item
|
||||
|
||||
|
||||
def _case_from_evalport(item: Dict[str, Any]) -> EvalCase:
|
||||
"""Map an EvalPort test-case dict to an ``EvalCase``.
|
||||
"""Map an EvalPort TestCase dict to an ``EvalCase``.
|
||||
|
||||
Prefers the dedicated top-level ``timeout_seconds`` field; falls back to a
|
||||
``metadata["timeout_seconds"]`` value for suites emitted by other tools,
|
||||
without mutating the caller's metadata.
|
||||
Reads ``timeout_ms`` (spec); falls back to a legacy top-level or
|
||||
``metadata["timeout_seconds"]`` value, without mutating caller metadata.
|
||||
"""
|
||||
metadata = dict(item.get("metadata") or {})
|
||||
if "timeout_seconds" in item:
|
||||
native = metadata.get(_META_KEY) or {}
|
||||
if "timeout_ms" in item:
|
||||
timeout = item["timeout_ms"] / 1000.0
|
||||
elif "timeout_seconds" in item:
|
||||
timeout = item["timeout_seconds"]
|
||||
elif "timeout_seconds" in native:
|
||||
timeout = native["timeout_seconds"] # nonpositive timeout preserved on export
|
||||
else:
|
||||
timeout = metadata.get("timeout_seconds", 30.0)
|
||||
criteria = []
|
||||
for g in item.get("graders") or item.get("criteria") or []:
|
||||
gid = g.get("id") if isinstance(g, dict) else g
|
||||
if gid and gid != VERDICT_GRADER_ID:
|
||||
criteria.append(gid)
|
||||
return EvalCase(
|
||||
name=item.get("id") or item.get("name") or "case",
|
||||
input=item.get("input", ""),
|
||||
expected=item.get("expected_output", item.get("expected")),
|
||||
criteria=list(item.get("graders") or item.get("criteria") or []),
|
||||
criteria=criteria,
|
||||
metadata=metadata,
|
||||
timeout_seconds=timeout,
|
||||
)
|
||||
@@ -88,16 +132,28 @@ def to_evalport(package: EvalPackage) -> Dict[str, Any]:
|
||||
Returns:
|
||||
A spec-shaped dict suitable for ``openeval.validate.validate_suite()``.
|
||||
"""
|
||||
return {
|
||||
"evalport_version": EVALPORT_SPEC_VERSION,
|
||||
"kind": "suite",
|
||||
criteria = dict.fromkeys(c for case in package.cases for c in case.criteria)
|
||||
criteria.pop(VERDICT_GRADER_ID, None)
|
||||
graders = [_grader_def(VERDICT_GRADER_ID, "PraisonAI per-case verdict (EvalResult.passed/score)")]
|
||||
graders += [_grader_def(c, f"PraisonAI criterion: {c}") for c in criteria]
|
||||
|
||||
suite: Dict[str, Any] = {
|
||||
"version": EVALPORT_SPEC_VERSION,
|
||||
"id": package.name,
|
||||
"name": package.name,
|
||||
"description": package.description,
|
||||
"version": package.version,
|
||||
"cases": [_case_to_evalport(c) for c in package.cases],
|
||||
"thresholds": dict(package.thresholds or {}),
|
||||
"seed": package.seed,
|
||||
"graders": graders,
|
||||
"test_cases": [_case_to_evalport(c) for c in package.cases],
|
||||
"metadata": {
|
||||
_META_KEY: {
|
||||
"version": package.version,
|
||||
"thresholds": dict(package.thresholds or {}),
|
||||
"seed": package.seed,
|
||||
}
|
||||
},
|
||||
}
|
||||
if package.description:
|
||||
suite["description"] = package.description
|
||||
return suite
|
||||
|
||||
|
||||
def from_evalport(suite: Dict[str, Any]) -> EvalPackage:
|
||||
@@ -110,22 +166,68 @@ def from_evalport(suite: Dict[str, Any]) -> EvalPackage:
|
||||
An ``EvalPackage`` whose cases can be run via ``EvalSuite`` /
|
||||
``HarnessEvaluator`` as native ``EvalCase`` objects.
|
||||
"""
|
||||
cases: List[EvalCase] = [_case_from_evalport(c) for c in suite.get("cases", [])]
|
||||
items = suite.get("test_cases", suite.get("cases", []))
|
||||
cases: List[EvalCase] = [_case_from_evalport(c) for c in items]
|
||||
meta = (suite.get("metadata") or {}).get(_META_KEY) or {}
|
||||
legacy = "evalport_version" in suite # documents emitted before spec alignment
|
||||
return EvalPackage(
|
||||
name=suite.get("name", "suite"),
|
||||
name=suite.get("name") or suite.get("id") or "suite",
|
||||
description=suite.get("description", ""),
|
||||
version=suite.get("version", "1.0.0"),
|
||||
version=meta.get("version") or (suite.get("version") if legacy else None) or "1.0.0",
|
||||
cases=cases,
|
||||
thresholds=dict(suite.get("thresholds") or {}),
|
||||
seed=suite.get("seed"),
|
||||
thresholds=dict(meta.get("thresholds") or suite.get("thresholds") or {}),
|
||||
seed=meta.get("seed", suite.get("seed")),
|
||||
)
|
||||
|
||||
|
||||
def report_to_evalport(report: EvalReport) -> Dict[str, Any]:
|
||||
def _unit_score(value: Any) -> Optional[float]:
|
||||
if isinstance(value, bool) or not isinstance(value, (int, float)) or value != value:
|
||||
return None
|
||||
return min(1.0, max(0.0, float(value)))
|
||||
|
||||
|
||||
def _grader_result(grader_id: str, raw_score: Any, passed: bool) -> Dict[str, Any]:
|
||||
"""Build one spec-shaped ``grader_result`` entry.
|
||||
|
||||
Scores are clamped to the spec's [0, 1] range; the raw value is preserved
|
||||
under ``metadata`` when clamping changes it. A null score means "not
|
||||
verified", so ``passed`` is forced ``False`` in that case.
|
||||
"""
|
||||
score = _unit_score(raw_score)
|
||||
grader: Dict[str, Any] = {
|
||||
"grader_id": grader_id,
|
||||
"type": "custom",
|
||||
"score": score,
|
||||
"passed": bool(passed) if score is not None else False,
|
||||
}
|
||||
if score is not None and score != raw_score:
|
||||
grader["metadata"] = {"openeval.raw_score": raw_score}
|
||||
return grader
|
||||
|
||||
|
||||
def _runner_info() -> Dict[str, str]:
|
||||
runner = {"name": "praisonaiagents"}
|
||||
try:
|
||||
from importlib.metadata import version
|
||||
|
||||
runner["version"] = version("praisonaiagents")
|
||||
except Exception:
|
||||
pass
|
||||
return runner
|
||||
|
||||
|
||||
def report_to_evalport(
|
||||
report: EvalReport,
|
||||
*,
|
||||
run_id: Optional[str] = None,
|
||||
started_at: Optional[str] = None,
|
||||
) -> Dict[str, Any]:
|
||||
"""Convert an ``EvalReport`` to an EvalPort ``ResultSet`` dict.
|
||||
|
||||
Args:
|
||||
report: The aggregated report from running an eval package.
|
||||
run_id: Unique run identifier (defaults to a random UUID).
|
||||
started_at: RFC 3339 run start time (defaults to now, UTC).
|
||||
|
||||
Returns:
|
||||
A spec-shaped dict suitable for
|
||||
@@ -133,33 +235,52 @@ def report_to_evalport(report: EvalReport) -> Dict[str, Any]:
|
||||
"""
|
||||
results: List[Dict[str, Any]] = []
|
||||
for r in report.results:
|
||||
verdict = _grader_result(VERDICT_GRADER_ID, r.score, r.passed)
|
||||
grader_results = [verdict]
|
||||
# Surface each per-criterion score as its own grader_result so EvalPort
|
||||
# consumers can diff criteria across tools (they match the criterion
|
||||
# grader ids referenced by the exported Suite). ``passed`` is unknown at
|
||||
# the criterion level, so mirror the case verdict.
|
||||
for name, score in (r.criteria_scores or {}).items():
|
||||
if name == VERDICT_GRADER_ID:
|
||||
continue
|
||||
grader_results.append(_grader_result(name, score, r.passed))
|
||||
|
||||
item: Dict[str, Any] = {
|
||||
"case_id": r.case_name,
|
||||
"passed": r.passed,
|
||||
"score": r.score,
|
||||
"latency_ms": r.latency_ms,
|
||||
"test_case_id": r.case_name,
|
||||
"passed": verdict["passed"],
|
||||
"grader_results": grader_results,
|
||||
}
|
||||
if r.actual_output is not None:
|
||||
item["actual_output"] = r.actual_output
|
||||
item["actual_output"] = str(r.actual_output)
|
||||
if r.latency_ms is not None and r.latency_ms >= 0:
|
||||
item["duration_ms"] = int(round(r.latency_ms))
|
||||
if r.error is not None:
|
||||
item["error"] = r.error
|
||||
if r.criteria_scores:
|
||||
item["grader_scores"] = dict(r.criteria_scores)
|
||||
item["error"] = {"type": "runner_error", "message": str(r.error)}
|
||||
if r.record is not None:
|
||||
item["record"] = r.record
|
||||
item["metadata"] = {_META_KEY: {"record": r.record}}
|
||||
results.append(item)
|
||||
|
||||
summary: Dict[str, Any] = {
|
||||
"total": report.total_cases,
|
||||
"passed": report.passed_cases,
|
||||
"failed": report.failed_cases,
|
||||
"pass_rate": report.pass_rate,
|
||||
}
|
||||
meta: Dict[str, Any] = {"thresholds_met": dict(report.thresholds_met or {})}
|
||||
avg = _unit_score(report.average_score)
|
||||
if avg is not None:
|
||||
summary["avg_score"] = avg
|
||||
if avg != report.average_score:
|
||||
meta["average_score"] = report.average_score # clamped
|
||||
|
||||
return {
|
||||
"evalport_version": EVALPORT_SPEC_VERSION,
|
||||
"kind": "result_set",
|
||||
"suite_name": report.package_name,
|
||||
"summary": {
|
||||
"total": report.total_cases,
|
||||
"passed": report.passed_cases,
|
||||
"failed": report.failed_cases,
|
||||
"pass_rate": report.pass_rate,
|
||||
"average_score": report.average_score,
|
||||
"thresholds_met": dict(report.thresholds_met or {}),
|
||||
},
|
||||
"version": EVALPORT_SPEC_VERSION,
|
||||
"suite_id": report.package_name,
|
||||
"run_id": run_id or uuid.uuid4().hex,
|
||||
"started_at": started_at or datetime.now(timezone.utc).isoformat(),
|
||||
"runner": _runner_info(),
|
||||
"results": results,
|
||||
"summary": summary,
|
||||
"metadata": {_META_KEY: meta},
|
||||
}
|
||||
@@ -35,6 +35,14 @@ _LAZY_IMPORTS = {
|
||||
"SessionProjection": ("praisonaiagents.gateway.session_projection", "SessionProjection"),
|
||||
"SessionProjectionState": ("praisonaiagents.gateway.session_projection", "SessionProjectionState"),
|
||||
"RunView": ("praisonaiagents.gateway.session_projection", "RunView"),
|
||||
# Canonical session-address grammar (Issue #5383)
|
||||
"SessionRef": ("praisonaiagents.gateway.addressing", "SessionRef"),
|
||||
"build_session_path": ("praisonaiagents.gateway.addressing", "build_session_path"),
|
||||
"parse_session_path": ("praisonaiagents.gateway.addressing", "parse_session_path"),
|
||||
"derive_short_id": ("praisonaiagents.gateway.addressing", "derive_short_id"),
|
||||
"slugify": ("praisonaiagents.gateway.addressing", "slugify"),
|
||||
"RESERVED_NAMES": ("praisonaiagents.gateway.addressing", "RESERVED_NAMES"),
|
||||
"SHORT_ID_LEN": ("praisonaiagents.gateway.addressing", "SHORT_ID_LEN"),
|
||||
"GatewayCloseCode": ("praisonaiagents.gateway.protocols", "GatewayCloseCode"),
|
||||
# Declarative method -> required-scope registry (Issue #3206)
|
||||
"GatewayMethodDescriptor": ("praisonaiagents.gateway.protocols", "GatewayMethodDescriptor"),
|
||||
@@ -49,6 +57,11 @@ _LAZY_IMPORTS = {
|
||||
# Saturation / back-pressure observability (Issue #4265)
|
||||
"HealthPressure": ("praisonaiagents.gateway.protocols", "HealthPressure"),
|
||||
"evaluate_pressure": ("praisonaiagents.gateway.protocols", "evaluate_pressure"),
|
||||
# Trusted-proxy ingress attribution (Issue #5312)
|
||||
"IngressTrust": ("praisonaiagents.gateway.protocols", "IngressTrust"),
|
||||
"IngressAttribution": ("praisonaiagents.gateway.protocols", "IngressAttribution"),
|
||||
"resolve_ingress_attribution": ("praisonaiagents.gateway.protocols", "resolve_ingress_attribution"),
|
||||
"parse_forwarded_for": ("praisonaiagents.gateway.protocols", "parse_forwarded_for"),
|
||||
# Push protocols and dataclasses
|
||||
"PushChannelProtocol": ("praisonaiagents.gateway.protocols", "PushChannelProtocol"),
|
||||
"PresenceProtocol": ("praisonaiagents.gateway.protocols", "PresenceProtocol"),
|
||||
@@ -239,6 +252,8 @@ _LAZY_IMPORTS = {
|
||||
# liveness.py — Event-loop liveness watchdog (Issue #3385)
|
||||
"LoopWatchdogPolicy": ("praisonaiagents.gateway.liveness", "LoopWatchdogPolicy"),
|
||||
"LoopWatchdog": ("praisonaiagents.gateway.liveness", "LoopWatchdog"),
|
||||
# liveness.py — Startup-phase watchdog (Issue #5265)
|
||||
"StartupWatchdog": ("praisonaiagents.gateway.liveness", "StartupWatchdog"),
|
||||
# degraded_state.py — Unified degraded-capability registry (Issue #3518)
|
||||
"DegradedOwner": ("praisonaiagents.gateway.degraded_state", "DegradedOwner"),
|
||||
"DegradedCapabilityProtocol": ("praisonaiagents.gateway.degraded_state", "DegradedCapabilityProtocol"),
|
||||
@@ -385,6 +400,14 @@ __all__ = [
|
||||
"SessionProjection",
|
||||
"SessionProjectionState",
|
||||
"RunView",
|
||||
# Canonical session-address grammar (Issue #5383)
|
||||
"SessionRef",
|
||||
"build_session_path",
|
||||
"parse_session_path",
|
||||
"derive_short_id",
|
||||
"slugify",
|
||||
"RESERVED_NAMES",
|
||||
"SHORT_ID_LEN",
|
||||
"GatewayCloseCode",
|
||||
# Declarative method -> required-scope registry (Issue #3206)
|
||||
"GatewayMethodDescriptor",
|
||||
@@ -398,6 +421,11 @@ __all__ = [
|
||||
"compute_config_revision",
|
||||
"HealthPressure",
|
||||
"evaluate_pressure",
|
||||
# Trusted-proxy ingress attribution (Issue #5312)
|
||||
"IngressTrust",
|
||||
"IngressAttribution",
|
||||
"resolve_ingress_attribution",
|
||||
"parse_forwarded_for",
|
||||
# Push protocols (always available)
|
||||
"PushChannelProtocol",
|
||||
"PresenceProtocol",
|
||||
@@ -580,6 +608,8 @@ __all__ = [
|
||||
# Event-loop liveness watchdog (Issue #3385)
|
||||
"LoopWatchdogPolicy",
|
||||
"LoopWatchdog",
|
||||
# Startup-phase watchdog (Issue #5265)
|
||||
"StartupWatchdog",
|
||||
# Unified degraded-capability registry (Issue #3518)
|
||||
"DegradedOwner",
|
||||
"DegradedCapabilityProtocol",
|
||||
|
||||
@@ -0,0 +1,227 @@
|
||||
"""Canonical session-address grammar for the Gateway (Issue #5383).
|
||||
|
||||
A pure, dependency-free grammar that maps between an opaque runtime
|
||||
``session_key`` (a UUID, or an ``agent:<id>:<uuid>``-style key) and a stable,
|
||||
human-readable, reconnect-survivable path — e.g. ``chat/<agentId>/<slug>-<shortId>``.
|
||||
It gives every surface (dashboard deep-links, bot channels, the CLI, the
|
||||
``praisonai-ts`` mirror, reconnecting clients) a single addressing contract to
|
||||
agree on, instead of each one carrying the raw UUID or re-inventing a resolver.
|
||||
|
||||
This module lives in **core** because the grammar is a set of pure functions
|
||||
that define a protocol/contract every surface must share. It has **no**
|
||||
third-party dependencies and does no I/O — the concrete store-backed lookup
|
||||
(``session.resolve``) is implemented in the wrapper behind this contract.
|
||||
|
||||
The grammar is deliberately lossy in one direction: ``build_session_path``
|
||||
derives a short id from the trailing UUID so the whole opaque instance id never
|
||||
appears in a URL, and an optional slug from the session label. ``parse_session_path``
|
||||
recovers a :class:`SessionRef` (agent id + short id + slug hint) that a resolver
|
||||
maps back to the concrete ``session_id``; it never assumes it can reconstruct the
|
||||
key byte-for-byte, except via the exact-key escape hatch (a ``!``-prefixed final
|
||||
segment) for sessions that must keep their literal key.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from typing import Optional
|
||||
from urllib.parse import quote, unquote
|
||||
|
||||
__all__ = [
|
||||
"SessionRef",
|
||||
"SHORT_ID_LEN",
|
||||
"RESERVED_NAMES",
|
||||
"derive_short_id",
|
||||
"slugify",
|
||||
"build_session_path",
|
||||
"parse_session_path",
|
||||
]
|
||||
|
||||
# Length of the short id derived from the trailing hex of a session key. Eight
|
||||
# hex chars (32 bits) keeps URLs terse while staying collision-resistant enough
|
||||
# for a per-agent namespace, where a resolver disambiguates the rare clash.
|
||||
SHORT_ID_LEN = 8
|
||||
|
||||
# Reserved final-segment names that denote a well-known / sentinel session
|
||||
# (e.g. the agent's ``main`` conversation) rather than a ``slug-shortId`` pair.
|
||||
# A resolver treats these as a friendly sentinel, not a short-id lookup.
|
||||
RESERVED_NAMES = frozenset({"main", "global", "default", "root"})
|
||||
|
||||
# Trailing run of hex digits that terminates the key — the opaque UUID tail we
|
||||
# shorten from. Anchored at end-of-string so a *hex run followed by non-hex*
|
||||
# (e.g. ``report-deadbeef.js``) is NOT shortened and instead round-trips via the
|
||||
# literal-key escape hatch; only a genuine hex terminus (a UUID tail) is lossy.
|
||||
_HEX_TAIL_RE = re.compile(r"([0-9a-fA-F]{%d,})$" % SHORT_ID_LEN)
|
||||
|
||||
# A ``slug-shortId`` final segment: optional slug, then a hex short id.
|
||||
_SLUG_SHORT_RE = re.compile(r"^(?:(?P<slug>.+)-)?(?P<short>[0-9a-fA-F]{%d})$" % SHORT_ID_LEN)
|
||||
|
||||
# Characters allowed verbatim in a slug; everything else collapses to ``-``.
|
||||
_SLUG_STRIP_RE = re.compile(r"[^a-z0-9]+")
|
||||
|
||||
# Exact-key escape hatch: a final segment prefixed with ``!`` carries the
|
||||
# percent-encoded literal key verbatim (for sessions that must not be shortened).
|
||||
_LITERAL_PREFIX = "!"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SessionRef:
|
||||
"""A parsed friendly reference to a session, for a resolver to map to an id.
|
||||
|
||||
Exactly one of the resolution hints is authoritative depending on how the
|
||||
path was written:
|
||||
|
||||
- ``literal_key`` set -> resolve by exact key (escape hatch).
|
||||
- ``short_id`` set -> resolve by short id (optionally aided by ``slug_hint``).
|
||||
- ``slug_hint`` only, matching a reserved name -> resolve the sentinel session.
|
||||
|
||||
``agent_id`` scopes the lookup so short ids only need to be unique per agent.
|
||||
"""
|
||||
|
||||
agent_id: str
|
||||
short_id: Optional[str] = None
|
||||
slug_hint: Optional[str] = None
|
||||
literal_key: Optional[str] = None
|
||||
|
||||
|
||||
def derive_short_id(session_key: str) -> Optional[str]:
|
||||
"""Derive a stable short id from the terminating hex run of ``session_key``.
|
||||
|
||||
Returns the last :data:`SHORT_ID_LEN` hex chars of the key's *terminating*
|
||||
hex run (e.g. a UUID tail), lower-cased. The run must end the key: a key
|
||||
whose hex is followed by non-hex characters (e.g. ``report-deadbeef.js``)
|
||||
is deliberately not shortened, so it round-trips via the literal-key escape
|
||||
hatch instead of silently losing its suffix. Returns ``None`` when the key
|
||||
has no terminating hex run long enough to shorten — the caller then falls
|
||||
back to the literal key or reserved-name form.
|
||||
"""
|
||||
if not session_key:
|
||||
return None
|
||||
match = _HEX_TAIL_RE.search(session_key)
|
||||
if not match:
|
||||
return None
|
||||
hex_tail = match.group(1)
|
||||
return hex_tail[-SHORT_ID_LEN:].lower()
|
||||
|
||||
|
||||
def slugify(display_name: Optional[str], max_len: int = 48) -> Optional[str]:
|
||||
"""Turn a human label into a lowercase, dash-separated URL slug.
|
||||
|
||||
Non-alphanumeric runs collapse to a single ``-``; leading/trailing dashes are
|
||||
trimmed. Returns ``None`` for an empty/whitespace-only or unusable name so
|
||||
the path omits the slug rather than emitting an empty segment.
|
||||
"""
|
||||
if not display_name:
|
||||
return None
|
||||
slug = _SLUG_STRIP_RE.sub("-", display_name.strip().lower()).strip("-")
|
||||
if not slug:
|
||||
return None
|
||||
if len(slug) > max_len:
|
||||
slug = slug[:max_len].rstrip("-")
|
||||
return slug or None
|
||||
|
||||
|
||||
def build_session_path(
|
||||
agent_id: str,
|
||||
session_key: str,
|
||||
display_name: Optional[str] = None,
|
||||
namespace: str = "chat",
|
||||
) -> Optional[str]:
|
||||
"""Build a stable, shareable path for a session.
|
||||
|
||||
Produces ``<namespace>/<agentId>/<slug>-<shortId>`` when a short id can be
|
||||
derived (with the slug omitted when there is no usable ``display_name``), so
|
||||
the whole opaque UUID never appears in the URL. When the key has no hex tail
|
||||
to shorten, falls back to an exact-key segment (``!<percent-encoded-key>``)
|
||||
so the address round-trips losslessly. Returns ``None`` if ``agent_id`` or
|
||||
``session_key`` is empty.
|
||||
|
||||
Path segments are percent-escaped, so a key that looks like a file
|
||||
(e.g. ``report.js``) is never mistaken for a static asset.
|
||||
"""
|
||||
if not agent_id or not session_key:
|
||||
return None
|
||||
ns = _seg(namespace) or "chat"
|
||||
agent_seg = _seg(agent_id)
|
||||
if not agent_seg:
|
||||
return None
|
||||
|
||||
short_id = derive_short_id(session_key)
|
||||
if short_id:
|
||||
slug = slugify(display_name)
|
||||
tail = f"{slug}-{short_id}" if slug else short_id
|
||||
else:
|
||||
# No hex tail to shorten: keep the literal key behind the escape hatch.
|
||||
tail = _LITERAL_PREFIX + quote(session_key, safe="")
|
||||
return f"{ns}/{agent_seg}/{tail}"
|
||||
|
||||
|
||||
def parse_session_path(path: str) -> Optional[SessionRef]:
|
||||
"""Parse a session path back into a :class:`SessionRef`.
|
||||
|
||||
Accepts the forms produced by :func:`build_session_path`, with or without a
|
||||
leading namespace segment:
|
||||
|
||||
- ``chat/<agent>/<slug>-<shortId>`` / ``<agent>/<slug>-<shortId>``
|
||||
- ``chat/<agent>/<shortId>`` / ``<agent>/<shortId>``
|
||||
- ``chat/<agent>/main`` (reserved sentinel) / ``<agent>/main``
|
||||
- ``chat/<agent>/!<percent-encoded-key>`` (exact-key escape hatch)
|
||||
|
||||
Returns ``None`` when the path has no agent segment or is otherwise
|
||||
unparseable. A resolver combines the returned hints with its store to reach
|
||||
the concrete ``session_id``.
|
||||
"""
|
||||
if not path:
|
||||
return None
|
||||
parts = [p for p in path.strip().strip("/").split("/") if p]
|
||||
# ``build_session_path`` percent-escapes the namespace and agent into single
|
||||
# segments, so a canonical address is exactly ``<ns>/<agent>/<ref>`` (3) or
|
||||
# ``<agent>/<ref>`` (2). Reject anything longer so a malformed link like
|
||||
# ``chat/x/y/z`` never silently acquires a different meaning by using only
|
||||
# its last two segments.
|
||||
if len(parts) == 3:
|
||||
# Drop the leading namespace segment; agent + final ref remain.
|
||||
agent_id = unquote(parts[-2])
|
||||
final = parts[-1]
|
||||
elif len(parts) == 2:
|
||||
agent_id = unquote(parts[0])
|
||||
final = parts[1]
|
||||
else:
|
||||
return None
|
||||
|
||||
agent_id = agent_id.strip()
|
||||
if not agent_id or not final:
|
||||
return None
|
||||
|
||||
# Exact-key escape hatch.
|
||||
if final.startswith(_LITERAL_PREFIX):
|
||||
literal = unquote(final[len(_LITERAL_PREFIX):])
|
||||
if not literal:
|
||||
return None
|
||||
return SessionRef(agent_id=agent_id, literal_key=literal)
|
||||
|
||||
final = unquote(final)
|
||||
|
||||
# Reserved sentinel (main/global/...): a friendly named session.
|
||||
if final.lower() in RESERVED_NAMES:
|
||||
return SessionRef(agent_id=agent_id, slug_hint=final.lower())
|
||||
|
||||
match = _SLUG_SHORT_RE.match(final)
|
||||
if match:
|
||||
return SessionRef(
|
||||
agent_id=agent_id,
|
||||
short_id=match.group("short").lower(),
|
||||
slug_hint=match.group("slug") or None,
|
||||
)
|
||||
|
||||
# No short id present: treat the whole segment as a slug hint the resolver
|
||||
# can match against session labels.
|
||||
return SessionRef(agent_id=agent_id, slug_hint=final)
|
||||
|
||||
|
||||
def _seg(value: Optional[str]) -> str:
|
||||
"""Percent-escape a single path segment (``/`` and dots are encoded)."""
|
||||
if not value:
|
||||
return ""
|
||||
return quote(value.strip(), safe="")
|
||||
@@ -941,6 +941,73 @@ class AttachmentConfig:
|
||||
)
|
||||
|
||||
|
||||
def _parse_drain_timeout(value: Any) -> Optional[float]:
|
||||
"""Coerce a ``gateway.drain_timeout`` YAML value to a float (or None).
|
||||
|
||||
Tolerant of env-substituted strings (e.g. ``"20"``); an unparseable /
|
||||
negative value falls back to None (no drain) rather than raising, matching
|
||||
the wrapper's fail-open handling of a malformed drain window (#5362).
|
||||
"""
|
||||
if value is None:
|
||||
return None
|
||||
try:
|
||||
parsed = float(value)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
return parsed if parsed >= 0 else None
|
||||
|
||||
|
||||
def _parse_watchdog_enabled(value: Any) -> bool:
|
||||
"""Whether a ``gateway.watchdog`` YAML value enables the loop watchdog.
|
||||
|
||||
Accepts both a bare truthy scalar and the documented block form
|
||||
(``watchdog: { enabled: true, ... }``), mirroring the wrapper's watchdog
|
||||
wiring so the public config reflects the same on/off decision (#5362).
|
||||
"""
|
||||
if isinstance(value, dict):
|
||||
flag = value.get("enabled", False)
|
||||
else:
|
||||
flag = value
|
||||
if isinstance(flag, str):
|
||||
return flag.strip().lower() in ("1", "true", "yes", "on")
|
||||
return bool(flag)
|
||||
|
||||
|
||||
def _parse_watchdog_timeout(value: Any) -> Optional[float]:
|
||||
"""Derive the loop-stall budget (seconds) from a ``gateway.watchdog`` block.
|
||||
|
||||
Reads ``liveness_interval * liveness_strikes`` from the block form when
|
||||
present so the public config exposes the effective wedge budget the CLI
|
||||
``--watchdog-timeout`` flag sets; returns None when unset/unparseable.
|
||||
"""
|
||||
if not isinstance(value, dict):
|
||||
return None
|
||||
try:
|
||||
interval = float(value.get("liveness_interval", 5.0))
|
||||
strikes = int(value.get("liveness_strikes", 3)) or 3
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
budget = interval * strikes
|
||||
return budget if budget > 0 else None
|
||||
|
||||
|
||||
def _parse_watchdog_timeout_scalar(value: Any) -> Optional[float]:
|
||||
"""Coerce an explicit ``gateway.watchdog_timeout`` scalar to a float.
|
||||
|
||||
Mirrors the ``watchdog_timeout > 0`` invariant (see ``GatewayConfig``):
|
||||
tolerant of env-substituted strings (e.g. ``"15"``); an unparseable or
|
||||
non-positive value falls back to None so a malformed timeout disables the
|
||||
budget rather than raising (#5362).
|
||||
"""
|
||||
if value is None:
|
||||
return None
|
||||
try:
|
||||
parsed = float(value)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
return parsed if parsed > 0 else None
|
||||
|
||||
|
||||
@dataclass
|
||||
class GatewayConfig:
|
||||
"""Configuration for the gateway server.
|
||||
@@ -973,6 +1040,13 @@ class GatewayConfig:
|
||||
host: str = "127.0.0.1"
|
||||
port: int = 8765
|
||||
bind_host: Optional[str] = None
|
||||
# Issue #5312: CIDRs/IPs of upstream proxies/tunnels the operator declares
|
||||
# trusted. Empty (default) means the gateway is directly exposed: forwarded
|
||||
# headers are never believed and per-IP policy keys on the socket peer.
|
||||
# When set, the real client IP is resolved by walking the forwarded chain
|
||||
# across trusted hops (see ``resolve_ingress_attribution``); anything
|
||||
# proxy-shaped but unattributable fails closed to the socket peer.
|
||||
trusted_proxies: List[str] = field(default_factory=list)
|
||||
cors_origins: List[str] = field(default_factory=lambda: [])
|
||||
allowed_origins: List[str] = field(default_factory=lambda: [])
|
||||
auth_token: Optional[str] = None
|
||||
@@ -996,6 +1070,14 @@ class GatewayConfig:
|
||||
max_concurrent_runs: int = 0 # Aggregate concurrency ceiling (0 = unlimited)
|
||||
queue_depth: int = 0 # Bounded wait queue when at the ceiling
|
||||
overflow_policy: str = "reject" # reject | queue | shed_oldest
|
||||
# Issue #5362: complete the back-pressure/lifecycle posture on the public
|
||||
# config so a Python embedder / gateway.yaml can pin it declaratively —
|
||||
# parity with the CLI ``--drain-timeout`` / ``--watchdog`` /
|
||||
# ``--watchdog-timeout`` flags. ``None``/False keep today's behaviour
|
||||
# (no drain, watchdog off); a CLI flag still overrides these at start.
|
||||
drain_timeout: Optional[float] = None # graceful-drain window on shutdown (s)
|
||||
watchdog: bool = False # event-loop liveness watchdog (opt-in)
|
||||
watchdog_timeout: Optional[float] = None # loop-stall budget before restart (s)
|
||||
# Issue #5168: per-tenant/per-scope concurrency sub-limit within the global
|
||||
# ceiling so one tenant cannot occupy every run slot. 0 disables the
|
||||
# sub-limit (today's global-only behaviour).
|
||||
@@ -1075,6 +1157,14 @@ class GatewayConfig:
|
||||
raise ValueError(
|
||||
"overflow_policy must be one of 'reject', 'queue', 'shed_oldest'"
|
||||
)
|
||||
if self.drain_timeout is not None and self.drain_timeout < 0:
|
||||
raise ValueError(
|
||||
"drain_timeout must be >= 0 (use 0/None to disable the drain window)"
|
||||
)
|
||||
if self.watchdog_timeout is not None and self.watchdog_timeout <= 0:
|
||||
raise ValueError(
|
||||
"watchdog_timeout must be > 0 (leave unset to use the default budget)"
|
||||
)
|
||||
if self.max_concurrent_runs_per_scope < 0:
|
||||
raise ValueError(
|
||||
"max_concurrent_runs_per_scope must be >= 0 "
|
||||
@@ -1147,6 +1237,7 @@ class GatewayConfig:
|
||||
return {
|
||||
"host": self.host,
|
||||
"port": self.port,
|
||||
"trusted_proxies": list(self.trusted_proxies),
|
||||
"cors_origins": self.cors_origins,
|
||||
"allowed_origins": self.allowed_origins,
|
||||
"auth_token": "***" if self.auth_token else None,
|
||||
@@ -1162,6 +1253,9 @@ class GatewayConfig:
|
||||
"max_concurrent_runs": self.max_concurrent_runs,
|
||||
"queue_depth": self.queue_depth,
|
||||
"overflow_policy": self.overflow_policy,
|
||||
"drain_timeout": self.drain_timeout,
|
||||
"watchdog": self.watchdog,
|
||||
"watchdog_timeout": self.watchdog_timeout,
|
||||
"max_concurrent_runs_per_scope": self.max_concurrent_runs_per_scope,
|
||||
"preauth_max_connections_per_ip": self.preauth_max_connections_per_ip,
|
||||
"max_unauthorized_frames": self.max_unauthorized_frames,
|
||||
@@ -1341,9 +1435,17 @@ class MultiChannelGatewayConfig:
|
||||
for tok, scopes in gw_data["auth_scopes"].items()
|
||||
}
|
||||
|
||||
raw_trusted = gw_data.get("trusted_proxies", [])
|
||||
trusted_proxies = (
|
||||
[str(p).strip() for p in raw_trusted if str(p).strip()]
|
||||
if isinstance(raw_trusted, (list, tuple))
|
||||
else []
|
||||
)
|
||||
|
||||
gateway_config = GatewayConfig(
|
||||
host=gw_data.get("host", "127.0.0.1"),
|
||||
port=gw_data.get("port", 8765),
|
||||
trusted_proxies=trusted_proxies,
|
||||
cors_origins=gw_data.get("cors_origins", []),
|
||||
allowed_origins=gw_data.get("allowed_origins", []),
|
||||
auth_token=gw_data.get("auth_token"),
|
||||
@@ -1360,6 +1462,17 @@ class MultiChannelGatewayConfig:
|
||||
max_concurrent_runs=int(gw_data.get("max_concurrent_runs", 0) or 0),
|
||||
queue_depth=int(gw_data.get("queue_depth", 0) or 0),
|
||||
overflow_policy=str(gw_data.get("overflow_policy", "reject") or "reject"),
|
||||
drain_timeout=_parse_drain_timeout(gw_data.get("drain_timeout")),
|
||||
watchdog=_parse_watchdog_enabled(gw_data.get("watchdog")),
|
||||
# Prefer an explicit ``gateway.watchdog_timeout`` scalar (as emitted
|
||||
# by ``to_dict`` and settable directly, matching the CLI
|
||||
# ``--watchdog-timeout`` flag); otherwise derive the budget from the
|
||||
# ``watchdog`` block so both round-trip and block forms work (#5362).
|
||||
watchdog_timeout=(
|
||||
_parse_watchdog_timeout_scalar(gw_data.get("watchdog_timeout"))
|
||||
if gw_data.get("watchdog_timeout") is not None
|
||||
else _parse_watchdog_timeout(gw_data.get("watchdog"))
|
||||
),
|
||||
max_concurrent_runs_per_scope=int(
|
||||
gw_data.get("max_concurrent_runs_per_scope", 0) or 0
|
||||
),
|
||||
|
||||
@@ -52,6 +52,7 @@ from .protocols import GATEWAY_RESTART_EXIT_CODE
|
||||
__all__ = [
|
||||
"LoopWatchdogPolicy",
|
||||
"LoopWatchdog",
|
||||
"StartupWatchdog",
|
||||
]
|
||||
|
||||
|
||||
@@ -437,3 +438,271 @@ class LoopWatchdog:
|
||||
faulthandler.dump_traceback(file=fh, all_threads=True)
|
||||
except Exception: # pragma: no cover - fail open
|
||||
pass
|
||||
|
||||
|
||||
class StartupWatchdog:
|
||||
"""Front-rung liveness guard for the pre-loop startup window (Issue #5265).
|
||||
|
||||
:class:`LoopWatchdog` only becomes meaningful once the asyncio event loop
|
||||
is running — it probes the loop via ``loop.call_soon_threadsafe``, so it can
|
||||
do nothing before a loop exists. That leaves the window between process
|
||||
entry and "loop is live" (heavy imports, config/secret loading, the initial
|
||||
adapter ``connect()``\\ s) uncovered. A wedge there — a hanging SDK import, a
|
||||
blocking synchronous DNS/network call while minting credentials, a deadlock
|
||||
inside ``connect()`` — leaves the process *alive but making no progress*: it
|
||||
holds no serving loop, answers no probes, and never exits, so an external
|
||||
supervisor never restarts it. It becomes a silent zombie with no diagnostics.
|
||||
|
||||
This primitive closes that gap. It arms a wall-clock **deadline** on a
|
||||
dedicated daemon OS thread *before* the heavy work begins, so it keeps
|
||||
working precisely when the main thread is wedged. If the deadline elapses
|
||||
without :meth:`confirm_loop_live`, it dumps all-thread stacks via
|
||||
:mod:`faulthandler` (so the hang is diagnosable) and hands the process back
|
||||
to the supervisor via ``os._exit`` with a restart-intent code. A legitimately
|
||||
slow-but-alive boot extends the deadline via :meth:`report_startup_progress`
|
||||
(bounded, so a truly wedged phase still trips). :meth:`confirm_loop_live`
|
||||
disarms it the instant the serving loop begins — at which point
|
||||
:class:`LoopWatchdog` takes over.
|
||||
|
||||
Design mirrors :class:`LoopWatchdog`: **opt-in** (nothing arms it unless the
|
||||
embedder calls :meth:`arm`), **fail-open** (any error inside the watchdog
|
||||
must never kill a healthy startup), and **stdlib-only** (it must run when the
|
||||
rest of the process cannot, so its correctness cannot depend on the code it
|
||||
guards). Typical use at the gateway entry point::
|
||||
|
||||
sw = StartupWatchdog()
|
||||
sw.arm(deadline_s=config.startup_watchdog_timeout)
|
||||
load_config(config); sw.report_startup_progress("config loaded")
|
||||
connect_adapters(); sw.report_startup_progress("adapters connected")
|
||||
loop = asyncio.new_event_loop()
|
||||
LoopWatchdog().arm(loop)
|
||||
sw.confirm_loop_live() # disarm; loop watchdog now owns liveness
|
||||
loop.run_forever()
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
on_expire: Literal["dump_and_exit", "dump_only"] = "dump_and_exit",
|
||||
exit_code: int = GATEWAY_RESTART_EXIT_CODE,
|
||||
dump_file: Optional[str] = None,
|
||||
max_extensions: int = 10,
|
||||
) -> None:
|
||||
if on_expire not in ("dump_and_exit", "dump_only"):
|
||||
raise ValueError(
|
||||
"on_expire must be 'dump_and_exit' or 'dump_only'"
|
||||
)
|
||||
if max_extensions < 0:
|
||||
raise ValueError("max_extensions must be >= 0")
|
||||
self.on_expire = on_expire
|
||||
self.exit_code = exit_code
|
||||
self.dump_file = dump_file
|
||||
self.max_extensions = max_extensions
|
||||
self._thread: Optional[threading.Thread] = None
|
||||
# Each armed generation gets its own stop event handed straight to the
|
||||
# thread, so a confirm/re-arm race can never let a stale generation
|
||||
# observe a fresh generation's cleared flag and self-exit (mirrors the
|
||||
# per-generation discipline of LoopWatchdog's deadline mode). The stop
|
||||
# event, once set, stays set: a healthy confirm/disarm is monotonic.
|
||||
self._stop = threading.Event()
|
||||
# A separate wake event carries progress reports: it is briefly set to
|
||||
# nudge the waiting thread to re-read the (extended) deadline, then
|
||||
# cleared. Keeping it distinct from _stop means a progress nudge can
|
||||
# never be mistaken for a disarm (and vice versa).
|
||||
self._wake = threading.Event()
|
||||
self._deadline_s = 0.0
|
||||
self._extensions_used = 0
|
||||
# Progress reports enqueue here; the watchdog thread drains this under
|
||||
# the lock so budget is spent exactly once per report even when several
|
||||
# reports batch between two wakes.
|
||||
self._pending_progress = 0
|
||||
self._expired = False
|
||||
self._confirmed = False
|
||||
self._lock = threading.Lock()
|
||||
|
||||
@property
|
||||
def armed(self) -> bool:
|
||||
return self._thread is not None and self._thread.is_alive()
|
||||
|
||||
@property
|
||||
def expired(self) -> bool:
|
||||
"""True once the startup deadline expired (mainly for ``dump_only``)."""
|
||||
return self._expired
|
||||
|
||||
@property
|
||||
def confirmed(self) -> bool:
|
||||
"""True once :meth:`confirm_loop_live` has disarmed the watchdog."""
|
||||
return self._confirmed
|
||||
|
||||
def arm(self, *, deadline_s: float, dump_stacks: bool = True) -> None:
|
||||
"""Arm a startup deadline on a dedicated daemon thread. Fail-open.
|
||||
|
||||
Idempotent: arming while already armed is a no-op. A non-finite or
|
||||
non-positive ``deadline_s`` is ignored (nothing to guard). ``dump_stacks``
|
||||
toggles the all-thread stack dump on expiry.
|
||||
"""
|
||||
if not math.isfinite(deadline_s) or deadline_s <= 0:
|
||||
return
|
||||
with self._lock:
|
||||
if self.armed:
|
||||
return
|
||||
stop = threading.Event()
|
||||
wake = threading.Event()
|
||||
self._stop = stop
|
||||
self._wake = wake
|
||||
self._deadline_s = deadline_s
|
||||
self._extensions_used = 0
|
||||
self._pending_progress = 0
|
||||
self._expired = False
|
||||
self._confirmed = False
|
||||
self._dump_stacks_on_expire = dump_stacks
|
||||
try:
|
||||
thread = threading.Thread(
|
||||
target=self._run,
|
||||
args=(stop, wake),
|
||||
name="praisonai-startup-watchdog",
|
||||
daemon=True,
|
||||
)
|
||||
thread.start()
|
||||
except Exception: # pragma: no cover - fail open
|
||||
self._thread = None
|
||||
return
|
||||
self._thread = thread
|
||||
|
||||
def report_startup_progress(self, phase: str) -> None:
|
||||
"""Signal forward progress, extending the deadline by one interval.
|
||||
|
||||
A legitimately slow boot (e.g. many adapters, cold DNS) calls this at
|
||||
each phase so it is not killed. Extensions are bounded by
|
||||
``max_extensions`` so a phase that itself wedges still trips the
|
||||
deadline. Fail-open and a no-op when not armed.
|
||||
|
||||
Budget accounting is done by the watchdog thread, not here: each report
|
||||
merely enqueues a pending nudge. If several reports land before the
|
||||
thread wakes, each still costs exactly one extension *and* each still
|
||||
earns one fresh deadline window when the thread drains them — so a later
|
||||
slow-but-progressing phase can never find the budget spent on resets it
|
||||
never received (Issue #5265 review: "progress reports lose extensions").
|
||||
"""
|
||||
if not self.armed:
|
||||
return
|
||||
with self._lock:
|
||||
self._pending_progress += 1
|
||||
wake = self._wake
|
||||
# Wake the waiting thread so it drains the pending nudges, spends budget,
|
||||
# and resets its window once per still-affordable report. The thread
|
||||
# keeps waiting against its own (still-unset) stop event.
|
||||
wake.set()
|
||||
|
||||
def confirm_loop_live(self) -> None:
|
||||
"""Disarm the startup watchdog; the event loop is now live.
|
||||
|
||||
Called immediately before :class:`LoopWatchdog` is armed. Safe to call
|
||||
multiple times / when not armed. Sets the *current* generation's stop
|
||||
event so that exact thread suppresses its self-exit even if it is
|
||||
mid-dump, then drops the reference.
|
||||
"""
|
||||
with self._lock:
|
||||
self._confirmed = True
|
||||
self._stop.set()
|
||||
# Nudge the thread out of its wait immediately so the join is prompt.
|
||||
self._wake.set()
|
||||
thread = self._thread
|
||||
self._thread = None
|
||||
if thread is not None and thread is not threading.current_thread():
|
||||
try:
|
||||
thread.join(timeout=1.0)
|
||||
except Exception: # pragma: no cover - fail open
|
||||
pass
|
||||
|
||||
def _run(self, stop: "threading.Event", wake: "threading.Event") -> None:
|
||||
# Wait out the deadline against *this generation's* events. ``stop`` is
|
||||
# set (and stays set) by a confirm/disarm — that must break us out and
|
||||
# suppress the self-exit. ``wake`` is a transient progress nudge — on
|
||||
# each wake we drain the queued progress reports, spending one extension
|
||||
# of budget per report and resetting the window each time we can still
|
||||
# afford it. A plain timeout with neither event set (and no affordable
|
||||
# pending progress) means the startup genuinely wedged.
|
||||
start = time.monotonic()
|
||||
while not stop.is_set():
|
||||
remaining = (start + self._deadline_s) - time.monotonic()
|
||||
if remaining <= 0:
|
||||
break
|
||||
if wake.wait(remaining):
|
||||
wake.clear()
|
||||
if stop.is_set():
|
||||
return
|
||||
# Drain queued progress reports. Each report costs one extension
|
||||
# and, while budget remains, grants one fresh deadline window —
|
||||
# so batched reports never lose the resets they paid for.
|
||||
extended = False
|
||||
with self._lock:
|
||||
pending = self._pending_progress
|
||||
self._pending_progress = 0
|
||||
for _ in range(pending):
|
||||
if self._extensions_used >= self.max_extensions:
|
||||
break
|
||||
self._extensions_used += 1
|
||||
extended = True
|
||||
if extended:
|
||||
start = time.monotonic()
|
||||
# Whether or not budget remained, loop back and re-check the
|
||||
# (possibly unchanged) deadline against this generation's stop.
|
||||
continue
|
||||
self._on_expire(stop)
|
||||
|
||||
def _on_expire(self, stop: "threading.Event") -> None:
|
||||
# Only fire if this generation was not disarmed. A confirm_loop_live()
|
||||
# that raced in has set this exact event, so a stale generation stays
|
||||
# inert and never self-exits a healthy startup.
|
||||
if stop.is_set():
|
||||
return
|
||||
self._expired = True
|
||||
if getattr(self, "_dump_stacks_on_expire", True):
|
||||
try:
|
||||
self._dump_stacks()
|
||||
except Exception: # pragma: no cover - fail open
|
||||
pass
|
||||
if self.on_expire == "dump_and_exit":
|
||||
# If confirm_loop_live() raced in while we were dumping, honour it.
|
||||
if stop.is_set():
|
||||
return
|
||||
try:
|
||||
sys.stderr.flush()
|
||||
except Exception: # pragma: no cover - fail open
|
||||
pass
|
||||
# Final re-check the instant before the irreversible os._exit: the
|
||||
# flush above can block (e.g. a full pipe) long enough for a
|
||||
# concurrent confirm_loop_live() to land. A startup that has just
|
||||
# confirmed a live loop must never be terminated (Issue #5265
|
||||
# review: "confirmation cannot prevent late exit").
|
||||
if stop.is_set():
|
||||
return
|
||||
os._exit(self.exit_code)
|
||||
|
||||
def _dump_stacks(self) -> None:
|
||||
reason = (
|
||||
f"gateway startup did not confirm a live event loop within "
|
||||
f"~{self._deadline_s:.0f}s"
|
||||
)
|
||||
header = (
|
||||
f"praisonai-startup-watchdog: {reason}; "
|
||||
f"dumping all-thread stacks\n"
|
||||
)
|
||||
try:
|
||||
sys.stderr.write(header)
|
||||
except Exception: # pragma: no cover - fail open
|
||||
pass
|
||||
try:
|
||||
faulthandler.dump_traceback(file=sys.stderr, all_threads=True)
|
||||
except Exception: # pragma: no cover - fail open
|
||||
pass
|
||||
dump_file = self.dump_file
|
||||
if dump_file:
|
||||
try:
|
||||
with open(dump_file, "a", encoding="utf-8") as fh:
|
||||
fh.write(header)
|
||||
fh.flush()
|
||||
faulthandler.dump_traceback(file=fh, all_threads=True)
|
||||
except Exception: # pragma: no cover - fail open
|
||||
pass
|
||||
@@ -2555,6 +2555,225 @@ def resolve_auth_mode(bind_host: str, configured: Optional[AuthMode] = None) ->
|
||||
return "local" if is_loopback(bind_host) else "token"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Ingress attribution / trusted-proxy real-client-IP resolution (Issue #5312)
|
||||
# ---------------------------------------------------------------------------
|
||||
#
|
||||
# The gateway keys rate-limiting, the pre-auth connection budget, and the
|
||||
# operator identity on the *raw socket peer*. Behind a reverse proxy or tunnel
|
||||
# (nginx, Caddy, Traefik, a cloud LB, Cloudflare, Tailscale Serve/Funnel) that
|
||||
# peer is the *proxy* for every request, so every per-IP bucket collapses to
|
||||
# one and the operator id becomes identical for all callers. Trusting
|
||||
# ``X-Forwarded-For`` unconditionally is worse — a directly-reachable gateway
|
||||
# would accept a spoofed client IP any attacker can set.
|
||||
#
|
||||
# This pure, import-free contract resolves the real client address *only* by
|
||||
# walking the forwarded chain across hops the operator has explicitly declared
|
||||
# trusted, and **fails closed** for anything proxy-shaped but unattributable
|
||||
# (rate-limited as one bucket keyed on the socket peer; never trusts a header).
|
||||
# It lives in core so every seam consults one unit-testable decision, exactly
|
||||
# like the sibling ``resolve_route`` / ``evaluate_pressure`` contracts.
|
||||
|
||||
IngressTrust = Literal[
|
||||
"direct-local",
|
||||
"direct-remote",
|
||||
"trusted-proxy",
|
||||
"tunnel",
|
||||
"unattributable-proxy",
|
||||
]
|
||||
"""Classification of where an inbound request actually came from.
|
||||
|
||||
- ``direct-local``: loopback peer, no proxy headers (local CLI / same host).
|
||||
- ``direct-remote``: a remote peer talking to us directly, no proxy headers.
|
||||
- ``trusted-proxy``: arrived via one or more hops all in ``trusted_proxies``;
|
||||
the real client IP was resolved from the forwarded chain.
|
||||
- ``tunnel``: a trusted loopback/private tunnel hop (e.g. Tailscale/Cloudflare
|
||||
sidecar on loopback) presented a forwarded client — resolved like a trusted
|
||||
proxy.
|
||||
- ``unattributable-proxy``: proxy-shaped (forwarded headers present) but the
|
||||
immediate hop is not declared trusted — fails closed, header not trusted.
|
||||
"""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class IngressAttribution:
|
||||
"""Resolved answer to "who/where is this request from", consulted before
|
||||
any per-IP policy.
|
||||
|
||||
Attributes:
|
||||
trust: The :data:`IngressTrust` classification.
|
||||
client_ip: The subject to key per-IP policy on. The *real* client only
|
||||
when it was resolved across trusted hops; otherwise the raw socket
|
||||
peer (never a spoofable header value).
|
||||
via_proxy: Whether forwarded headers were present at all.
|
||||
fail_closed: True when the request was proxy-shaped but unattributable —
|
||||
the caller must rate-limit it as a single bucket keyed on the socket
|
||||
peer and must never trust a client IP from a header.
|
||||
"""
|
||||
|
||||
trust: IngressTrust
|
||||
client_ip: str
|
||||
via_proxy: bool
|
||||
fail_closed: bool
|
||||
|
||||
|
||||
def parse_forwarded_for(header: Optional[str]) -> "List[str]":
|
||||
"""Parse an ``X-Forwarded-For`` header into an ordered list of hops.
|
||||
|
||||
The chain is left→right, client-first: ``client, proxy1, proxy2``. Empty
|
||||
tokens and surrounding whitespace are dropped; ``None``/empty yields an
|
||||
empty list. Pure string handling — no name resolution, no validation of
|
||||
whether each token is a real IP (that is the caller's trust decision).
|
||||
"""
|
||||
if not header:
|
||||
return []
|
||||
return [tok.strip() for tok in header.split(",") if tok.strip()]
|
||||
|
||||
|
||||
def _is_valid_ip(host: Optional[str]) -> bool:
|
||||
"""Return whether ``host`` parses as a literal IPv4/IPv6 address.
|
||||
|
||||
Forwarded-header tokens are attacker-influenced strings; a value such as
|
||||
``unknown`` or an arbitrary label must never become a per-IP policy subject
|
||||
(it would group unrelated callers or split one caller across buckets). The
|
||||
caller uses this to fail closed to the socket peer for a non-IP hop.
|
||||
"""
|
||||
if not host:
|
||||
return False
|
||||
import ipaddress
|
||||
|
||||
try:
|
||||
ipaddress.ip_address(str(host).strip())
|
||||
return True
|
||||
except ValueError:
|
||||
return False
|
||||
|
||||
|
||||
def _is_trusted_hop(host: str, trusted_proxies: "Sequence[str]") -> bool:
|
||||
"""Return whether ``host`` is a member of the declared trusted set.
|
||||
|
||||
A trusted entry may be a bare IP or a CIDR (``10.0.0.0/8``). Loopback is
|
||||
*not* implicitly trusted here — the caller decides how to treat a loopback
|
||||
tunnel — so this stays a pure membership test.
|
||||
"""
|
||||
import ipaddress
|
||||
|
||||
try:
|
||||
ip = ipaddress.ip_address(host)
|
||||
except ValueError:
|
||||
return False
|
||||
for entry in trusted_proxies:
|
||||
entry = str(entry).strip()
|
||||
if not entry:
|
||||
continue
|
||||
try:
|
||||
if "/" in entry:
|
||||
if ip in ipaddress.ip_network(entry, strict=False):
|
||||
return True
|
||||
elif ip == ipaddress.ip_address(entry):
|
||||
return True
|
||||
except ValueError:
|
||||
continue
|
||||
return False
|
||||
|
||||
|
||||
def resolve_ingress_attribution(
|
||||
*,
|
||||
peer_ip: str,
|
||||
forwarded_for: "Sequence[str]" = (),
|
||||
real_ip: Optional[str] = None,
|
||||
trusted_proxies: "Sequence[str]" = (),
|
||||
) -> IngressAttribution:
|
||||
"""Resolve the real client attribution for an inbound request.
|
||||
|
||||
Pure and side-effect-free so it is unit-testable in isolation and every
|
||||
seam (rate limiters, connection budget, operator id) consults the same
|
||||
decision.
|
||||
|
||||
Args:
|
||||
peer_ip: Raw socket peer (``request.client.host``).
|
||||
forwarded_for: Parsed ``X-Forwarded-For`` chain, left→right
|
||||
(client-first). See :func:`parse_forwarded_for`.
|
||||
real_ip: Optional ``X-Real-IP`` value (used only when there is no
|
||||
forwarded chain and the immediate hop is trusted).
|
||||
trusted_proxies: CIDRs/IPs the operator declares trusted.
|
||||
|
||||
Returns:
|
||||
An :class:`IngressAttribution`. The real client IP is resolved *only*
|
||||
across hops in ``trusted_proxies``; anything proxy-shaped but
|
||||
unattributable fails closed to the socket peer.
|
||||
|
||||
Resolution rules:
|
||||
* No proxy headers → ``direct-local`` (loopback peer) or
|
||||
``direct-remote``; ``client_ip`` is the peer, never fail-closed.
|
||||
* Proxy headers present but the socket peer is not a trusted hop →
|
||||
``unattributable-proxy``, ``fail_closed=True``, ``client_ip`` = peer
|
||||
(a spoofable header is never trusted on a directly-reachable
|
||||
gateway).
|
||||
* Socket peer is a trusted hop → walk the forwarded chain right→left,
|
||||
peeling trusted hops, and stop at the first untrusted address: that
|
||||
address is the real client. If the chain is empty, fall back to
|
||||
``real_ip`` then the peer. The resolved client must be a literal IP —
|
||||
a malformed/non-IP hop fails closed to the peer. A loopback trusted
|
||||
hop is classified ``tunnel``; otherwise ``trusted-proxy``.
|
||||
"""
|
||||
has_headers = bool(forwarded_for) or bool(real_ip)
|
||||
|
||||
# No forwarded headers at all — attribute straight to the socket peer.
|
||||
if not has_headers:
|
||||
trust: IngressTrust = "direct-local" if is_loopback(peer_ip) else "direct-remote"
|
||||
return IngressAttribution(
|
||||
trust=trust,
|
||||
client_ip=peer_ip,
|
||||
via_proxy=False,
|
||||
fail_closed=False,
|
||||
)
|
||||
|
||||
# Proxy-shaped. The immediate hop (socket peer) must itself be trusted for
|
||||
# any header to be believed; otherwise fail closed to the socket peer.
|
||||
if not _is_trusted_hop(peer_ip, trusted_proxies):
|
||||
return IngressAttribution(
|
||||
trust="unattributable-proxy",
|
||||
client_ip=peer_ip,
|
||||
via_proxy=True,
|
||||
fail_closed=True,
|
||||
)
|
||||
|
||||
# Trusted immediate hop. Walk the forwarded chain right→left, peeling hops
|
||||
# that are themselves trusted; the first untrusted address is the real
|
||||
# client. Stop as soon as we hit it so a client-supplied spoofed prefix
|
||||
# cannot be peeled past.
|
||||
client_ip = None
|
||||
for hop in reversed(list(forwarded_for)):
|
||||
if _is_trusted_hop(hop, trusted_proxies):
|
||||
continue
|
||||
client_ip = hop
|
||||
break
|
||||
if client_ip is None:
|
||||
# Entire chain was trusted (or empty). Prefer X-Real-IP, then peer.
|
||||
client_ip = (real_ip.strip() if real_ip else None) or peer_ip
|
||||
|
||||
# The resolved client must be a literal IP before it keys per-IP policy. A
|
||||
# malformed hop (e.g. ``unknown`` or an arbitrary label injected upstream)
|
||||
# would otherwise group unrelated callers or split one caller across
|
||||
# buckets — fail closed to the socket peer instead (Greptile P2).
|
||||
if not _is_valid_ip(client_ip):
|
||||
return IngressAttribution(
|
||||
trust="unattributable-proxy",
|
||||
client_ip=peer_ip,
|
||||
via_proxy=True,
|
||||
fail_closed=True,
|
||||
)
|
||||
|
||||
trust = "tunnel" if is_loopback(peer_ip) else "trusted-proxy"
|
||||
return IngressAttribution(
|
||||
trust=trust,
|
||||
client_ip=client_ip,
|
||||
via_proxy=True,
|
||||
fail_closed=False,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Weak / placeholder secret guard (Issue #3259)
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -7582,6 +7801,10 @@ def _register_core_gateway_methods() -> None:
|
||||
"server_reply": OperatorScope.WRITE,
|
||||
"session.status": OperatorScope.READ,
|
||||
"session.transcript": OperatorScope.READ,
|
||||
# Resolve a friendly reference (short id / slug / label / ``main``
|
||||
# sentinel, optionally agent-scoped) to a concrete session id (Issue
|
||||
# #5383). Read-only lookup: it reveals no more than session.status.
|
||||
"session.resolve": OperatorScope.READ,
|
||||
"approvals.resolve": OperatorScope.APPROVALS,
|
||||
"pairing.approve": OperatorScope.PAIRING,
|
||||
"pairing.revoke": OperatorScope.PAIRING,
|
||||
|
||||
@@ -211,6 +211,8 @@ class ChromaKnowledgeAdapter:
|
||||
**kwargs: Any):
|
||||
"""Search for relevant content in ChromaDB."""
|
||||
from ..models import SearchResult, SearchResultItem
|
||||
from ..protocols import require_scope
|
||||
require_scope(user_id, agent_id, run_id, "search", backend="chroma")
|
||||
|
||||
# Get embedding for query
|
||||
embedding_model, embedding_kwargs = self._embedding_call()
|
||||
@@ -236,9 +238,15 @@ class ChromaKnowledgeAdapter:
|
||||
if run_id is not None:
|
||||
where_filter["run_id"] = run_id
|
||||
|
||||
# Merge with additional filters
|
||||
# Merge with additional filters, but never let a caller-supplied filter
|
||||
# overwrite the tenant-scope constraint: filters={"user_id": "bob"} must
|
||||
# not widen an already-scoped query back onto another tenant's data
|
||||
# (issue #5319).
|
||||
if filters:
|
||||
where_filter.update(filters)
|
||||
for key, value in filters.items():
|
||||
if key in where_filter:
|
||||
continue
|
||||
where_filter[key] = value
|
||||
|
||||
# Search ChromaDB
|
||||
try:
|
||||
@@ -289,6 +297,8 @@ class ChromaKnowledgeAdapter:
|
||||
**kwargs: Any):
|
||||
"""Add content to ChromaDB."""
|
||||
from ..models import AddResult
|
||||
from ..protocols import require_scope
|
||||
require_scope(user_id, agent_id, run_id, "add", backend="chroma")
|
||||
|
||||
# Convert content to string
|
||||
content_str = str(content)
|
||||
@@ -375,6 +385,8 @@ class ChromaKnowledgeAdapter:
|
||||
run_id: Optional[str] = None, limit: int = 100, **kwargs: Any):
|
||||
"""Get all items from ChromaDB."""
|
||||
from ..models import SearchResult, SearchResultItem
|
||||
from ..protocols import require_scope
|
||||
require_scope(user_id, agent_id, run_id, "get_all", backend="chroma")
|
||||
|
||||
try:
|
||||
# ChromaDB doesn't support get_all with filters easily, so we'll use peek
|
||||
@@ -510,8 +522,11 @@ class SQLiteKnowledgeAdapter:
|
||||
**kwargs: Any):
|
||||
"""Search for relevant content in SQLite."""
|
||||
from ..models import SearchResult, SearchResultItem
|
||||
from ..protocols import require_scope
|
||||
import json
|
||||
|
||||
require_scope(user_id, agent_id, run_id, "search", backend="sqlite")
|
||||
|
||||
conn = self._get_conn()
|
||||
|
||||
# Build query with filters
|
||||
@@ -528,6 +543,19 @@ class SQLiteKnowledgeAdapter:
|
||||
sql += " AND run_id = ?"
|
||||
params.append(run_id)
|
||||
|
||||
# Apply metadata filters against the stored JSON metadata column so a
|
||||
# filtered search returns the same subset chroma/mem0 already return
|
||||
# instead of silently discarding the filters (issue #5319).
|
||||
if filters:
|
||||
for key, value in filters.items():
|
||||
# Quote the key in the JSON path so a top-level metadata key that
|
||||
# itself contains dots (e.g. "source.type") is matched literally
|
||||
# instead of being interpreted as a nested path (issue #5319).
|
||||
escaped_key = key.replace('"', '""')
|
||||
sql += " AND json_extract(metadata, ?) = ?"
|
||||
params.append(f'$."{escaped_key}"')
|
||||
params.append(value)
|
||||
|
||||
sql += " LIMIT ?"
|
||||
params.append(limit)
|
||||
|
||||
@@ -551,9 +579,12 @@ class SQLiteKnowledgeAdapter:
|
||||
**kwargs: Any):
|
||||
"""Add content to SQLite."""
|
||||
from ..models import AddResult
|
||||
from ..protocols import require_scope
|
||||
import json
|
||||
import time
|
||||
|
||||
require_scope(user_id, agent_id, run_id, "add", backend="sqlite")
|
||||
|
||||
content_str = str(content)
|
||||
doc_id = str(time.time_ns())
|
||||
metadata_json = json.dumps(metadata or {})
|
||||
@@ -596,8 +627,11 @@ class SQLiteKnowledgeAdapter:
|
||||
run_id: Optional[str] = None, limit: int = 100, **kwargs: Any):
|
||||
"""Get all items from SQLite."""
|
||||
from ..models import SearchResult, SearchResultItem
|
||||
from ..protocols import require_scope
|
||||
import json
|
||||
|
||||
require_scope(user_id, agent_id, run_id, "get_all", backend="sqlite")
|
||||
|
||||
conn = self._get_conn()
|
||||
|
||||
# Build query with filters
|
||||
|
||||
@@ -169,7 +169,12 @@ class MongoDBKnowledgeAdapter:
|
||||
|
||||
Returns:
|
||||
Document ID of inserted content
|
||||
|
||||
Raises:
|
||||
ScopeRequiredError: If no scope identifier is provided
|
||||
"""
|
||||
from ..protocols import require_scope
|
||||
require_scope(user_id, agent_id, run_id, "add", backend="mongodb")
|
||||
try:
|
||||
# Generate embedding if model is available
|
||||
embedding = None
|
||||
@@ -210,6 +215,7 @@ class MongoDBKnowledgeAdapter:
|
||||
user_id: Optional[str] = None,
|
||||
agent_id: Optional[str] = None,
|
||||
run_id: Optional[str] = None,
|
||||
filters: Optional[Dict[str, Any]] = None,
|
||||
**kwargs
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
@@ -221,11 +227,17 @@ class MongoDBKnowledgeAdapter:
|
||||
user_id: Optional user scope for tenant isolation
|
||||
agent_id: Optional agent scope for tenant isolation
|
||||
run_id: Optional run scope for tenant isolation
|
||||
filters: Optional metadata filters applied to the query
|
||||
**kwargs: Additional search parameters
|
||||
|
||||
Returns:
|
||||
List of matching documents
|
||||
|
||||
Raises:
|
||||
ScopeRequiredError: If no scope identifier is provided
|
||||
"""
|
||||
from ..protocols import require_scope
|
||||
require_scope(user_id, agent_id, run_id, "search", backend="mongodb")
|
||||
try:
|
||||
results = []
|
||||
|
||||
@@ -237,6 +249,13 @@ class MongoDBKnowledgeAdapter:
|
||||
"run_id": run_id,
|
||||
}.items() if v is not None
|
||||
}
|
||||
|
||||
# Fold metadata filters in alongside the scope so a filtered search
|
||||
# returns the same subset chroma/mem0 already return instead of
|
||||
# silently discarding the filters (issue #5319).
|
||||
if filters:
|
||||
for key, value in filters.items():
|
||||
scope_filter[f"metadata.{key}"] = value
|
||||
|
||||
# Try vector search first if available
|
||||
if self.use_vector_search and self.embedding_model and self._is_atlas_connection():
|
||||
@@ -332,6 +351,127 @@ class MongoDBKnowledgeAdapter:
|
||||
logger.error(f"Failed to delete document {document_id}: {e}")
|
||||
return False
|
||||
|
||||
def get(self, item_id: str, **kwargs):
|
||||
"""Get a specific document by ID from the knowledge store.
|
||||
|
||||
Returns a ``SearchResultItem`` (or ``None``) so the result shape matches
|
||||
the shared KnowledgeStoreProtocol and the other adapters — a caller that
|
||||
reads ``.text``/``.metadata`` or calls ``.to_dict()`` must not break when
|
||||
it switches to the MongoDB backend (issue #5319).
|
||||
"""
|
||||
from ..models import SearchResultItem
|
||||
try:
|
||||
from bson import ObjectId
|
||||
doc = self.collection.find_one({"_id": ObjectId(item_id)})
|
||||
if not doc:
|
||||
return None
|
||||
return SearchResultItem(
|
||||
id=str(doc["_id"]),
|
||||
text=doc.get("content", ""),
|
||||
metadata=doc.get("metadata", {}) or {},
|
||||
score=1.0,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to get document {item_id}: {e}")
|
||||
return None
|
||||
|
||||
def get_all(
|
||||
self,
|
||||
*,
|
||||
user_id: Optional[str] = None,
|
||||
agent_id: Optional[str] = None,
|
||||
run_id: Optional[str] = None,
|
||||
limit: int = 100,
|
||||
**kwargs
|
||||
):
|
||||
"""Get all documents matching the given scope.
|
||||
|
||||
Returns a ``SearchResult`` to match the shared KnowledgeStoreProtocol
|
||||
and the other adapters.
|
||||
|
||||
Raises:
|
||||
ScopeRequiredError: If no scope identifier is provided
|
||||
"""
|
||||
from ..models import SearchResult, SearchResultItem
|
||||
from ..protocols import require_scope
|
||||
require_scope(user_id, agent_id, run_id, "get_all", backend="mongodb")
|
||||
try:
|
||||
scope_filter = {
|
||||
k: v for k, v in {
|
||||
"user_id": user_id,
|
||||
"agent_id": agent_id,
|
||||
"run_id": run_id,
|
||||
}.items() if v is not None
|
||||
}
|
||||
items = []
|
||||
for doc in self.collection.find(scope_filter).limit(limit):
|
||||
items.append(SearchResultItem(
|
||||
id=str(doc["_id"]),
|
||||
text=doc.get("content", ""),
|
||||
metadata=doc.get("metadata", {}) or {},
|
||||
score=1.0,
|
||||
))
|
||||
return SearchResult(results=items)
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to get_all from MongoDB: {e}")
|
||||
return SearchResult(results=[])
|
||||
|
||||
def update(self, item_id: str, content: Any, **kwargs):
|
||||
"""Update an existing document's content/metadata by ID.
|
||||
|
||||
Returns an ``AddResult`` to match the shared KnowledgeStoreProtocol and
|
||||
the other adapters (issue #5319).
|
||||
"""
|
||||
from ..models import AddResult
|
||||
try:
|
||||
from bson import ObjectId
|
||||
update_fields: Dict[str, Any] = {"content": str(content)}
|
||||
if "metadata" in kwargs and kwargs["metadata"] is not None:
|
||||
update_fields["metadata"] = kwargs["metadata"]
|
||||
result = self.collection.update_one(
|
||||
{"_id": ObjectId(item_id)},
|
||||
{"$set": update_fields}
|
||||
)
|
||||
if result.matched_count > 0:
|
||||
return AddResult(success=True, id=item_id)
|
||||
return AddResult(success=False, message=f"No document matched id {item_id}")
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to update document {item_id}: {e}")
|
||||
return AddResult(success=False, message=str(e))
|
||||
|
||||
def delete_all(
|
||||
self,
|
||||
*,
|
||||
user_id: Optional[str] = None,
|
||||
agent_id: Optional[str] = None,
|
||||
run_id: Optional[str] = None,
|
||||
**kwargs
|
||||
) -> bool:
|
||||
"""Delete all documents matching the given scope.
|
||||
|
||||
Enforces the same tenant-scope contract as the mem0 backend: an
|
||||
unscoped call raises ``ScopeRequiredError`` instead of silently wiping
|
||||
every tenant's documents from the (typically shared) collection.
|
||||
|
||||
Raises:
|
||||
ScopeRequiredError: If no scope identifier is provided
|
||||
"""
|
||||
from ..protocols import require_scope
|
||||
require_scope(user_id, agent_id, run_id, "delete_all", backend="mongodb")
|
||||
try:
|
||||
scope_filter = {
|
||||
k: v for k, v in {
|
||||
"user_id": user_id,
|
||||
"agent_id": agent_id,
|
||||
"run_id": run_id,
|
||||
}.items() if v is not None
|
||||
}
|
||||
self.collection.delete_many(scope_filter)
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to delete_all from MongoDB: {e}")
|
||||
return False
|
||||
|
||||
def close(self):
|
||||
"""Close MongoDB connection."""
|
||||
if hasattr(self, 'client'):
|
||||
|
||||
@@ -588,12 +588,48 @@ def get_retry_delay(category: ErrorCategory, attempt: int = 1, base_delay: float
|
||||
return 0
|
||||
|
||||
|
||||
def extract_retry_after(error: Exception) -> Optional[float]:
|
||||
def _retry_after_http_date_delay(value: str) -> Optional[float]:
|
||||
"""Return seconds until an HTTP-date ``Retry-After`` value, else ``None``.
|
||||
|
||||
RFC 7231 allows ``Retry-After`` to be either delta-seconds or an absolute
|
||||
HTTP-date. Delta-seconds is handled by the caller; this parses the date
|
||||
form using the stdlib (no new dependency) and returns the non-negative
|
||||
delay from now. A value in the past or unparseable yields ``None`` so the
|
||||
caller falls through to its other signals.
|
||||
"""
|
||||
try:
|
||||
from email.utils import parsedate_to_datetime
|
||||
except Exception: # pragma: no cover - stdlib always present
|
||||
return None
|
||||
try:
|
||||
when = parsedate_to_datetime(value.strip())
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
if when is None:
|
||||
return None
|
||||
import time as _time
|
||||
from datetime import timezone
|
||||
|
||||
# An HTTP-date is UTC; a naive parse result is treated as UTC per the RFC.
|
||||
if when.tzinfo is None:
|
||||
when = when.replace(tzinfo=timezone.utc)
|
||||
delay = when.timestamp() - _time.time()
|
||||
return delay if delay > 0 else None
|
||||
|
||||
|
||||
def extract_retry_after(
|
||||
error: Exception, cap_seconds: float = 300.0,
|
||||
) -> Optional[float]:
|
||||
"""Extract Retry-After header value from rate limit errors.
|
||||
|
||||
|
||||
Args:
|
||||
error: Exception potentially containing Retry-After info
|
||||
|
||||
cap_seconds: Upper bound applied to the parsed value. Defaults to 300s
|
||||
(5 minutes) for in-tick backoff callers. A scheduler *hold* passes a
|
||||
larger cap so a job can be parked past a long provider reset window
|
||||
(e.g. an hourly quota) instead of re-firing every tick — capping at
|
||||
5 minutes there would defeat the park.
|
||||
|
||||
Returns:
|
||||
Delay in seconds if found, None otherwise
|
||||
"""
|
||||
@@ -608,14 +644,21 @@ def extract_retry_after(error: Exception) -> Optional[float]:
|
||||
retry_after = None
|
||||
if retry_after is not None:
|
||||
try:
|
||||
return min(float(retry_after), 300.0) # Cap at 5 minutes
|
||||
return min(float(retry_after), cap_seconds)
|
||||
except (ValueError, TypeError):
|
||||
pass # Not a plain number (could be an HTTP-date); fall through
|
||||
# Not a plain delta-seconds value — RFC 7231 also permits an
|
||||
# HTTP-date. Parse it so a provider that sends an absolute reset
|
||||
# instant still yields a usable delay (else a long quota window
|
||||
# would leave the job un-parked). A past/unparseable date yields
|
||||
# no delay and falls through to the other signals.
|
||||
delay = _retry_after_http_date_delay(str(retry_after))
|
||||
if delay is not None:
|
||||
return min(delay, cap_seconds)
|
||||
|
||||
# 2. Some SDKs expose a numeric ``retry_after`` attribute directly.
|
||||
retry_after_attr = getattr(error, "retry_after", None)
|
||||
if isinstance(retry_after_attr, (int, float)):
|
||||
return min(float(retry_after_attr), 300.0)
|
||||
return min(float(retry_after_attr), cap_seconds)
|
||||
|
||||
error_str = str(error)
|
||||
|
||||
@@ -632,7 +675,7 @@ def extract_retry_after(error: Exception) -> Optional[float]:
|
||||
if match:
|
||||
try:
|
||||
delay = float(match.group(1))
|
||||
return min(delay, 300.0) # Cap at 5 minutes
|
||||
return min(delay, cap_seconds)
|
||||
except (ValueError, IndexError):
|
||||
continue
|
||||
|
||||
|
||||
@@ -8,7 +8,6 @@ import re
|
||||
import inspect
|
||||
import asyncio
|
||||
import threading
|
||||
import contextvars
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Dict, List, Optional, Union, Literal, Callable, TYPE_CHECKING, Protocol
|
||||
|
||||
@@ -599,11 +598,11 @@ Respond with ONLY a valid JSON tool call in this format:
|
||||
# Token tracking
|
||||
self.last_token_metrics: Optional[TokenMetrics] = None
|
||||
self.session_token_metrics: Optional[TokenMetrics] = None
|
||||
# Agent attribution is task-local: a single LLM instance is shared by
|
||||
# concurrent agents (gateway channels, asyncio.gather), so a plain
|
||||
# attribute would let one agent's set_current_agent() overwrite another's
|
||||
# while it is still awaiting its own completion, misattributing tokens
|
||||
# (issue #5052). A ContextVar isolates the value per async task/context.
|
||||
# Agent attribution is task-local: a single LLM instance is often shared
|
||||
# by several concurrently-running agents (gateway channels, asyncio.gather),
|
||||
# so a plain attribute would let one agent's set_current_agent() clobber
|
||||
# another's mid-await and misattribute token spend (issues #5052/#3933).
|
||||
# A ContextVar keeps the value isolated per asyncio task / thread.
|
||||
self._current_agent_name_var: contextvars.ContextVar[Optional[str]] = (
|
||||
contextvars.ContextVar("current_agent_name", default=None)
|
||||
)
|
||||
@@ -1003,14 +1002,6 @@ Respond with ONLY a valid JSON tool call in this format:
|
||||
]):
|
||||
return "idle_timeout"
|
||||
|
||||
# Format errors
|
||||
if any(indicator in error_str for indicator in [
|
||||
"validation error", "invalid format", "parse error",
|
||||
"parsing error", "decode error", "malformed",
|
||||
"invalid json", "schema error"
|
||||
]):
|
||||
return "format_error"
|
||||
|
||||
# Default fallback
|
||||
return "unknown"
|
||||
|
||||
@@ -2040,98 +2031,6 @@ Respond with ONLY a valid JSON tool call in this format:
|
||||
schemas.append(f"- {name}({', '.join(param_strs)})")
|
||||
return '\n'.join(schemas) if schemas else "None"
|
||||
|
||||
def _param_accepts_string(self, function_name, param_name, tools) -> bool:
|
||||
"""True when a tool declares this parameter as accepting a string.
|
||||
|
||||
Used to decide whether an argument whose value matches a previous tool's
|
||||
name is a legitimate string or a weak model's way of referring to that
|
||||
tool's result. A declared string parameter is left alone.
|
||||
"""
|
||||
try:
|
||||
for tool in tools or []:
|
||||
fn = tool.get("function") if isinstance(tool, dict) else None
|
||||
if not fn or fn.get("name") != function_name:
|
||||
continue
|
||||
spec = (fn.get("parameters") or {}).get("properties", {}).get(param_name)
|
||||
if not isinstance(spec, dict):
|
||||
return False
|
||||
return self._schema_accepts_string(spec)
|
||||
except Exception: # noqa: BLE001 -- never break a tool call on a schema read
|
||||
return False
|
||||
return False
|
||||
|
||||
def _schema_accepts_string(self, spec) -> bool:
|
||||
"""True when a JSON-Schema fragment can accept a string value.
|
||||
|
||||
Handles the shapes this repo's own generator emits: a bare
|
||||
``{"type": "string"}``, a list type ``{"type": ["string", "null"]}``,
|
||||
and the ``anyOf``/``oneOf``/``allOf`` unions produced for ``Optional[str]``
|
||||
and ``Union`` parameters (``tools/schema.py``). Without the union case an
|
||||
``Optional[str]`` argument was still treated as a result reference and
|
||||
silently overwritten.
|
||||
"""
|
||||
if not isinstance(spec, dict):
|
||||
return False
|
||||
declared = spec.get("type")
|
||||
if isinstance(declared, list):
|
||||
if "string" in declared:
|
||||
return True
|
||||
elif declared == "string":
|
||||
return True
|
||||
# enum without an explicit type is string-typed in JSON Schema practice
|
||||
if declared is None and isinstance(spec.get("enum"), list):
|
||||
if any(isinstance(v, str) for v in spec["enum"]):
|
||||
return True
|
||||
for key in ("anyOf", "oneOf", "allOf"):
|
||||
members = spec.get(key)
|
||||
if isinstance(members, list) and any(
|
||||
self._schema_accepts_string(m) for m in members):
|
||||
return True
|
||||
return False
|
||||
|
||||
def _force_tool_usage_message(self, response_text, tool_calls, formatted_tools,
|
||||
iteration_count):
|
||||
"""The nudge to send when a model ignored tools it should have used.
|
||||
|
||||
Returns the message content, or None when no nudge is warranted. Shared
|
||||
so every response path applies the same policy -- this lived inline in
|
||||
get_response only, so an agent that was awaited instead of called
|
||||
silently lost a setting it had accepted.
|
||||
"""
|
||||
if not self._should_force_tool_usage(
|
||||
response_text, tool_calls, formatted_tools, iteration_count):
|
||||
return None
|
||||
tool_names = self._get_tool_names_for_prompt(formatted_tools)
|
||||
logging.debug(
|
||||
f"[OLLAMA_RELIABILITY] Force tool usage triggered. Adding prompt for tools: {tool_names}")
|
||||
return self.FORCE_TOOL_USAGE_PROMPT.format(tool_names=tool_names)
|
||||
|
||||
def _tool_repair_message(self, tool_calls, formatted_tools):
|
||||
"""The correction to send when a model produced an invalid tool call.
|
||||
|
||||
Returns the message content and charges the repair budget, or None when
|
||||
the calls are valid or the budget is spent. Shared for the same reason
|
||||
as _force_tool_usage_message.
|
||||
"""
|
||||
repair_attempt_count = getattr(self, '_current_repair_count', 0)
|
||||
max_tool_repairs = getattr(self, 'max_tool_repairs', 0)
|
||||
if not (tool_calls and max_tool_repairs > 0
|
||||
and repair_attempt_count < max_tool_repairs):
|
||||
return None
|
||||
validation_errors = [e for e in
|
||||
(self._validate_tool_call(tc, formatted_tools) for tc in tool_calls)
|
||||
if e]
|
||||
if not validation_errors:
|
||||
return None
|
||||
error_msg = "; ".join(validation_errors)
|
||||
tool_schemas = self._get_tool_schemas_for_prompt(formatted_tools)
|
||||
logging.debug(
|
||||
f"[OLLAMA_RELIABILITY] Tool call repair attempt "
|
||||
f"{repair_attempt_count + 1}/{max_tool_repairs}: {error_msg}")
|
||||
self._current_repair_count = repair_attempt_count + 1
|
||||
return self.TOOL_CALL_REPAIR_PROMPT.format(
|
||||
error=error_msg, tool_schemas=tool_schemas)
|
||||
|
||||
def _should_force_tool_usage(self, response_text: str, tool_calls: Optional[List], formatted_tools: Optional[List], iteration_count: int) -> bool:
|
||||
"""
|
||||
Determine if we should force tool usage based on settings and context.
|
||||
@@ -6623,11 +6522,11 @@ Output MUST be JSON with 'reflection' and 'satisfactory'.
|
||||
|
||||
@property
|
||||
def current_agent_name(self) -> Optional[str]:
|
||||
"""The agent name attributed to the current async task/context.
|
||||
"""Task-local name of the agent currently driving this LLM.
|
||||
|
||||
Backed by a per-instance ContextVar (issue #5052) so concurrent agents
|
||||
sharing one LLM instance keep independent, task-local attribution and do
|
||||
not clobber each other's value while awaiting their own completions.
|
||||
Backed by a per-instance ContextVar so concurrent agents sharing one
|
||||
LLM instance each read/write their own value instead of racing on a
|
||||
shared attribute (issues #5052/#3933).
|
||||
"""
|
||||
var = getattr(self, "_current_agent_name_var", None)
|
||||
if var is None:
|
||||
@@ -6673,32 +6572,35 @@ Output MUST be JSON with 'reflection' and 'satisfactory'.
|
||||
"""Deep-copy the LLM while giving the clone fresh attribution ContextVars.
|
||||
|
||||
``contextvars.ContextVar`` has no ``__deepcopy__``/``__reduce__`` and is
|
||||
not copyable, so ``Agent.__deepcopy__`` (which recursively copies
|
||||
``_llm_instance``) would raise ``TypeError: cannot pickle ContextVar``.
|
||||
Thread locks on LLM subclasses get fresh locks (issue #1746 / #5052).
|
||||
not copyable/pickleable, so ``Agent.__deepcopy__`` (which recursively
|
||||
copies ``_llm_instance`` per gateway channel, issue #5052) would raise
|
||||
``TypeError: cannot pickle ContextVar``. Every other attribute is copied
|
||||
normally, brand-new ContextVars start the clone's attribution clean, and
|
||||
thread locks on LLM subclasses (RateLimiter/FailoverManager) get fresh
|
||||
locks so configured LLMs clone cleanly (issues #1746 / #5052 / #3933).
|
||||
"""
|
||||
cls = self.__class__
|
||||
new = cls.__new__(cls)
|
||||
memo[id(self)] = new
|
||||
clone = cls.__new__(cls)
|
||||
memo[id(self)] = clone
|
||||
for key, value in self.__dict__.items():
|
||||
if key == "_current_agent_name_var":
|
||||
new.__dict__[key] = contextvars.ContextVar(
|
||||
clone.__dict__[key] = contextvars.ContextVar(
|
||||
"current_agent_name", default=None
|
||||
)
|
||||
continue
|
||||
if key == "_current_agent_id_var":
|
||||
new.__dict__[key] = contextvars.ContextVar(
|
||||
clone.__dict__[key] = contextvars.ContextVar(
|
||||
"current_agent_id", default=None
|
||||
)
|
||||
continue
|
||||
if type(value) is type(threading.RLock()):
|
||||
new.__dict__[key] = threading.RLock()
|
||||
clone.__dict__[key] = threading.RLock()
|
||||
continue
|
||||
if type(value) is type(threading.Lock()):
|
||||
new.__dict__[key] = threading.Lock()
|
||||
clone.__dict__[key] = threading.Lock()
|
||||
continue
|
||||
new.__dict__[key] = copy.deepcopy(value, memo)
|
||||
return new
|
||||
clone.__dict__[key] = copy.deepcopy(value, memo)
|
||||
return clone
|
||||
|
||||
def _resolve_openai_compatible_model(self) -> str:
|
||||
"""Route a bare model name through the OpenAI-compatible client.
|
||||
@@ -7126,7 +7028,9 @@ Output MUST be JSON with 'reflection' and 'satisfactory'.
|
||||
# (input_text / input_image); local image paths become
|
||||
# data URLs. Plain-string content is passed through.
|
||||
from .openai_client import OpenAIClient
|
||||
item = dict(msg)
|
||||
# Do not forward persisted metadata/provider extensions as
|
||||
# Responses message properties.
|
||||
item = {"role": role}
|
||||
item["content"] = OpenAIClient._build_responses_content(
|
||||
msg.get("content", "")
|
||||
)
|
||||
|
||||
@@ -781,7 +781,9 @@ class OpenAIClient:
|
||||
"output": msg.get("content", ""),
|
||||
})
|
||||
else:
|
||||
item = dict(msg)
|
||||
# Persisted history may carry local metadata/provider fields.
|
||||
# Responses message items accept role/content, not that record.
|
||||
item = {"role": role}
|
||||
item["content"] = self._build_responses_content(
|
||||
msg.get("content", "")
|
||||
)
|
||||
|
||||
@@ -210,7 +210,27 @@ class MCPToolRunner(threading.Thread):
|
||||
return f"Error: MCP tool call timed out after {self.timeout} seconds"
|
||||
if not success:
|
||||
return f"Error: {result}"
|
||||
|
||||
|
||||
# A tool that fails at the application level returns a normal
|
||||
# CallToolResult with isError=True per the MCP spec. Surface it as a
|
||||
# structured error so retry/circuit-breaker/hook logic treats it as a
|
||||
# failure instead of a successful string result (issue #5319).
|
||||
if getattr(result, 'isError', False):
|
||||
if hasattr(result, 'content') and result.content:
|
||||
if hasattr(result.content[0], 'text'):
|
||||
error_text = result.content[0].text
|
||||
else:
|
||||
error_text = str(result.content[0])
|
||||
else:
|
||||
error_text = str(result)
|
||||
# Guarantee a non-empty error string: an isError result with an
|
||||
# empty text block would otherwise yield {"error": ""}, which
|
||||
# retry/circuit-breaker/hook logic can misread as a success
|
||||
# (issue #5319).
|
||||
if not error_text:
|
||||
error_text = f"MCP tool '{tool_name}' reported an error (isError=True)"
|
||||
return {"error": error_text, "mcp_error": True}
|
||||
|
||||
# Process result
|
||||
if hasattr(result, 'content') and result.content:
|
||||
if hasattr(result.content[0], 'text'):
|
||||
@@ -401,8 +421,8 @@ class MCP:
|
||||
base = re.sub(r'\.(cmd|exe|bat|ps1)$', '', base, flags=re.IGNORECASE).lower()
|
||||
return base in cls._COLD_START_LAUNCHERS
|
||||
|
||||
def __init__(self, command_or_string=None, args=None, *, command=None, timeout=None, debug=False,
|
||||
allowed_tools: Optional[List[str]] = None, disabled_tools: Optional[List[str]] = None, **kwargs):
|
||||
def __init__(self, command_or_string=None, args=None, *, command=None, timeout=None, debug=False,
|
||||
allowed_tools: Optional[List[str]] = None, disabled_tools: Optional[List[str]] = None, max_response_bytes: Optional[int] = None, **kwargs):
|
||||
"""
|
||||
Initialize the MCP connection and get tools.
|
||||
|
||||
@@ -422,6 +442,7 @@ class MCP:
|
||||
debug: Enable debug logging for MCP operations (default: False)
|
||||
allowed_tools: Include whitelist - only these tools will be available (default: None = all tools)
|
||||
disabled_tools: Exclude blacklist - these tools will be filtered out (default: None = no exclusions)
|
||||
max_response_bytes: Maximum body size for each Streamable HTTP response.
|
||||
**kwargs: Additional parameters for StdioServerParameters
|
||||
|
||||
Note:
|
||||
@@ -439,6 +460,12 @@ class MCP:
|
||||
if command_or_string is None and command is not None:
|
||||
command_or_string = command
|
||||
|
||||
if max_response_bytes is not None:
|
||||
if isinstance(max_response_bytes, bool) or not isinstance(max_response_bytes, int) or max_response_bytes <= 0:
|
||||
raise ValueError("max_response_bytes must be a positive integer")
|
||||
if not isinstance(command_or_string, str) or not re.match(r'^https?://', command_or_string):
|
||||
raise ValueError("max_response_bytes is only supported for Streamable HTTP MCP servers")
|
||||
|
||||
# Set up logging - default to WARNING level to hide INFO messages
|
||||
if debug:
|
||||
get_logger("mcp-wrapper").setLevel(logging.DEBUG)
|
||||
@@ -510,6 +537,8 @@ class MCP:
|
||||
# single source of truth (see mcp_transport.get_transport_type).
|
||||
from .mcp_transport import get_transport_type
|
||||
if get_transport_type(command_or_string) == "sse" and 'transport_type' not in kwargs:
|
||||
if max_response_bytes is not None:
|
||||
raise ValueError("max_response_bytes is only supported for Streamable HTTP MCP servers")
|
||||
# Legacy SSE URL - use SSE transport for backward compatibility
|
||||
from .mcp_sse import SSEMCPClient
|
||||
self.sse_client = SSEMCPClient(command_or_string, debug=debug, timeout=timeout)
|
||||
@@ -533,6 +562,8 @@ class MCP:
|
||||
transport_options['session'] = kwargs.pop('session')
|
||||
if 'resumability' in kwargs:
|
||||
transport_options['resumability'] = kwargs.pop('resumability')
|
||||
if max_response_bytes is not None:
|
||||
transport_options['max_response_bytes'] = max_response_bytes
|
||||
|
||||
self.http_stream_client = HTTPStreamMCPClient(
|
||||
command_or_string,
|
||||
|
||||
@@ -71,6 +71,61 @@ def _register_cleanup():
|
||||
atexit.register(_cleanup_all_clients)
|
||||
_cleanup_registered = True
|
||||
|
||||
def _bounded_httpx_client_factory(max_response_bytes):
|
||||
import httpx
|
||||
|
||||
class BoundedStream(httpx.AsyncByteStream):
|
||||
def __init__(self, stream):
|
||||
self.stream = stream
|
||||
|
||||
async def __aiter__(self):
|
||||
total_bytes = 0
|
||||
async for chunk in self.stream:
|
||||
total_bytes += len(chunk)
|
||||
if total_bytes > max_response_bytes:
|
||||
raise ValueError(f"MCP response exceeded maximum of {max_response_bytes} bytes")
|
||||
yield chunk
|
||||
|
||||
async def aclose(self):
|
||||
await self.stream.aclose()
|
||||
|
||||
class BoundedClient(httpx.AsyncClient):
|
||||
async def send(self, request, *, stream=False, **kwargs):
|
||||
response = await super().send(request, stream=True, **kwargs)
|
||||
encoding = response.headers.get("content-encoding", "identity").lower()
|
||||
if encoding not in ("", "identity"):
|
||||
await response.aclose()
|
||||
raise ValueError("Compressed MCP responses are not supported with a response byte limit")
|
||||
content_length = response.headers.get("content-length")
|
||||
try:
|
||||
content_length = int(content_length) if content_length is not None else None
|
||||
except ValueError:
|
||||
content_length = None
|
||||
if content_length is not None and content_length > max_response_bytes:
|
||||
await response.aclose()
|
||||
raise ValueError(f"MCP response exceeded maximum of {max_response_bytes} bytes")
|
||||
response.stream = BoundedStream(response.stream)
|
||||
if not stream:
|
||||
try:
|
||||
await response.aread()
|
||||
except BaseException:
|
||||
await response.aclose()
|
||||
raise
|
||||
await response.aclose()
|
||||
return response
|
||||
|
||||
def create_client(headers=None, timeout=None, auth=None):
|
||||
request_headers = dict(headers or {})
|
||||
request_headers["Accept-Encoding"] = "identity"
|
||||
return BoundedClient(
|
||||
headers=request_headers,
|
||||
timeout=timeout,
|
||||
auth=auth,
|
||||
trust_env=True,
|
||||
)
|
||||
|
||||
return create_client
|
||||
|
||||
class HTTPStreamMCPTool:
|
||||
"""A wrapper for an MCP tool that can be used with praisonaiagents."""
|
||||
|
||||
@@ -137,6 +192,8 @@ class HTTPStreamMCPTool:
|
||||
logger.debug(f"Async calling tool {self.name} with args: {kwargs}")
|
||||
try:
|
||||
result = await self.session.call_tool(self.name, kwargs)
|
||||
if getattr(result, 'isError', False):
|
||||
raise RuntimeError(f"MCP tool {self.name} failed: {result.content}")
|
||||
|
||||
# Extract text from result
|
||||
if hasattr(result, 'content') and result.content:
|
||||
@@ -490,6 +547,13 @@ class HTTPStreamMCPClient:
|
||||
self.debug = debug
|
||||
self.timeout = timeout
|
||||
self.options = options or {}
|
||||
self.max_response_bytes = self.options.get('max_response_bytes')
|
||||
if self.max_response_bytes is not None and (
|
||||
isinstance(self.max_response_bytes, bool)
|
||||
or not isinstance(self.max_response_bytes, int)
|
||||
or self.max_response_bytes <= 0
|
||||
):
|
||||
raise ValueError("max_response_bytes must be a positive integer")
|
||||
self.session = None
|
||||
self.tools = []
|
||||
self.transport = None
|
||||
@@ -510,17 +574,16 @@ class HTTPStreamMCPClient:
|
||||
|
||||
def _initialize(self):
|
||||
"""Initialize the connection and tools."""
|
||||
# Use the global event loop
|
||||
loop = get_event_loop()
|
||||
|
||||
# Start a background thread to run the event loop
|
||||
def run_event_loop():
|
||||
asyncio.set_event_loop(loop)
|
||||
loop.run_forever()
|
||||
|
||||
self.loop_thread = threading.Thread(target=run_event_loop, daemon=True)
|
||||
self.loop_thread.start()
|
||||
|
||||
self.loop_thread = None
|
||||
if not loop.is_running():
|
||||
def run_event_loop():
|
||||
asyncio.set_event_loop(loop)
|
||||
loop.run_forever()
|
||||
|
||||
self.loop_thread = threading.Thread(target=run_event_loop, daemon=True)
|
||||
self.loop_thread.start()
|
||||
|
||||
# Run the initialization in the event loop
|
||||
future = asyncio.run_coroutine_threadsafe(self._async_initialize(), loop)
|
||||
self.tools = future.result(timeout=self.timeout)
|
||||
@@ -542,11 +605,16 @@ class HTTPStreamMCPClient:
|
||||
|
||||
headers = self.options.get('headers') or None
|
||||
|
||||
self._streams_context = streamablehttp_client(
|
||||
url=self.base_url,
|
||||
headers=headers,
|
||||
timeout=self.timeout,
|
||||
)
|
||||
client_options = {
|
||||
"url": self.base_url,
|
||||
"headers": headers,
|
||||
"timeout": self.timeout,
|
||||
}
|
||||
if self.max_response_bytes is not None:
|
||||
client_options["httpx_client_factory"] = _bounded_httpx_client_factory(
|
||||
self.max_response_bytes
|
||||
)
|
||||
self._streams_context = streamablehttp_client(**client_options)
|
||||
|
||||
# If any step of the handshake fails after a context has been entered,
|
||||
# unwind the already-entered contexts so we don't leak the httpx/anyio
|
||||
@@ -689,4 +757,4 @@ class HTTPStreamMCPClient:
|
||||
self._force_cleanup()
|
||||
except Exception:
|
||||
# Never raise exceptions in __del__
|
||||
pass
|
||||
pass
|
||||
@@ -73,7 +73,7 @@ class SqliteMemoryAdapter:
|
||||
|
||||
def _get_stm_conn(self):
|
||||
"""Get thread-local short-term memory connection."""
|
||||
if not hasattr(self._local, 'stm_conn'):
|
||||
if getattr(self._local, 'stm_conn', None) is None:
|
||||
self._local.stm_conn = sqlite3.connect(
|
||||
self.short_db,
|
||||
check_same_thread=False,
|
||||
@@ -97,7 +97,7 @@ class SqliteMemoryAdapter:
|
||||
|
||||
def _get_ltm_conn(self):
|
||||
"""Get thread-local long-term memory connection."""
|
||||
if not hasattr(self._local, 'ltm_conn'):
|
||||
if getattr(self._local, 'ltm_conn', None) is None:
|
||||
self._local.ltm_conn = sqlite3.connect(
|
||||
self.long_db,
|
||||
check_same_thread=False,
|
||||
@@ -272,4 +272,30 @@ class SqliteMemoryAdapter:
|
||||
conn.close()
|
||||
except Exception:
|
||||
pass
|
||||
self._all_connections.clear()
|
||||
self._all_connections.clear()
|
||||
|
||||
def close_thread_connections(self):
|
||||
"""Close only the calling thread's connections.
|
||||
|
||||
Leaves the shared adapter and other threads' connections intact so the
|
||||
adapter can be reused; the calling thread lazily reopens its own
|
||||
connections on next access.
|
||||
|
||||
A connection is only removed from the registry and cleared from
|
||||
thread-local storage after it closes successfully. If ``close()`` raises,
|
||||
the connection stays tracked so a later ``close_connections()`` /
|
||||
``close_thread_connections()`` can retry it, and the failure is logged
|
||||
rather than silently swallowed.
|
||||
"""
|
||||
for attr in ("stm_conn", "ltm_conn"):
|
||||
conn = getattr(self._local, attr, None)
|
||||
if conn is None:
|
||||
continue
|
||||
try:
|
||||
conn.close()
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to close {attr}; keeping it tracked for retry: {e}")
|
||||
continue
|
||||
with self._connection_lock:
|
||||
self._all_connections.discard(conn)
|
||||
setattr(self._local, attr, None)
|
||||
@@ -266,11 +266,12 @@ class FileMemory:
|
||||
return True
|
||||
except (IOError, OSError) as e:
|
||||
self._log(f"Error writing {filepath}: {e}", logging.ERROR)
|
||||
return False
|
||||
finally:
|
||||
try:
|
||||
tmp_path.unlink()
|
||||
except OSError:
|
||||
pass
|
||||
return False
|
||||
|
||||
def commit_memory_batch(
|
||||
self,
|
||||
|
||||
@@ -2532,16 +2532,31 @@ class Memory(SearchMixin, MemoryCoreMixin):
|
||||
# Other active threads should manage their own connections
|
||||
# Only clear full registry in __del__ or explicit shutdown
|
||||
|
||||
# Close memory adapter if it exists (protocol-driven path)
|
||||
# Close memory adapter if it exists (protocol-driven path).
|
||||
#
|
||||
# SQLite is thread-scoped by contract (#1562/#1563): close only the
|
||||
# calling thread's adapter connections and KEEP the shared adapter so
|
||||
# other threads and subsequent same-thread searches keep using the
|
||||
# adapter's tables. Tearing it down here would make searches fall back
|
||||
# to legacy SQL against short_mem/long_mem (#5486).
|
||||
if hasattr(self, 'memory_adapter') and self.memory_adapter:
|
||||
try:
|
||||
if hasattr(self.memory_adapter, 'close') and callable(self.memory_adapter.close):
|
||||
self.memory_adapter.close()
|
||||
logger.debug("Memory adapter closed successfully")
|
||||
except Exception as e:
|
||||
logger.warning(f"Error closing memory adapter: {e}")
|
||||
finally:
|
||||
self.memory_adapter = None
|
||||
if isinstance(self.memory_adapter, SqliteMemoryAdapter):
|
||||
try:
|
||||
if hasattr(self.memory_adapter, 'close_thread_connections'):
|
||||
self.memory_adapter.close_thread_connections()
|
||||
logger.debug("SQLite adapter thread connections closed")
|
||||
except Exception as e:
|
||||
logger.warning(f"Error closing SQLite adapter thread connections: {e}")
|
||||
# Intentionally preserve self.memory_adapter for reuse.
|
||||
else:
|
||||
try:
|
||||
if hasattr(self.memory_adapter, 'close') and callable(self.memory_adapter.close):
|
||||
self.memory_adapter.close()
|
||||
logger.debug("Memory adapter closed successfully")
|
||||
except Exception as e:
|
||||
logger.warning(f"Error closing memory adapter: {e}")
|
||||
finally:
|
||||
self.memory_adapter = None
|
||||
|
||||
# Close MongoDB client if it exists (legacy path)
|
||||
if hasattr(self, 'mongo_client') and self.mongo_client:
|
||||
|
||||
@@ -1306,6 +1306,51 @@ def _adapt_plugin_hooks(plugin: Plugin) -> Iterator[Tuple["HookEvent", Callable]
|
||||
return HookResult.allow()
|
||||
yield HookEvent.CLI_BACKEND_EXECUTE, cli_backend_execute_hook
|
||||
|
||||
# Generic, declaration-driven tail. The explicit branches above give
|
||||
# rewrite/deny semantics to the conversation/tool/channel events that need
|
||||
# them. Everything else a plugin declares in ``PluginInfo.hooks`` — the
|
||||
# gateway/schedule/kanban/compaction/subagent/model-fallback lifecycle — is
|
||||
# observe-only, so a single loop bridges it by method name (``event.value``)
|
||||
# without a hand-written branch per event. This keeps the coverage matrix
|
||||
# from drifting: any future emitted event becomes first-class the moment a
|
||||
# plugin declares it and defines the matching method.
|
||||
yielded = {
|
||||
HookEvent.BEFORE_AGENT, HookEvent.AFTER_AGENT,
|
||||
HookEvent.BEFORE_LLM, HookEvent.AFTER_LLM,
|
||||
HookEvent.BEFORE_TOOL, HookEvent.AFTER_TOOL,
|
||||
HookEvent.BEFORE_TOOL_DEFINITIONS,
|
||||
HookEvent.MESSAGE_RECEIVED, HookEvent.MESSAGE_SENDING,
|
||||
HookEvent.MESSAGE_SENT, HookEvent.MESSAGE_UNDELIVERED,
|
||||
HookEvent.ON_PERMISSION_ASK, HookEvent.ON_CONFIG, HookEvent.ON_AUTH,
|
||||
HookEvent.SESSION_START, HookEvent.SESSION_END, HookEvent.ON_ERROR,
|
||||
HookEvent.CLI_BACKEND_EXECUTE,
|
||||
}
|
||||
|
||||
def _make_observe_hook(method):
|
||||
def observe_hook(data, _m=method):
|
||||
payload = data.to_dict() if hasattr(data, "to_dict") else {}
|
||||
_m(payload)
|
||||
return HookResult.allow()
|
||||
return observe_hook
|
||||
|
||||
declared = getattr(plugin.info, "hooks", None) or []
|
||||
seen: set = set()
|
||||
for event in declared:
|
||||
if not isinstance(event, HookEvent) or event in yielded or event in seen:
|
||||
continue
|
||||
seen.add(event)
|
||||
method = getattr(plugin, event.value, None)
|
||||
if not callable(method):
|
||||
continue
|
||||
parent = getattr(base, event.value, None)
|
||||
own = getattr(type(plugin), event.value, None)
|
||||
# A base no-op that the plugin did not override yields nothing (avoids
|
||||
# registering dead hooks). Methods with no base counterpart (e.g.
|
||||
# ``kanban_task_created``) are treated as genuine overrides.
|
||||
if parent is not None and own is parent:
|
||||
continue
|
||||
yield event, _make_observe_hook(method)
|
||||
|
||||
|
||||
# Global plugin manager instance
|
||||
_default_manager: Optional[PluginManager] = None
|
||||
|
||||
@@ -248,6 +248,42 @@ class Plugin(ABC):
|
||||
def cli_backend_execute(self, context: Dict[str, Any]) -> None:
|
||||
"""Called after a CLI backend delegates a turn. Observe-only."""
|
||||
pass
|
||||
|
||||
def gateway_start(self, context: Dict[str, Any]) -> None:
|
||||
"""Called when the gateway/BotOS starts. Observe-only (platforms, bot_count)."""
|
||||
pass
|
||||
|
||||
def gateway_stop(self, context: Dict[str, Any]) -> None:
|
||||
"""Called when the gateway/BotOS stops. Observe-only (platforms, reason)."""
|
||||
pass
|
||||
|
||||
def schedule_add(self, context: Dict[str, Any]) -> None:
|
||||
"""Called when a scheduled job is persisted. Observe-only."""
|
||||
pass
|
||||
|
||||
def schedule_remove(self, context: Dict[str, Any]) -> None:
|
||||
"""Called when a scheduled job is deleted. Observe-only."""
|
||||
pass
|
||||
|
||||
def schedule_trigger(self, context: Dict[str, Any]) -> None:
|
||||
"""Called when a scheduled job fires. Observe-only."""
|
||||
pass
|
||||
|
||||
def before_compaction(self, context: Dict[str, Any]) -> None:
|
||||
"""Called before conversation memory is compacted. Observe-only."""
|
||||
pass
|
||||
|
||||
def after_compaction(self, context: Dict[str, Any]) -> None:
|
||||
"""Called after conversation memory is compacted. Observe-only."""
|
||||
pass
|
||||
|
||||
def subagent_stop(self, context: Dict[str, Any]) -> None:
|
||||
"""Called when a spawned subagent reaches a terminal state. Observe-only."""
|
||||
pass
|
||||
|
||||
def model_fallback(self, context: Dict[str, Any]) -> None:
|
||||
"""Called when the runtime falls back to a secondary model. Observe-only."""
|
||||
pass
|
||||
|
||||
def on_permission_ask(self, target: str, reason: str) -> Optional[bool]:
|
||||
"""Called when permission is requested. Return True/False to auto-approve/deny."""
|
||||
|
||||
@@ -844,6 +844,13 @@ Subtask: {st.name}
|
||||
break
|
||||
else:
|
||||
logging.debug(f"No subtasks created yet for {current_task.name}")
|
||||
# Mirror the sync workflow(): default a mid-workflow loop
|
||||
# task (reached via next_tasks/condition, not the start task)
|
||||
# with no explicit input_file to the tasks.csv convention so
|
||||
# its subtasks are still expanded instead of being silently
|
||||
# skipped (issue #5319).
|
||||
if current_task is not start_task and not current_task.input_file:
|
||||
current_task.input_file = "tasks.csv"
|
||||
# Create subtasks if needed
|
||||
if current_task.input_file:
|
||||
self._create_loop_subtasks(current_task)
|
||||
|
||||
@@ -7,6 +7,14 @@ re-exports them lazily so user code can keep importing from
|
||||
import importlib
|
||||
|
||||
from .models import ChannelMessage
|
||||
from .policy import (
|
||||
DEFAULT_PREFERENCE,
|
||||
DetailLevel,
|
||||
NotificationCategory,
|
||||
NotificationDecision,
|
||||
NotificationPreference,
|
||||
evaluate,
|
||||
)
|
||||
from .protocols import PushTransportProtocol
|
||||
|
||||
_lazy_cache: dict = {}
|
||||
@@ -33,6 +41,12 @@ def __getattr__(name: str):
|
||||
__all__ = [
|
||||
"ChannelMessage",
|
||||
"PushTransportProtocol",
|
||||
"NotificationCategory",
|
||||
"DetailLevel",
|
||||
"NotificationPreference",
|
||||
"NotificationDecision",
|
||||
"evaluate",
|
||||
"DEFAULT_PREFERENCE",
|
||||
"PushClient",
|
||||
"WebSocketTransport",
|
||||
"PollingTransport",
|
||||
|
||||
@@ -0,0 +1,212 @@
|
||||
"""Notification policy for gateway push — pure, dependency-free core.
|
||||
|
||||
The push stack elsewhere is transport + delivery-guarantee only: it can get
|
||||
bytes to a device reliably, but it has no notion of *whether* or *what* to
|
||||
notify. This module supplies that missing policy layer as pure protocol code so
|
||||
every notifier (the gateway's delivery manager, scheduler completions,
|
||||
background-job alerts) can make the same decision identically:
|
||||
|
||||
decision = evaluate(category, prefs, now=..., is_urgent=..., content=...)
|
||||
if decision.send:
|
||||
deliver(recipient, decision.redacted_content)
|
||||
|
||||
Three concerns compose into one :class:`NotificationDecision`:
|
||||
|
||||
* **Categories** — a recipient opts in per event class (``approval_requested``,
|
||||
``agent_finished`` …); an un-opted category is suppressed.
|
||||
* **Quiet hours** — a per-recipient local window suppresses non-urgent
|
||||
categories; urgent categories may override (``urgent_overrides_quiet``).
|
||||
* **Detail level** — ``private`` / ``identified`` / ``detailed`` redacts how
|
||||
much message content leaves the gateway in a device notification.
|
||||
|
||||
The heavy enforcement + durable preference store + CLI/YAML surface live in the
|
||||
``praisonai``/``praisonai-bot`` wrapper; this stays pure so it carries no heavy
|
||||
imports and is trivially testable.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime, time
|
||||
from enum import Enum
|
||||
from typing import Optional, Set
|
||||
|
||||
__all__ = [
|
||||
"NotificationCategory",
|
||||
"DetailLevel",
|
||||
"NotificationPreference",
|
||||
"NotificationDecision",
|
||||
"evaluate",
|
||||
"DEFAULT_PREFERENCE",
|
||||
]
|
||||
|
||||
|
||||
class NotificationCategory(str, Enum):
|
||||
"""Event classes a recipient can subscribe to independently."""
|
||||
|
||||
APPROVAL_REQUESTED = "approval_requested"
|
||||
AGENT_FINISHED = "agent_finished"
|
||||
AGENT_QUESTION = "agent_question"
|
||||
HUMAN_MENTIONED = "human_mentioned"
|
||||
SCHEDULED_TASK_FAILED = "scheduled_task_failed"
|
||||
BACKGROUND_TASK_FAILED = "background_task_failed"
|
||||
|
||||
|
||||
class DetailLevel(str, Enum):
|
||||
"""How much content leaves the gateway in a notification.
|
||||
|
||||
* ``PRIVATE`` — a bare "you have activity"; no agent/session, no content.
|
||||
* ``IDENTIFIED`` — which agent/session, but no message content.
|
||||
* ``DETAILED`` — content included verbatim.
|
||||
"""
|
||||
|
||||
PRIVATE = "private"
|
||||
IDENTIFIED = "identified"
|
||||
DETAILED = "detailed"
|
||||
|
||||
|
||||
def _in_quiet_window(now: time, start: time, end: time) -> bool:
|
||||
"""Whether ``now`` falls inside the ``[start, end)`` local window.
|
||||
|
||||
Handles a window that wraps past midnight (e.g. ``22:00``–``07:30``) by
|
||||
treating start > end as "outside the daytime gap".
|
||||
"""
|
||||
if start == end:
|
||||
return False
|
||||
if start < end:
|
||||
return start <= now < end
|
||||
return now >= start or now < end
|
||||
|
||||
|
||||
@dataclass
|
||||
class NotificationPreference:
|
||||
"""A recipient's durable notification policy.
|
||||
|
||||
Attributes:
|
||||
categories: Opted-in categories; a category not present is suppressed.
|
||||
quiet_hours: Optional ``(start, end)`` local time window during which
|
||||
non-urgent categories are suppressed. ``None`` means no quiet hours.
|
||||
timezone: IANA name used to localise ``now`` for the quiet-hours test.
|
||||
detail: Redaction level applied to the notification content.
|
||||
urgent_overrides_quiet: When True, an urgent event is delivered even
|
||||
inside quiet hours.
|
||||
"""
|
||||
|
||||
categories: Set[NotificationCategory] = field(default_factory=set)
|
||||
quiet_hours: Optional[tuple] = None
|
||||
timezone: str = "UTC"
|
||||
detail: DetailLevel = DetailLevel.IDENTIFIED
|
||||
urgent_overrides_quiet: bool = True
|
||||
|
||||
def allows(self, category: NotificationCategory) -> bool:
|
||||
"""Whether this recipient has opted into ``category``."""
|
||||
return category in self.categories
|
||||
|
||||
|
||||
# A sensible default applied when a recipient has set nothing: the two
|
||||
# interaction-blocking categories are on, at the identified (no-content) detail
|
||||
# level, with no quiet hours.
|
||||
DEFAULT_PREFERENCE = NotificationPreference(
|
||||
categories={
|
||||
NotificationCategory.APPROVAL_REQUESTED,
|
||||
NotificationCategory.AGENT_QUESTION,
|
||||
},
|
||||
detail=DetailLevel.IDENTIFIED,
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class NotificationDecision:
|
||||
"""Outcome of :func:`evaluate` for one (category, recipient, moment).
|
||||
|
||||
Attributes:
|
||||
send: Whether the notification should be delivered at all.
|
||||
reason: Machine/operator-readable reason (e.g. ``"quiet_hours"``,
|
||||
``"category_not_subscribed"``, ``"ok"``).
|
||||
redacted_content: The content to actually push, redacted per the
|
||||
recipient's detail level (``None`` when ``send`` is False).
|
||||
"""
|
||||
|
||||
send: bool
|
||||
reason: str
|
||||
redacted_content: Optional[str] = None
|
||||
|
||||
|
||||
def _redact(detail: DetailLevel, content: Optional[str], identity: Optional[str]) -> str:
|
||||
"""Apply the detail-level redaction to ``content``.
|
||||
|
||||
``identity`` is an agent/session label used at the ``identified`` level so a
|
||||
recipient learns *what* has activity without the transcript leaving the
|
||||
gateway.
|
||||
"""
|
||||
if detail == DetailLevel.DETAILED:
|
||||
return content if content is not None else "You have new activity"
|
||||
if detail == DetailLevel.IDENTIFIED:
|
||||
return f"Activity in {identity}" if identity else "You have new activity"
|
||||
return "You have new activity"
|
||||
|
||||
|
||||
def evaluate(
|
||||
category: NotificationCategory,
|
||||
prefs: Optional[NotificationPreference] = None,
|
||||
*,
|
||||
now: Optional[datetime] = None,
|
||||
is_urgent: bool = False,
|
||||
content: Optional[str] = None,
|
||||
identity: Optional[str] = None,
|
||||
) -> NotificationDecision:
|
||||
"""Decide whether/what to notify for one event — pure, no side effects.
|
||||
|
||||
Args:
|
||||
category: The event class being considered.
|
||||
prefs: The recipient's preference; ``DEFAULT_PREFERENCE`` when ``None``.
|
||||
now: The moment of evaluation (defaults to ``datetime.now`` localised to
|
||||
the recipient's timezone). A naive ``now`` is treated as already in
|
||||
the recipient's local zone.
|
||||
is_urgent: Whether this specific event is urgent (may override quiet
|
||||
hours when the recipient allows it).
|
||||
content: The full message content, redacted per ``prefs.detail``.
|
||||
identity: An agent/session label used at the ``identified`` detail level.
|
||||
|
||||
Returns:
|
||||
A :class:`NotificationDecision`. ``send`` is False (with a ``reason``)
|
||||
when the category is not subscribed or quiet hours suppress it.
|
||||
"""
|
||||
if prefs is None:
|
||||
prefs = DEFAULT_PREFERENCE
|
||||
|
||||
if not prefs.allows(category):
|
||||
return NotificationDecision(send=False, reason="category_not_subscribed")
|
||||
|
||||
if prefs.quiet_hours is not None:
|
||||
local_now = _localised_time(now, prefs.timezone)
|
||||
start, end = prefs.quiet_hours
|
||||
if _in_quiet_window(local_now, start, end):
|
||||
if not (is_urgent and prefs.urgent_overrides_quiet):
|
||||
return NotificationDecision(send=False, reason="quiet_hours")
|
||||
|
||||
return NotificationDecision(
|
||||
send=True,
|
||||
reason="urgent_override" if is_urgent and prefs.quiet_hours else "ok",
|
||||
redacted_content=_redact(prefs.detail, content, identity),
|
||||
)
|
||||
|
||||
|
||||
def _localised_time(now: Optional[datetime], tz_name: str) -> time:
|
||||
"""Return the wall-clock ``time`` at ``now`` in ``tz_name``.
|
||||
|
||||
A ``None`` ``now`` uses the current time. An aware ``now`` is converted to
|
||||
``tz_name``; a naive ``now`` is assumed to already be local. ``zoneinfo`` is
|
||||
stdlib (3.9+) so this stays dependency-free; an unknown zone falls back to
|
||||
the naive time rather than raising.
|
||||
"""
|
||||
if now is None:
|
||||
now = datetime.now()
|
||||
if now.tzinfo is not None:
|
||||
try:
|
||||
from zoneinfo import ZoneInfo
|
||||
|
||||
now = now.astimezone(ZoneInfo(tz_name))
|
||||
except Exception:
|
||||
pass
|
||||
return now.timetz().replace(tzinfo=None) if now.tzinfo else now.time()
|
||||
@@ -25,6 +25,7 @@ if TYPE_CHECKING:
|
||||
from .store import FileScheduleStore
|
||||
from .config_store import ConfigYamlScheduleStore
|
||||
from .parser import parse_schedule
|
||||
from .due import quota_hold_from_failure
|
||||
from .runner import ScheduleRunner
|
||||
from .loop import ScheduleLoop, InProcessScheduleProvider
|
||||
from .protocols import (
|
||||
@@ -106,6 +107,11 @@ def __getattr__(name: str):
|
||||
_module_cache[name] = parse_schedule
|
||||
return parse_schedule
|
||||
|
||||
if name == "quota_hold_from_failure":
|
||||
from .due import quota_hold_from_failure
|
||||
_module_cache[name] = quota_hold_from_failure
|
||||
return quota_hold_from_failure
|
||||
|
||||
if name == "ScheduleRunner":
|
||||
from .runner import ScheduleRunner
|
||||
_module_cache[name] = ScheduleRunner
|
||||
@@ -176,6 +182,7 @@ __all__ = [
|
||||
"FileScheduleStore",
|
||||
"ConfigYamlScheduleStore",
|
||||
"parse_schedule",
|
||||
"quota_hold_from_failure",
|
||||
"ScheduleRunner",
|
||||
"ScheduleLoop",
|
||||
"InProcessScheduleProvider",
|
||||
|
||||
@@ -15,7 +15,7 @@ import time
|
||||
from typing import Dict, Iterator, List, Optional
|
||||
|
||||
from .models import ScheduleJob, RunRecord
|
||||
from .due import is_due as _is_due, resolve_schedule_timezone
|
||||
from .due import is_due as _is_due, is_misfire as _is_misfire, resolve_schedule_timezone
|
||||
from .hook_emit import emit_schedule_add, emit_schedule_remove
|
||||
|
||||
logger = get_logger(__name__)
|
||||
@@ -202,6 +202,10 @@ class ConfigYamlScheduleStore:
|
||||
# a later ``remove(job.id)`` by the runner is then a no-op, so a spent
|
||||
# one-shot yields exactly one SCHEDULE_REMOVE.
|
||||
auto_removed: List[ScheduleJob] = []
|
||||
# (job_id, job_name) for occurrences suppressed by the misfire policy.
|
||||
# Logged as ``missed`` history after the lock so a dropped occurrence is
|
||||
# observable instead of vanishing silently.
|
||||
missed: List[tuple] = []
|
||||
with self._lock, self._file_lock():
|
||||
# Re-read from disk so we observe cross-process claims/leases.
|
||||
self._reload_locked()
|
||||
@@ -224,6 +228,16 @@ class ConfigYamlScheduleStore:
|
||||
self._job_state.pop(job.id, None)
|
||||
changed = True
|
||||
continue
|
||||
# Misfire policy: an occurrence that landed while the process
|
||||
# was down and is now older than ``misfire_grace_seconds`` is
|
||||
# recorded as ``missed`` (observable) rather than fired stale or
|
||||
# dropped silently. Advance past the slot so it is not seen due
|
||||
# again on the next poll.
|
||||
if _is_misfire(job, now, self._default_timezone):
|
||||
job.last_run_at = now
|
||||
missed.append((job.id, job.name))
|
||||
changed = True
|
||||
continue
|
||||
# Win the claim: pre-advance + lease atomically.
|
||||
job.last_run_at = now
|
||||
# Count the fire here (needed so ``should_retire()`` can drop a
|
||||
@@ -254,6 +268,14 @@ class ConfigYamlScheduleStore:
|
||||
return []
|
||||
for job in auto_removed:
|
||||
emit_schedule_remove(job)
|
||||
# Record suppressed occurrences so a missed run is visible in history
|
||||
# rather than silently dropped (log_run takes only the thread lock).
|
||||
for job_id, job_name in missed:
|
||||
self.log_run(
|
||||
job_id=job_id,
|
||||
status="missed",
|
||||
job_name=job_name,
|
||||
)
|
||||
return claimed
|
||||
|
||||
def complete(self, job_id: str, owner_id: str) -> None:
|
||||
|
||||
@@ -76,6 +76,158 @@ def next_fire_time(
|
||||
return croniter(cron_expr, base_datetime).get_next(datetime).timestamp()
|
||||
|
||||
|
||||
def scheduled_instant(
|
||||
job: Any,
|
||||
now: float,
|
||||
default_timezone: str | None = None,
|
||||
) -> float | None:
|
||||
"""Return the epoch of the occurrence this due job is firing *for*.
|
||||
|
||||
When a recurring job becomes due at ``now`` it is firing for a specific
|
||||
scheduled slot that may lie in the past (the process was down when the
|
||||
slot arrived). This returns that slot's canonical instant so a misfire
|
||||
policy can decide whether recovery is still within an acceptable grace
|
||||
window. ``None`` is returned when there is no meaningful past slot (a
|
||||
never-run interval job, or a kind/expression that cannot be evaluated).
|
||||
|
||||
- ``every``: the *first* slot that came due after ``last_run_at``
|
||||
(``last_run_at + every_seconds``). Its age measures how long the job has
|
||||
been overdue — the meaningful lateness for a coalesced recovery.
|
||||
- ``cron``: the last cron fire at or before ``now`` (the slot being
|
||||
recovered for a daily/weekly schedule).
|
||||
- ``at``: the target instant itself.
|
||||
"""
|
||||
sched = job.schedule
|
||||
if sched.kind == "every":
|
||||
if sched.every_seconds is None or job.last_run_at is None:
|
||||
return None
|
||||
if now - job.last_run_at < sched.every_seconds:
|
||||
return None
|
||||
return job.last_run_at + sched.every_seconds
|
||||
if sched.kind == "at":
|
||||
if sched.at is None:
|
||||
return None
|
||||
try:
|
||||
return localize_wall_clock(
|
||||
datetime.fromisoformat(sched.at),
|
||||
sched.tz or default_timezone,
|
||||
).timestamp()
|
||||
except (ValueError, TypeError):
|
||||
return None
|
||||
if sched.kind == "cron":
|
||||
if sched.cron_expr is None:
|
||||
return None
|
||||
try:
|
||||
from croniter import croniter # type: ignore[import-untyped]
|
||||
except ImportError:
|
||||
return None
|
||||
try:
|
||||
base_datetime = datetime.fromtimestamp(
|
||||
now, resolve_schedule_timezone(sched.tz or default_timezone)
|
||||
)
|
||||
return croniter(sched.cron_expr, base_datetime).get_prev(datetime).timestamp()
|
||||
except (ValueError, KeyError, TypeError):
|
||||
return None
|
||||
return None
|
||||
|
||||
|
||||
def is_misfire(
|
||||
job: Any,
|
||||
now: float,
|
||||
default_timezone: str | None = None,
|
||||
) -> bool:
|
||||
"""Whether a *due* job's occurrence should be suppressed as a misfire.
|
||||
|
||||
Shared by every atomic-claim store so the misfire decision is defined once.
|
||||
Returns ``True`` when the occurrence the job is firing for is older than
|
||||
``misfire_grace_seconds`` — a stale run whose slot passed while the process
|
||||
was down. Such an occurrence is recorded as ``missed`` (observable) instead
|
||||
of firing a late, no-longer-useful run. An occurrence still within the
|
||||
grace window (or a first run) coalesces into a single normal fire on
|
||||
recovery, the pre-existing behaviour.
|
||||
|
||||
With no grace set this is always ``False`` — existing jobs keep firing
|
||||
exactly as before. A first run (``last_run_at is None``) is never a misfire
|
||||
for any kind: there is no missed *recurrence* to suppress, only the initial
|
||||
fire, which must always run (matching ``is_due``'s never-run branches).
|
||||
"""
|
||||
grace = getattr(job, "misfire_grace_seconds", None)
|
||||
if grace is None:
|
||||
return False
|
||||
# A first run is the initial fire, not a recovered recurrence — always let
|
||||
# it run. ``every`` already encodes this (``scheduled_instant`` returns
|
||||
# ``None`` with no ``last_run_at``); make it explicit for ``cron``/``at``
|
||||
# too so a never-run job past a stale slot is not marked missed unfired.
|
||||
if getattr(job, "last_run_at", None) is None:
|
||||
return False
|
||||
# Tolerate a mis-typed grace (e.g. a hand-edited quoted YAML value that
|
||||
# slipped past coercion): a bad policy value must never raise and abort the
|
||||
# whole claim pass — treat it as "no misfire" so other due jobs still fire.
|
||||
try:
|
||||
grace_val = float(grace)
|
||||
except (TypeError, ValueError):
|
||||
return False
|
||||
instant = scheduled_instant(job, now, default_timezone)
|
||||
if instant is None:
|
||||
return False
|
||||
age = now - instant
|
||||
if age <= 0:
|
||||
return False
|
||||
return age > grace_val
|
||||
|
||||
|
||||
# Upper bound on a single quota hold (24h) so a bogus/huge provider Retry-After
|
||||
# cannot park a recurring job indefinitely.
|
||||
_MAX_HOLD_SECONDS = 86_400.0
|
||||
|
||||
|
||||
def quota_hold_from_failure(
|
||||
error: Any,
|
||||
now: float,
|
||||
slack_seconds: float = 60.0,
|
||||
) -> float | None:
|
||||
"""Return an epoch to park a job past a provider's rate-limit window.
|
||||
|
||||
Pure decision helper for the "provider-quota-aware hold": given a run's
|
||||
failure it decides whether the job should stop re-firing until the
|
||||
provider's own reset window elapses. Returns ``now + Retry-After + slack``
|
||||
when the failure is a rate-limit / quota signal carrying a usable reset
|
||||
hint, else ``None`` (no hold — a non-quota failure keeps today's behaviour).
|
||||
|
||||
Reuses the core error classifier so a 429/quota error is recognised by
|
||||
exception type, HTTP status code, or message, and the reset window is read
|
||||
from the provider's ``Retry-After`` header / ``retry_after`` attribute /
|
||||
message text — no new parsing surface. The small ``slack_seconds`` avoids
|
||||
re-firing the instant the window reopens (clock skew / provider rounding).
|
||||
|
||||
Args:
|
||||
error: The failure — an ``Exception`` (preferred, so the classifier can
|
||||
read the response headers) or an error string.
|
||||
now: Epoch the failure was observed at.
|
||||
slack_seconds: Extra seconds added past the provider's reset window.
|
||||
|
||||
Returns:
|
||||
The epoch before which the job should be parked, or ``None`` when the
|
||||
failure is not a quota signal or carries no usable reset window.
|
||||
"""
|
||||
from praisonaiagents.llm.error_classifier import (
|
||||
ErrorCategory,
|
||||
classify_error,
|
||||
extract_retry_after,
|
||||
)
|
||||
|
||||
exc = error if isinstance(error, Exception) else Exception(str(error or ""))
|
||||
if classify_error(exc) != ErrorCategory.RATE_LIMIT:
|
||||
return None
|
||||
# Honour the provider's full reset window (unlike in-tick backoff, which caps
|
||||
# at 5 minutes) so a long quota window parks the job instead of re-firing —
|
||||
# but bound it to 24h so a bogus/huge value can't park a job indefinitely.
|
||||
retry_after = extract_retry_after(exc, cap_seconds=_MAX_HOLD_SECONDS)
|
||||
if not retry_after or retry_after <= 0:
|
||||
return None
|
||||
return now + float(retry_after) + max(float(slack_seconds), 0.0)
|
||||
|
||||
|
||||
def is_due(
|
||||
job: Any,
|
||||
now: float,
|
||||
@@ -101,6 +253,17 @@ def is_due(
|
||||
run_count = getattr(job, "run_count", 0)
|
||||
if max_runs is not None and run_count >= max_runs:
|
||||
return False
|
||||
# Provider-quota hold short-circuits before the kind check: while a job is
|
||||
# parked past a provider's known-closed window (a 429/quota ``Retry-After``
|
||||
# captured on a prior failure) it is not due, so it stops re-firing and
|
||||
# re-failing every tick against a benched provider. Any intervening fires
|
||||
# coalesce — the job becomes due again at the first legal instant once the
|
||||
# hold elapses. The executor clears ``hold_until`` on the first run that
|
||||
# reaches the model (proof the window reopened).
|
||||
hold_until = getattr(job, "hold_until", None)
|
||||
if hold_until is not None and now < hold_until:
|
||||
return False
|
||||
|
||||
until = getattr(job, "until", None)
|
||||
if until is not None:
|
||||
try:
|
||||
|
||||
@@ -10,6 +10,23 @@ from dataclasses import dataclass, field
|
||||
from typing import Any, Dict, List, Literal, Optional
|
||||
|
||||
|
||||
def _coerce_optional_float(value: Any) -> Optional[float]:
|
||||
"""Coerce a persisted/hand-edited value to ``Optional[float]``.
|
||||
|
||||
A hand-written config.yaml may quote a numeric field (``'3600'``) so it
|
||||
arrives as a string. Left as-is it would raise ``TypeError`` in a later
|
||||
numeric comparison (e.g. the misfire age check), aborting the whole claim
|
||||
pass and starving every other due job. Coerce leniently: ``None`` stays
|
||||
``None``; a non-numeric value is treated as unset rather than exploding.
|
||||
"""
|
||||
if value is None:
|
||||
return None
|
||||
try:
|
||||
return float(value)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
@dataclass
|
||||
class Schedule:
|
||||
"""When to run a scheduled job.
|
||||
@@ -198,12 +215,16 @@ class RunRecord:
|
||||
job_id: ID of the job that was executed.
|
||||
job_name: Human-readable name of the job.
|
||||
status: Execution status. One of ``"succeeded"``, ``"failed"``,
|
||||
``"skipped"``, or ``"no_change"``. ``"no_change"`` is the
|
||||
stateful "monitor mode" outcome — a watched source was
|
||||
``"skipped"``, ``"no_change"``, or ``"missed"``. ``"no_change"``
|
||||
is the stateful "monitor mode" outcome — a watched source was
|
||||
unchanged since the last tick, so the model turn was suppressed
|
||||
silently (no tokens, no delivery). It is distinct from
|
||||
``"skipped"`` (a generic gate go/no-go) so an operator can tell
|
||||
"nothing changed" apart from "the gate said don't run".
|
||||
``"missed"`` records a recurring occurrence that landed while
|
||||
the process was down and fell outside the job's
|
||||
``misfire_grace_seconds`` — it was not run, but is recorded
|
||||
(not silently dropped) so an operator can see the gap.
|
||||
result: Agent response text (truncated if very long).
|
||||
error: Error message if status is ``"failed"``.
|
||||
duration: Wall-clock seconds for execution.
|
||||
@@ -213,7 +234,7 @@ class RunRecord:
|
||||
|
||||
job_id: str
|
||||
job_name: str = ""
|
||||
status: Literal["succeeded", "failed", "skipped", "no_change"] = "succeeded"
|
||||
status: Literal["succeeded", "failed", "skipped", "no_change", "missed"] = "succeeded"
|
||||
result: Optional[str] = None
|
||||
error: Optional[str] = None
|
||||
duration: float = 0.0
|
||||
@@ -374,6 +395,28 @@ class ScheduleJob:
|
||||
``skipped`` (no tokens, no delivery) so a downstream never runs
|
||||
on empty inputs. Consistent with the existing ``skipped``
|
||||
outcome.
|
||||
misfire_grace_seconds: Misfire policy for a recurring occurrence missed
|
||||
while the process was down — the maximum age (seconds) of the
|
||||
occurrence being recovered for it to still run. When the slot
|
||||
the job is firing for is older than this on recovery, the
|
||||
occurrence is recorded as ``missed`` (an observable
|
||||
:class:`RunRecord`) rather than fired — so a daily brief whose
|
||||
gateway was down for hours does not surface a stale run long
|
||||
after it was useful, yet the gap is never silently dropped. An
|
||||
occurrence still within the window (or a first run) coalesces
|
||||
into a single normal fire on recovery. ``None`` (default)
|
||||
disables the check, so existing jobs are byte-for-byte
|
||||
unchanged and keep firing exactly as before.
|
||||
hold_until: Optional epoch instant before which the job is parked and
|
||||
not due, even when its schedule would otherwise fire. Set when
|
||||
a run fails with a provider rate-limit / quota signal (the
|
||||
provider's own ``Retry-After`` / reset window plus a small
|
||||
slack): the recurring job stops re-firing and re-failing every
|
||||
tick against a benched provider. Intervening fires coalesce
|
||||
into the first legal instant after the hold; the executor
|
||||
clears it on the first run that reaches the model (proof the
|
||||
window reopened). ``None`` (default) keeps today's behaviour,
|
||||
so jobs that never hit a quota wall are unchanged.
|
||||
"""
|
||||
|
||||
name: str = ""
|
||||
@@ -403,8 +446,10 @@ class ScheduleJob:
|
||||
context_from: Optional[List[str]] = None
|
||||
context_max_chars: int = 4000
|
||||
on_missing_context: Literal["run", "skip"] = "run"
|
||||
misfire_grace_seconds: Optional[float] = None
|
||||
backend: Optional[str] = None
|
||||
backend_options: Dict[str, Any] = field(default_factory=dict)
|
||||
hold_until: Optional[float] = None
|
||||
|
||||
# ── bounded-run retirement ───────────────────────────────────────
|
||||
|
||||
@@ -519,6 +564,10 @@ class ScheduleJob:
|
||||
d["context_max_chars"] = self.context_max_chars
|
||||
if self.on_missing_context != "run":
|
||||
d["on_missing_context"] = self.on_missing_context
|
||||
# Misfire policy. Only persist when configured so a job with no misfire
|
||||
# grace stays byte-for-byte unchanged.
|
||||
if self.misfire_grace_seconds is not None:
|
||||
d["misfire_grace_seconds"] = self.misfire_grace_seconds
|
||||
# External CLI backend action. Only persist when configured so agent
|
||||
# and command jobs stay byte-for-byte unchanged; options are opaque to
|
||||
# the core (the executor validates them against the backend registry).
|
||||
@@ -526,6 +575,11 @@ class ScheduleJob:
|
||||
d["backend"] = self.backend
|
||||
if self.backend_options:
|
||||
d["backend_options"] = dict(self.backend_options)
|
||||
# Provider-quota hold. Only persist when parked so a job that never hit
|
||||
# a quota wall stays byte-for-byte unchanged; the parked instant must
|
||||
# survive a restart so a benched job stays parked across processes.
|
||||
if self.hold_until is not None:
|
||||
d["hold_until"] = self.hold_until
|
||||
# Atomic-claim lease metadata (set dynamically by stores that support
|
||||
# ``claim_due``). Persisted so a lease is visible across processes and
|
||||
# survives a restart; omitted when no lease is held.
|
||||
@@ -577,12 +631,16 @@ class ScheduleJob:
|
||||
context_from=context_from,
|
||||
context_max_chars=d.get("context_max_chars", 4000),
|
||||
on_missing_context=d.get("on_missing_context", "run"),
|
||||
misfire_grace_seconds=_coerce_optional_float(
|
||||
d.get("misfire_grace_seconds")
|
||||
),
|
||||
backend=d.get("backend"),
|
||||
backend_options=(
|
||||
dict(d["backend_options"])
|
||||
if isinstance(d.get("backend_options"), dict)
|
||||
else {}
|
||||
),
|
||||
hold_until=d.get("hold_until"),
|
||||
)
|
||||
# Restore atomic-claim lease metadata if present (see ``to_dict``).
|
||||
job._lease_until = d.get("lease_until", 0.0) or 0.0
|
||||
|
||||
@@ -14,7 +14,7 @@ import time
|
||||
from typing import Dict, Iterator, List, Optional
|
||||
|
||||
from .models import ScheduleJob, RunRecord
|
||||
from .due import is_due as _is_due
|
||||
from .due import is_due as _is_due, is_misfire as _is_misfire
|
||||
from .hook_emit import emit_schedule_add, emit_schedule_remove
|
||||
|
||||
logger = get_logger(__name__)
|
||||
@@ -187,6 +187,10 @@ class FileScheduleStore:
|
||||
# a later ``remove(job.id)`` by the runner is then a no-op, so a spent
|
||||
# one-shot yields exactly one SCHEDULE_REMOVE.
|
||||
auto_removed: List[ScheduleJob] = []
|
||||
# (job_id, job_name) for occurrences suppressed by the misfire policy.
|
||||
# Logged as ``missed`` history after the lock so a dropped occurrence is
|
||||
# observable instead of vanishing silently.
|
||||
missed: List[tuple] = []
|
||||
with self._lock, self._file_lock():
|
||||
# Re-read from disk so we observe cross-process claims/leases.
|
||||
self._reload_locked()
|
||||
@@ -208,6 +212,18 @@ class FileScheduleStore:
|
||||
auto_removed.append(self._jobs.pop(job.id))
|
||||
changed = True
|
||||
continue
|
||||
# Misfire policy: the job is due, but the occurrence it is
|
||||
# firing for may have landed while the process was down. Decide
|
||||
# whether to run it now or record it as ``missed`` (observable)
|
||||
# rather than firing a stale run or dropping it silently.
|
||||
if _is_misfire(job, now):
|
||||
# Advance the schedule past the missed slot (so the next
|
||||
# poll does not see it as due again) and record the miss,
|
||||
# without claiming or firing.
|
||||
job.last_run_at = now
|
||||
missed.append((job.id, job.name))
|
||||
changed = True
|
||||
continue
|
||||
# Win the claim: pre-advance + lease atomically.
|
||||
job.last_run_at = now
|
||||
# Count the fire here (needed so ``should_retire()`` can drop a
|
||||
@@ -237,6 +253,15 @@ class FileScheduleStore:
|
||||
return []
|
||||
for job in auto_removed:
|
||||
emit_schedule_remove(job)
|
||||
# Record suppressed occurrences so a missed run is visible in history
|
||||
# rather than silently dropped. Logged outside the file lock (log_run
|
||||
# takes only the thread lock) to keep the critical section short.
|
||||
for job_id, job_name in missed:
|
||||
self.log_run(
|
||||
job_id=job_id,
|
||||
status="missed",
|
||||
job_name=job_name,
|
||||
)
|
||||
return claimed
|
||||
|
||||
def complete(self, job_id: str, owner_id: str) -> None:
|
||||
|
||||
@@ -13,6 +13,7 @@ import uuid
|
||||
import json
|
||||
import time
|
||||
import logging
|
||||
from copy import deepcopy
|
||||
from typing import Any, Dict, List, Optional, TYPE_CHECKING
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -502,6 +503,10 @@ class Session:
|
||||
) -> bool:
|
||||
"""Persist one sub-agent transcript onto this session's own record.
|
||||
|
||||
Built-in stores merge the entry under their session file lock. Older
|
||||
custom stores retain the bounded read/write verification fallback below;
|
||||
that fallback cannot guarantee atomic preservation of concurrent entries.
|
||||
|
||||
The store replaces the whole ``AGENT_HISTORY_KEY`` map per write, so a
|
||||
naive read-then-replace would let a concurrent writer (another agent
|
||||
under the same parent) clobber this agent's entry, or vice versa. We
|
||||
@@ -509,6 +514,14 @@ class Session:
|
||||
*other* key changed underneath us, retrying a bounded number of times.
|
||||
Legacy per-agent records are always merged in (they stay on disk).
|
||||
"""
|
||||
merge_map = getattr(session_store, "merge_session_metadata_map", None)
|
||||
if callable(merge_map):
|
||||
# Built-in stores merge inside their locked read/modify/write path.
|
||||
# A post-write comparison cannot detect entries lost to a stale map.
|
||||
return merge_map(
|
||||
self.session_id, AGENT_HISTORY_KEY, {agent_key: messages},
|
||||
defaults=self._legacy_agent_histories(session_store),
|
||||
)
|
||||
update_meta = getattr(session_store, "update_session_metadata", None)
|
||||
if not callable(update_meta):
|
||||
return False
|
||||
@@ -564,6 +577,7 @@ class Session:
|
||||
{
|
||||
"role": msg.get("role", "user"),
|
||||
"content": msg.get("content", ""),
|
||||
**deepcopy(msg),
|
||||
}
|
||||
for msg in chat_history
|
||||
if isinstance(msg, dict)
|
||||
|
||||
@@ -34,7 +34,7 @@ from typing import Any, Dict, List, Optional
|
||||
|
||||
from praisonaiagents._logging import get_logger
|
||||
|
||||
from .store import DefaultSessionStore, SessionData
|
||||
from .store import DefaultSessionStore, FileLock, SessionData
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
@@ -102,7 +102,10 @@ class SqliteSessionStore(DefaultSessionStore):
|
||||
# DELETE fallback on NFS/SMB/FUSE/virtiofs (Issue #5264).
|
||||
from ..storage.sqlite import connect as _sqlite_connect
|
||||
|
||||
conn = _sqlite_connect(self.db_path, isolation_level=None)
|
||||
# guard=True: quick_check on open + forensic backup + bounded
|
||||
# least-destructive repair so a malformed index file recovers
|
||||
# instead of taking the durable path down (Issue #5387).
|
||||
conn = _sqlite_connect(self.db_path, isolation_level=None, guard=True)
|
||||
self._fts_available = self._init_schema(conn)
|
||||
self._conn = conn
|
||||
except Exception as exc:
|
||||
@@ -367,6 +370,27 @@ class SqliteSessionStore(DefaultSessionStore):
|
||||
self._index_session(session)
|
||||
return ok
|
||||
|
||||
def _save_imported_session(self, session: SessionData, *, overwrite: bool = True) -> bool:
|
||||
"""Refresh recall and routing after a verbatim portable restore."""
|
||||
ok = super()._save_imported_session(session, overwrite=overwrite)
|
||||
if ok:
|
||||
try:
|
||||
filepath = self._get_session_path(session.session_id)
|
||||
with FileLock(filepath, self.lock_timeout):
|
||||
# Read-only cache fallback is unsafe after an overwrite:
|
||||
# it can restore routing fields removed by the import.
|
||||
fresh = self._load_session_from_disk(session.session_id, filepath)
|
||||
self._reingest_spill(session.session_id, fresh)
|
||||
with self._lock:
|
||||
self._cache[session.session_id] = fresh
|
||||
self._index_session(fresh)
|
||||
except Exception as exc:
|
||||
# The JSON write succeeded, but the old index is no longer a
|
||||
# trustworthy view. Fail closed until a later refresh/rebuild.
|
||||
self._deindex_session(session.session_id)
|
||||
logger.debug("Post-import index refresh failed for %s: %s", session.session_id, exc)
|
||||
return ok
|
||||
|
||||
def _modify_session_locked(self, session_id, mutator, **kwargs) -> bool:
|
||||
"""Refresh the index after any locked read-modify-write.
|
||||
|
||||
|
||||
@@ -101,10 +101,14 @@ class SqliteTranscriptStore(DefaultSessionStore):
|
||||
# readers proceed concurrently with a writer where available.
|
||||
from ..storage.sqlite import connect as _sqlite_connect
|
||||
|
||||
# guard=True: quick_check on open + forensic backup + bounded
|
||||
# least-destructive repair so a malformed transcript DB recovers
|
||||
# instead of taking the durable path down (Issue #5387).
|
||||
conn = _sqlite_connect(
|
||||
self.db_path,
|
||||
isolation_level=None,
|
||||
busy_timeout_ms=int(self.lock_timeout * 1000),
|
||||
guard=True,
|
||||
)
|
||||
conn.execute(
|
||||
"CREATE TABLE IF NOT EXISTS sessions ("
|
||||
@@ -154,7 +158,9 @@ class SqliteTranscriptStore(DefaultSessionStore):
|
||||
if not session_dir or not os.path.isdir(session_dir):
|
||||
return
|
||||
try:
|
||||
filenames = [f for f in os.listdir(session_dir) if f.endswith(".json")]
|
||||
filenames = sorted(
|
||||
f for f in os.listdir(session_dir) if f.endswith(".json")
|
||||
)
|
||||
except OSError:
|
||||
return
|
||||
if not filenames:
|
||||
@@ -171,7 +177,7 @@ class SqliteTranscriptStore(DefaultSessionStore):
|
||||
try:
|
||||
with open(filepath, "r", encoding="utf-8") as f:
|
||||
data = json.load(f)
|
||||
except (OSError, json.JSONDecodeError, TypeError):
|
||||
except (OSError, UnicodeDecodeError, json.JSONDecodeError, TypeError):
|
||||
continue
|
||||
if not isinstance(data, dict):
|
||||
continue
|
||||
@@ -210,7 +216,7 @@ class SqliteTranscriptStore(DefaultSessionStore):
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
return None
|
||||
|
||||
def _write_row(self, session: SessionData, conn=None) -> bool:
|
||||
def _write_row(self, session: SessionData, conn=None, *, overwrite: bool = True) -> bool:
|
||||
own_lock = conn is None
|
||||
if conn is None:
|
||||
conn = self._connect()
|
||||
@@ -229,17 +235,21 @@ class SqliteTranscriptStore(DefaultSessionStore):
|
||||
getattr(session, "updated_at", None),
|
||||
)
|
||||
sql = (
|
||||
"INSERT OR REPLACE INTO sessions "
|
||||
("INSERT OR REPLACE" if overwrite else "INSERT OR IGNORE") + " INTO sessions "
|
||||
"(session_id, data, agent_name, gateway_session_id, agent_id, updated_at) "
|
||||
"VALUES (?, ?, ?, ?, ?, ?)"
|
||||
)
|
||||
try:
|
||||
if own_lock:
|
||||
with self._db_lock:
|
||||
conn.execute(sql, params)
|
||||
cursor = conn.execute(sql, params)
|
||||
else:
|
||||
conn.execute(sql, params)
|
||||
cursor = conn.execute(sql, params)
|
||||
if not overwrite and cursor.rowcount == 0:
|
||||
raise FileExistsError(session.session_id)
|
||||
return True
|
||||
except FileExistsError:
|
||||
raise
|
||||
except Exception as exc:
|
||||
logger.error("Failed to write session %s: %s", session.session_id, exc)
|
||||
return False
|
||||
@@ -282,6 +292,52 @@ class SqliteTranscriptStore(DefaultSessionStore):
|
||||
self._cache[session.session_id] = session
|
||||
return True
|
||||
|
||||
def _save_imported_session(self, session: SessionData, *, overwrite: bool = True) -> bool:
|
||||
"""Restore to the database without applying the destination's window."""
|
||||
session.updated_at = datetime.now(timezone.utc).isoformat()
|
||||
with self._db_lock:
|
||||
# INSERT OR IGNORE makes no-clobber atomic across store instances;
|
||||
# checking session_exists before a replace would still race.
|
||||
if not self._write_row(session, overwrite=overwrite):
|
||||
return False
|
||||
with self._lock:
|
||||
self._cache[session.session_id] = session
|
||||
return True
|
||||
|
||||
def export_all(self) -> Dict[str, Any]:
|
||||
"""Export durable database rows, not legacy JSON sidecar files."""
|
||||
return {
|
||||
"version": self.PORTABLE_VERSION,
|
||||
"sessions": [data for data in self._all_rows() if isinstance(data, dict)],
|
||||
}
|
||||
|
||||
def _collect_lineage(
|
||||
self, session: SessionData, *, exclude: str
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Collect persisted database records from the same conversation chain."""
|
||||
lineage = self._lineage_key(session.to_dict())
|
||||
if not lineage:
|
||||
return []
|
||||
|
||||
def record_lineage(raw):
|
||||
# Preserve the writer's Python JSON semantics (including NaN)
|
||||
# and the shared lineage precedence, without retaining other rows.
|
||||
try:
|
||||
data = json.loads(raw)
|
||||
return self._lineage_key(data) if isinstance(data, dict) else None
|
||||
except (json.JSONDecodeError, TypeError, AttributeError):
|
||||
return None
|
||||
|
||||
conn = self._connect()
|
||||
with self._db_lock:
|
||||
conn.create_function("portable_session_lineage", 1, record_lineage)
|
||||
rows = conn.execute(
|
||||
"SELECT data FROM sessions WHERE session_id != ? "
|
||||
"AND portable_session_lineage(data) = ? ORDER BY updated_at DESC",
|
||||
(exclude, lineage),
|
||||
).fetchall()
|
||||
return [json.loads(row[0]) for row in rows]
|
||||
|
||||
def _modify_session_locked(
|
||||
self,
|
||||
session_id: str,
|
||||
|
||||
@@ -1622,6 +1622,27 @@ class DefaultSessionStore:
|
||||
session_id, _apply, error_label="update session metadata"
|
||||
)
|
||||
|
||||
def merge_session_metadata_map(
|
||||
self, session_id: str, key: str, updates: Dict[str, Any],
|
||||
*, defaults: Optional[Dict[str, Any]] = None,
|
||||
) -> bool:
|
||||
"""Merge nested metadata entries using the freshest map under FileLock.
|
||||
|
||||
Defaults fill missing entries; persisted entries win over defaults,
|
||||
and the supplied updates win over both.
|
||||
"""
|
||||
def _apply(session: SessionData) -> None:
|
||||
merged = dict(defaults or {})
|
||||
current = session.metadata.get(key)
|
||||
if isinstance(current, dict):
|
||||
merged.update(current)
|
||||
merged.update(updates)
|
||||
session.metadata[key] = merged
|
||||
|
||||
return self._modify_session_locked(
|
||||
session_id, _apply, error_label="merge session metadata map"
|
||||
)
|
||||
|
||||
def rename_session(self, session_id: str, title: str) -> bool:
|
||||
"""Give a session a human-readable title (Issue #3737).
|
||||
|
||||
@@ -2306,7 +2327,7 @@ class DefaultSessionStore:
|
||||
out.append(data)
|
||||
return out
|
||||
|
||||
def _save_imported_session(self, session: SessionData) -> bool:
|
||||
def _save_imported_session(self, session: SessionData, *, overwrite: bool = True) -> bool:
|
||||
"""Persist a restored session verbatim (no retention/window applied).
|
||||
|
||||
Mirrors ``_save_session`` (timestamp + atomic, file-locked write) but
|
||||
@@ -2317,6 +2338,10 @@ class DefaultSessionStore:
|
||||
filepath = self._get_session_path(session.session_id)
|
||||
session.updated_at = datetime.now(timezone.utc).isoformat()
|
||||
with FileLock(filepath, self.lock_timeout):
|
||||
# The initial import check can race with a peer creating this file.
|
||||
# Check again inside the same lock that protects the replacement.
|
||||
if not overwrite and os.path.exists(filepath):
|
||||
raise FileExistsError(filepath)
|
||||
if not self._atomic_write_json(filepath, session.to_dict()):
|
||||
logger.error(f"Failed to save imported session {session.session_id}")
|
||||
return False
|
||||
@@ -2416,7 +2441,7 @@ class DefaultSessionStore:
|
||||
# Persist the imported record verbatim: an import is a restore,
|
||||
# so the destination's retention/active_window must not truncate
|
||||
# or compact a valid larger export before it lands on disk.
|
||||
if not self._save_imported_session(session):
|
||||
if not self._save_imported_session(session, overwrite=overwrite):
|
||||
report.skipped.append(
|
||||
{"session_id": session_id, "reason": "write failed"}
|
||||
)
|
||||
@@ -2424,6 +2449,10 @@ class DefaultSessionStore:
|
||||
with self._lock:
|
||||
self._cache[session_id] = session
|
||||
report.imported += 1
|
||||
except FileExistsError:
|
||||
report.skipped.append(
|
||||
{"session_id": session_id, "reason": "already exists (use overwrite)"}
|
||||
)
|
||||
except Exception as e: # pragma: no cover - defensive; one bad record
|
||||
report.skipped.append(
|
||||
{"session_id": session_id, "reason": f"import error: {e}"}
|
||||
|
||||
@@ -66,6 +66,9 @@ def generate_title(
|
||||
... )
|
||||
>>> # Returns something like: "Python Code Debugging Help"
|
||||
"""
|
||||
if max_length <= 0:
|
||||
return ""
|
||||
|
||||
# Fallback title from user message if generation fails
|
||||
fallback_title = _create_fallback_title(user_msg, max_length)
|
||||
|
||||
@@ -119,6 +122,9 @@ async def generate_title_async(
|
||||
Returns:
|
||||
Generated title string, or fallback based on user message
|
||||
"""
|
||||
if max_length <= 0:
|
||||
return ""
|
||||
|
||||
fallback_title = _create_fallback_title(user_msg, max_length)
|
||||
|
||||
try:
|
||||
@@ -152,8 +158,7 @@ Return ONLY the title text, no quotes, no explanation."""
|
||||
title = title.replace('\n', ' ').replace('\r', '')
|
||||
|
||||
# Truncate if too long
|
||||
if len(title) > max_length:
|
||||
title = title[:max_length-3] + "..."
|
||||
title = _truncate_title(title, max_length)
|
||||
|
||||
# Return if non-empty
|
||||
if title and len(title.strip()) > 0:
|
||||
@@ -176,6 +181,17 @@ Return ONLY the title text, no quotes, no explanation."""
|
||||
return fallback_title
|
||||
|
||||
|
||||
def _truncate_title(title: str, max_length: int) -> str:
|
||||
"""Apply the character cap without using negative ellipsis slices."""
|
||||
if max_length <= 0:
|
||||
return ""
|
||||
if len(title) <= max_length:
|
||||
return title
|
||||
if max_length <= 3:
|
||||
return title[:max_length]
|
||||
return title[:max_length - 3] + "..."
|
||||
|
||||
|
||||
def _create_fallback_title(user_msg: str, max_length: int) -> str:
|
||||
"""Create a fallback title from the user message.
|
||||
|
||||
@@ -187,7 +203,7 @@ def _create_fallback_title(user_msg: str, max_length: int) -> str:
|
||||
Simple title based on user message
|
||||
"""
|
||||
if not user_msg or not user_msg.strip():
|
||||
return "Chat Session"
|
||||
return _truncate_title("Chat Session", max_length)
|
||||
|
||||
# Clean up the message
|
||||
clean_msg = user_msg.strip()
|
||||
@@ -200,8 +216,4 @@ def _create_fallback_title(user_msg: str, max_length: int) -> str:
|
||||
if '.' in clean_msg:
|
||||
clean_msg = clean_msg.split('.')[0]
|
||||
|
||||
# Truncate if too long
|
||||
if len(clean_msg) > max_length:
|
||||
clean_msg = clean_msg[:max_length-3] + "..."
|
||||
|
||||
return clean_msg if clean_msg else "Chat Session"
|
||||
return _truncate_title(clean_msg or "Chat Session", max_length)
|
||||
@@ -31,6 +31,8 @@ __all__ = [
|
||||
"get_backend",
|
||||
# Hardened stdlib SQLite connection factory (WAL-with-safe-fallback)
|
||||
"sqlite_connect",
|
||||
# Structural integrity probe for durable SQLite stores (Issue #5387)
|
||||
"sqlite_quick_check",
|
||||
]
|
||||
|
||||
_LAZY_IMPORTS = {
|
||||
@@ -49,6 +51,7 @@ _LAZY_IMPORTS = {
|
||||
"SQLiteBackend": ("backends", "SQLiteBackend"),
|
||||
"get_backend": ("backends", "get_backend"),
|
||||
"sqlite_connect": ("sqlite", "connect"),
|
||||
"sqlite_quick_check": ("sqlite", "quick_check"),
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -69,6 +69,17 @@ class FileBackend:
|
||||
# Sanitize key for filesystem
|
||||
safe_key = "".join(c if c.isalnum() or c in "-_" else "_" for c in key)
|
||||
return self.storage_dir / f"{safe_key}{self.suffix}"
|
||||
|
||||
def _path_to_key(self, file_path: Path) -> Optional[str]:
|
||||
"""Recognize records using the full configured suffix."""
|
||||
if not file_path.is_file():
|
||||
return None
|
||||
if self.suffix:
|
||||
if file_path.name.endswith(self.suffix):
|
||||
return file_path.name[:-len(self.suffix)]
|
||||
return None
|
||||
# Preserve the existing no-extension selection for an empty suffix.
|
||||
return file_path.name if not file_path.suffix else None
|
||||
|
||||
def save(self, key: str, data: Dict[str, Any]) -> None:
|
||||
"""Save data with the given key."""
|
||||
@@ -84,11 +95,11 @@ class FileBackend:
|
||||
delete=False,
|
||||
suffix=".tmp"
|
||||
) as f:
|
||||
temp_path = f.name
|
||||
if self.pretty:
|
||||
json.dump(data, f, indent=2, default=str, ensure_ascii=False)
|
||||
else:
|
||||
json.dump(data, f, default=str, ensure_ascii=False)
|
||||
temp_path = f.name
|
||||
|
||||
os.replace(temp_path, file_path)
|
||||
except Exception as e:
|
||||
@@ -110,7 +121,7 @@ class FileBackend:
|
||||
try:
|
||||
with open(file_path, "r", encoding="utf-8") as f:
|
||||
return json.load(f)
|
||||
except (json.JSONDecodeError, IOError) as e:
|
||||
except (json.JSONDecodeError, UnicodeDecodeError, IOError) as e:
|
||||
logger.warning(f"Failed to load {key}: {e}")
|
||||
return None
|
||||
|
||||
@@ -133,8 +144,8 @@ class FileBackend:
|
||||
keys = []
|
||||
|
||||
for file_path in self.storage_dir.iterdir():
|
||||
if file_path.is_file() and file_path.suffix == self.suffix:
|
||||
key = file_path.stem
|
||||
key = self._path_to_key(file_path)
|
||||
if key is not None:
|
||||
if not prefix or key.startswith(prefix):
|
||||
keys.append(key)
|
||||
|
||||
@@ -149,7 +160,7 @@ class FileBackend:
|
||||
count = 0
|
||||
with self._lock:
|
||||
for file_path in self.storage_dir.iterdir():
|
||||
if file_path.is_file() and file_path.suffix == self.suffix:
|
||||
if self._path_to_key(file_path) is not None:
|
||||
try:
|
||||
file_path.unlink()
|
||||
count += 1
|
||||
@@ -191,13 +202,21 @@ class SQLiteBackend:
|
||||
if not isinstance(table_name, str) or not _re.match(r'^[a-zA-Z0-9_]+$', table_name):
|
||||
raise ValueError("table_name must contain only alphanumeric characters and underscores")
|
||||
self.table_name = table_name
|
||||
# Quote identifiers so validated names that are SQL keywords or start
|
||||
# with a digit (e.g. "select", "9table") are valid in all statements.
|
||||
self._quoted_table = f'"{table_name}"'
|
||||
self._quoted_index = f'"idx_{table_name}_key"'
|
||||
self._local = threading.local()
|
||||
|
||||
# Ensure directory exists
|
||||
Path(self.db_path).parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
if auto_create:
|
||||
self._create_table()
|
||||
try:
|
||||
self._create_table()
|
||||
except Exception:
|
||||
self.close()
|
||||
raise
|
||||
|
||||
def _get_conn(self):
|
||||
"""Get thread-local connection."""
|
||||
@@ -220,7 +239,7 @@ class SQLiteBackend:
|
||||
cur = conn.cursor()
|
||||
|
||||
cur.execute(f"""
|
||||
CREATE TABLE IF NOT EXISTS {self.table_name} (
|
||||
CREATE TABLE IF NOT EXISTS {self._quoted_table} (
|
||||
key TEXT PRIMARY KEY,
|
||||
data TEXT NOT NULL,
|
||||
created_at REAL DEFAULT (strftime('%s', 'now')),
|
||||
@@ -230,8 +249,8 @@ class SQLiteBackend:
|
||||
|
||||
# Index for prefix queries
|
||||
cur.execute(f"""
|
||||
CREATE INDEX IF NOT EXISTS idx_{self.table_name}_key
|
||||
ON {self.table_name}(key)
|
||||
CREATE INDEX IF NOT EXISTS {self._quoted_index}
|
||||
ON {self._quoted_table}(key)
|
||||
""")
|
||||
|
||||
conn.commit()
|
||||
@@ -244,7 +263,7 @@ class SQLiteBackend:
|
||||
json_data = json.dumps(data, default=str, ensure_ascii=False)
|
||||
|
||||
cur.execute(f"""
|
||||
INSERT INTO {self.table_name} (key, data, updated_at)
|
||||
INSERT INTO {self._quoted_table} (key, data, updated_at)
|
||||
VALUES (?, ?, strftime('%s', 'now'))
|
||||
ON CONFLICT(key) DO UPDATE SET
|
||||
data = excluded.data,
|
||||
@@ -259,7 +278,7 @@ class SQLiteBackend:
|
||||
cur = conn.cursor()
|
||||
|
||||
cur.execute(f"""
|
||||
SELECT data FROM {self.table_name} WHERE key = ?
|
||||
SELECT data FROM {self._quoted_table} WHERE key = ?
|
||||
""", (key,))
|
||||
|
||||
row = cur.fetchone()
|
||||
@@ -276,7 +295,7 @@ class SQLiteBackend:
|
||||
cur = conn.cursor()
|
||||
|
||||
cur.execute(f"""
|
||||
DELETE FROM {self.table_name} WHERE key = ?
|
||||
DELETE FROM {self._quoted_table} WHERE key = ?
|
||||
""", (key,))
|
||||
|
||||
deleted = cur.rowcount > 0
|
||||
@@ -290,13 +309,13 @@ class SQLiteBackend:
|
||||
|
||||
if prefix:
|
||||
cur.execute(f"""
|
||||
SELECT key FROM {self.table_name}
|
||||
SELECT key FROM {self._quoted_table}
|
||||
WHERE key LIKE ?
|
||||
ORDER BY key
|
||||
""", (f"{prefix}%",))
|
||||
else:
|
||||
cur.execute(f"""
|
||||
SELECT key FROM {self.table_name}
|
||||
SELECT key FROM {self._quoted_table}
|
||||
ORDER BY key
|
||||
""")
|
||||
|
||||
@@ -308,7 +327,7 @@ class SQLiteBackend:
|
||||
cur = conn.cursor()
|
||||
|
||||
cur.execute(f"""
|
||||
SELECT 1 FROM {self.table_name} WHERE key = ? LIMIT 1
|
||||
SELECT 1 FROM {self._quoted_table} WHERE key = ? LIMIT 1
|
||||
""", (key,))
|
||||
|
||||
return cur.fetchone() is not None
|
||||
@@ -318,10 +337,10 @@ class SQLiteBackend:
|
||||
conn = self._get_conn()
|
||||
cur = conn.cursor()
|
||||
|
||||
cur.execute(f"SELECT COUNT(*) as count FROM {self.table_name}")
|
||||
cur.execute(f"SELECT COUNT(*) as count FROM {self._quoted_table}")
|
||||
count = cur.fetchone()["count"]
|
||||
|
||||
cur.execute(f"DELETE FROM {self.table_name}")
|
||||
cur.execute(f"DELETE FROM {self._quoted_table}")
|
||||
conn.commit()
|
||||
|
||||
return count
|
||||
|
||||
@@ -262,8 +262,8 @@ class BaseJSONStore:
|
||||
delete=False,
|
||||
suffix=".tmp"
|
||||
) as f:
|
||||
json.dump(self._data, f, indent=2, default=str, ensure_ascii=False)
|
||||
temp_path = f.name
|
||||
json.dump(self._data, f, indent=2, default=str, ensure_ascii=False)
|
||||
|
||||
# Atomic rename
|
||||
if self.use_file_lock:
|
||||
@@ -343,7 +343,7 @@ class BaseJSONStore:
|
||||
def list_json_sessions(
|
||||
storage_dir: Path,
|
||||
suffix: str = ".json",
|
||||
limit: int = 50,
|
||||
limit: Optional[int] = 50,
|
||||
) -> List[BaseSessionInfo]:
|
||||
"""
|
||||
List all sessions in a directory.
|
||||
@@ -353,7 +353,7 @@ def list_json_sessions(
|
||||
Args:
|
||||
storage_dir: Directory to search
|
||||
suffix: File suffix to filter by
|
||||
limit: Maximum number of sessions to return
|
||||
limit: Maximum number of sessions to return, or None for all sessions
|
||||
|
||||
Returns:
|
||||
List of BaseSessionInfo, sorted by modification time (newest first)
|
||||
@@ -370,7 +370,7 @@ def list_json_sessions(
|
||||
# Try to get item count from file
|
||||
if suffix == ".json":
|
||||
try:
|
||||
with open(file_path, "r") as f:
|
||||
with open(file_path, "r", encoding="utf-8") as f:
|
||||
data = json.load(f)
|
||||
# Common patterns for item count
|
||||
if "iterations" in data:
|
||||
@@ -383,7 +383,7 @@ def list_json_sessions(
|
||||
pass
|
||||
elif suffix == ".jsonl":
|
||||
try:
|
||||
with open(file_path, "r") as f:
|
||||
with open(file_path, "r", encoding="utf-8") as f:
|
||||
info.item_count = sum(1 for _ in f)
|
||||
except Exception:
|
||||
pass
|
||||
@@ -424,7 +424,7 @@ def cleanup_old_sessions(
|
||||
now = datetime.now()
|
||||
|
||||
# Get all sessions sorted by age (oldest first)
|
||||
sessions = list_json_sessions(storage_dir, suffix, limit=10000)
|
||||
sessions = list_json_sessions(storage_dir, suffix, limit=None)
|
||||
sessions.sort(key=lambda s: s.modified_at)
|
||||
|
||||
# Delete old sessions
|
||||
|
||||
@@ -151,6 +151,343 @@ def apply_wal_with_fallback(
|
||||
return mode
|
||||
|
||||
|
||||
# Maximum forensic backups of a malformed file kept before the oldest is
|
||||
# pruned, so a boot-loop on a permanently-bad file cannot fill the disk.
|
||||
_MAX_FORENSIC_BACKUPS = 3
|
||||
# A malformed file is repaired at most this many times before the guard gives
|
||||
# up (records the exhaustion in a sidecar ledger) and stops re-repairing on
|
||||
# every boot, so a permanently-bad file is not churned each start.
|
||||
_MAX_REPAIR_ATTEMPTS = 3
|
||||
|
||||
|
||||
def _is_transient_lock_error(exc: Exception) -> bool:
|
||||
"""True if an error is a transient lock/busy condition, not corruption.
|
||||
|
||||
A healthy database that a peer momentarily locks raises ``database is
|
||||
locked`` / ``database is busy``. Treating that as corruption would trigger a
|
||||
needless forensic copy and repair on a file that was never corrupt, so the
|
||||
guard must distinguish it from a genuine "malformed image" error.
|
||||
"""
|
||||
text = str(exc).lower()
|
||||
return "locked" in text or "database is busy" in text
|
||||
|
||||
|
||||
def quick_check(conn) -> bool:
|
||||
"""Return True if ``PRAGMA quick_check`` reports the database is intact.
|
||||
|
||||
``quick_check`` is the cheap structural probe (it skips the exhaustive
|
||||
per-row index cross-check that ``integrity_check`` does), so it is safe to
|
||||
run once on open without touching the hot path. Any SQLite-level error
|
||||
(a malformed header, "database disk image is malformed", an I/O error) is
|
||||
treated as *not intact* so the caller can route to backup/repair. A
|
||||
transient lock/busy error is re-raised so callers can tell "locked" (leave
|
||||
alone) apart from "corrupt" (back up + repair).
|
||||
"""
|
||||
try:
|
||||
row = conn.execute("PRAGMA quick_check(1)").fetchone()
|
||||
except Exception as exc: # DatabaseError / OperationalError on a bad file
|
||||
if _is_transient_lock_error(exc):
|
||||
raise
|
||||
logger.warning("quick_check raised for database (%s); treating as corrupt.", exc)
|
||||
return False
|
||||
return bool(row) and str(row[0]).lower() == "ok"
|
||||
|
||||
|
||||
def _sidecar_paths(path: str):
|
||||
"""Return the WAL/SHM sidecar paths for a database file."""
|
||||
return [path + "-wal", path + "-shm"]
|
||||
|
||||
|
||||
def _forensic_backup(path: str) -> Optional[str]:
|
||||
"""Copy a malformed database (+WAL/SHM sidecars) aside for forensics.
|
||||
|
||||
Mirrors the JSON store's ``_quarantine_corrupt`` behaviour: the raw,
|
||||
possibly-recoverable bytes are preserved as ``<file>.corrupt-<epoch_ms>``
|
||||
instead of being clobbered by a repair. Fails **closed** — returns ``None``
|
||||
without copying — when free disk space is below the file size plus a margin,
|
||||
so a corruption event never itself fills the disk. Retention-capped so a
|
||||
boot-loop cannot accumulate unbounded copies.
|
||||
"""
|
||||
import shutil
|
||||
import time as _time
|
||||
|
||||
try:
|
||||
size = os.path.getsize(path)
|
||||
except OSError:
|
||||
return None
|
||||
try:
|
||||
free = shutil.disk_usage(os.path.dirname(path) or ".").free
|
||||
except OSError:
|
||||
free = None
|
||||
# Fail closed on low disk: need room for the file plus a small margin.
|
||||
if free is not None and free < size * 2 + (1 << 20):
|
||||
logger.error(
|
||||
"Skipping forensic backup of %s: insufficient free disk space.", path
|
||||
)
|
||||
return None
|
||||
|
||||
base = f"{path}.corrupt-{int(_time.time() * 1000)}"
|
||||
dest = base
|
||||
attempt = 1
|
||||
while os.path.exists(dest):
|
||||
dest = f"{base}-{attempt}"
|
||||
attempt += 1
|
||||
try:
|
||||
shutil.copy2(path, dest)
|
||||
except OSError as exc:
|
||||
logger.error("Forensic backup of %s failed: %s", path, exc)
|
||||
return None
|
||||
# Best-effort sidecar preservation; their absence is not fatal. Each sidecar
|
||||
# is ``path + suffix`` (e.g. ``-wal``), copied next to the primary backup so
|
||||
# a later analysis can replay the write-ahead log.
|
||||
for sidecar in _sidecar_paths(path):
|
||||
if os.path.exists(sidecar):
|
||||
suffix = sidecar[len(path):]
|
||||
try:
|
||||
shutil.copy2(sidecar, dest + suffix)
|
||||
except OSError:
|
||||
pass
|
||||
_prune_forensic_backups(path)
|
||||
return dest
|
||||
|
||||
|
||||
def _prune_forensic_backups(path: str) -> None:
|
||||
"""Keep only the newest ``_MAX_FORENSIC_BACKUPS`` forensic copies."""
|
||||
directory = os.path.dirname(path) or "."
|
||||
prefix = os.path.basename(path) + ".corrupt-"
|
||||
try:
|
||||
entries = [
|
||||
os.path.join(directory, name)
|
||||
for name in os.listdir(directory)
|
||||
if name.startswith(prefix)
|
||||
]
|
||||
except OSError:
|
||||
return
|
||||
# Only prune the primary DB copies (no sidecar suffix like ``-wal``).
|
||||
copies = [p for p in entries if not (p.endswith("-wal") or p.endswith("-shm"))]
|
||||
if len(copies) <= _MAX_FORENSIC_BACKUPS:
|
||||
return
|
||||
copies.sort(key=lambda p: os.path.getmtime(p))
|
||||
for stale in copies[: len(copies) - _MAX_FORENSIC_BACKUPS]:
|
||||
# Remove the primary copy together with its own ``-wal``/``-shm``
|
||||
# sidecars so a WAL database's forensic sidecars cannot accumulate past
|
||||
# the retention cap.
|
||||
for target in (stale, stale + "-wal", stale + "-shm"):
|
||||
try:
|
||||
os.remove(target)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
def _repair_ledger_path(path: str) -> str:
|
||||
return path + ".repair"
|
||||
|
||||
|
||||
def _repair_budget_ok(path: str) -> bool:
|
||||
"""Return True while the file still has repair attempts left in its budget.
|
||||
|
||||
A sidecar ledger (``<file>.repair``) records how many repairs have been
|
||||
attempted so a permanently-malformed file is not re-repaired on every boot
|
||||
(bounded persistent retry). Missing/unreadable ledger means a fresh budget.
|
||||
"""
|
||||
try:
|
||||
with open(_repair_ledger_path(path), "r", encoding="utf-8") as fh:
|
||||
attempts = int(fh.read().strip() or "0")
|
||||
except (OSError, ValueError):
|
||||
attempts = 0
|
||||
return attempts < _MAX_REPAIR_ATTEMPTS
|
||||
|
||||
|
||||
def _record_repair_attempt(path: str) -> None:
|
||||
try:
|
||||
with open(_repair_ledger_path(path), "r", encoding="utf-8") as fh:
|
||||
attempts = int(fh.read().strip() or "0")
|
||||
except (OSError, ValueError):
|
||||
attempts = 0
|
||||
try:
|
||||
with open(_repair_ledger_path(path), "w", encoding="utf-8") as fh:
|
||||
fh.write(str(attempts + 1))
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
def _clear_repair_ledger(path: str) -> None:
|
||||
for p in (_repair_ledger_path(path),):
|
||||
try:
|
||||
os.remove(p)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
def _repair_via_online_backup(path: str) -> bool:
|
||||
"""Attempt a least-destructive repair of a malformed database in place.
|
||||
|
||||
Recovers what SQLite's online-backup API can read into a fresh snapshot and
|
||||
promotes it back with an atomic ``os.replace`` so the canonical file is only
|
||||
swapped for a *verified-intact* rebuild — a failed attempt never touches the
|
||||
original bytes (which are already preserved by :func:`_forensic_backup`).
|
||||
Returns True only when the rebuilt file itself passes ``quick_check``.
|
||||
"""
|
||||
import sqlite3
|
||||
|
||||
_record_repair_attempt(path)
|
||||
# A per-attempt-unique snapshot name so two concurrent repairs of the same
|
||||
# file never delete or promote each other's in-flight rebuild.
|
||||
snapshot = f"{path}.repair-tmp-{os.getpid()}-{id(path)}"
|
||||
try:
|
||||
os.remove(snapshot)
|
||||
except OSError:
|
||||
pass
|
||||
src = dst = None
|
||||
try:
|
||||
src = sqlite3.connect(path)
|
||||
dst = sqlite3.connect(snapshot)
|
||||
src.backup(dst) # copies all readable pages into the fresh file
|
||||
dst.commit()
|
||||
except Exception as exc:
|
||||
logger.warning("Online-backup repair of %s failed to read pages: %s", path, exc)
|
||||
for c in (src, dst):
|
||||
try:
|
||||
if c is not None:
|
||||
c.close()
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
os.remove(snapshot)
|
||||
except OSError:
|
||||
pass
|
||||
return False
|
||||
finally:
|
||||
for c in (src, dst):
|
||||
try:
|
||||
if c is not None:
|
||||
c.close()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Verify the rebuilt snapshot before promoting it over the canonical file.
|
||||
verify = None
|
||||
try:
|
||||
verify = sqlite3.connect(snapshot)
|
||||
intact = quick_check(verify)
|
||||
except Exception:
|
||||
intact = False
|
||||
finally:
|
||||
try:
|
||||
if verify is not None:
|
||||
verify.close()
|
||||
except Exception:
|
||||
pass
|
||||
if not intact:
|
||||
try:
|
||||
os.remove(snapshot)
|
||||
except OSError:
|
||||
pass
|
||||
return False
|
||||
|
||||
try:
|
||||
# Drop stale sidecars of the malformed file so they cannot be replayed
|
||||
# over the rebuilt database. The malformed original + its sidecars are
|
||||
# already forensically backed up before we get here.
|
||||
for sidecar in _sidecar_paths(path):
|
||||
try:
|
||||
os.remove(sidecar)
|
||||
except OSError:
|
||||
pass
|
||||
os.replace(snapshot, path)
|
||||
except OSError as exc:
|
||||
logger.error("Could not promote repaired snapshot for %s: %s", path, exc)
|
||||
return False
|
||||
_clear_repair_ledger(path)
|
||||
logger.warning("Repaired malformed SQLite database %s via online backup.", path)
|
||||
return True
|
||||
|
||||
|
||||
def _quarantine_unrepairable(path: str) -> None:
|
||||
"""Move an unrepairable malformed file aside so a fresh DB can be created.
|
||||
|
||||
When the online-backup API cannot rebuild an intact snapshot (e.g. a
|
||||
corrupt header that reads as "file is not a database"), the malformed bytes
|
||||
are already preserved by :func:`_forensic_backup`; renaming the canonical
|
||||
file out of the way lets the store open a fresh, empty database instead of
|
||||
crashing on the first ``CREATE TABLE`` — mirroring the JSON store's
|
||||
quarantine-and-continue behaviour (Issue #5387). Sidecars go too so a stale
|
||||
``-wal`` cannot be replayed over the new file.
|
||||
"""
|
||||
for sidecar in _sidecar_paths(path):
|
||||
try:
|
||||
os.remove(sidecar)
|
||||
except OSError:
|
||||
pass
|
||||
try:
|
||||
os.remove(path)
|
||||
except OSError as exc:
|
||||
logger.error("Could not clear unrepairable malformed file %s: %s", path, exc)
|
||||
|
||||
|
||||
def _guard_integrity(path: str, *, repair: bool, backup: bool) -> None:
|
||||
"""Detect and (optionally) repair a malformed database before first use.
|
||||
|
||||
Runs a cheap ``quick_check`` on open; on failure it forensically backs up
|
||||
the malformed file (+sidecars) and attempts a bounded, least-destructive
|
||||
online-backup repair. If the file cannot be repaired, its (already backed
|
||||
up) bytes are quarantined so the store can open a fresh database rather than
|
||||
crash on first use. A no-op for ``:memory:``/absent/empty files. A transient
|
||||
lock/busy error is treated as "healthy but busy" and left untouched.
|
||||
"""
|
||||
import sqlite3
|
||||
|
||||
if not path or path == ":memory:" or not os.path.exists(path):
|
||||
return
|
||||
try:
|
||||
if os.path.getsize(path) == 0:
|
||||
return
|
||||
except OSError:
|
||||
return
|
||||
|
||||
probe = None
|
||||
try:
|
||||
probe = sqlite3.connect(path)
|
||||
if quick_check(probe):
|
||||
return
|
||||
except Exception as exc:
|
||||
# A transient lock is not corruption — never back up or repair a file
|
||||
# that a peer merely holds open. Leave it exactly as-is.
|
||||
if _is_transient_lock_error(exc):
|
||||
logger.info(
|
||||
"SQLite database %s is locked/busy on probe; skipping guard.", path
|
||||
)
|
||||
return
|
||||
finally:
|
||||
try:
|
||||
if probe is not None:
|
||||
probe.close()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
logger.error("SQLite database %s failed integrity check; corrupt.", path)
|
||||
|
||||
# Fail closed: if a forensic backup was requested but could not be taken
|
||||
# (low disk, copy error), do NOT proceed to repair — repair's atomic
|
||||
# promotion would otherwise destroy the only copy of the malformed bytes.
|
||||
if backup:
|
||||
if _forensic_backup(path) is None:
|
||||
logger.error(
|
||||
"Forensic backup of %s failed; skipping repair to preserve "
|
||||
"the malformed file for recovery.",
|
||||
path,
|
||||
)
|
||||
return
|
||||
|
||||
if repair and _repair_budget_ok(path):
|
||||
if _repair_via_online_backup(path):
|
||||
return
|
||||
# Repair could not rebuild an intact snapshot. The malformed bytes are
|
||||
# safely backed up, so quarantine the canonical file to let the durable
|
||||
# store open a fresh database instead of crashing on first query.
|
||||
_quarantine_unrepairable(path)
|
||||
|
||||
|
||||
def connect(
|
||||
path,
|
||||
*,
|
||||
@@ -158,6 +495,9 @@ def connect(
|
||||
isolation_level: Optional[str] = "",
|
||||
busy_timeout_ms: int = 5000,
|
||||
synchronous: str = "FULL",
|
||||
guard: bool = False,
|
||||
repair: bool = True,
|
||||
backup: bool = True,
|
||||
**kwargs,
|
||||
):
|
||||
"""Open a hardened SQLite connection used by core durable stores.
|
||||
@@ -177,6 +517,14 @@ def connect(
|
||||
synchronous: ``PRAGMA synchronous`` level. Defaults to ``FULL`` so
|
||||
committed writes survive an OS/power crash (matches pre-hardening
|
||||
durability). Pass ``"NORMAL"`` for higher throughput under WAL.
|
||||
guard: When True, run a one-time ``quick_check`` on open and, on a
|
||||
malformed file, forensically back it up and attempt a bounded
|
||||
least-destructive online-backup repair before connecting (Issue
|
||||
#5387). Off by default so existing callers are unchanged.
|
||||
repair: Attempt an online-backup repair on corruption (only when
|
||||
``guard`` is set). Bounded by a sidecar attempt-ledger.
|
||||
backup: Take a retention-capped, disk-space-aware forensic backup of the
|
||||
malformed file before repair (only when ``guard`` is set).
|
||||
**kwargs: Forwarded to ``sqlite3.connect``.
|
||||
"""
|
||||
import sqlite3 # lazy import — stdlib, no heavy dependency
|
||||
@@ -185,6 +533,12 @@ def connect(
|
||||
if isinstance(path, (str, os.PathLike)) and str(path) != ":memory:":
|
||||
db_path = str(path)
|
||||
|
||||
if guard:
|
||||
try:
|
||||
_guard_integrity(str(db_path), repair=repair, backup=backup)
|
||||
except Exception as exc: # detection/repair must never block open
|
||||
logger.debug("Integrity guard skipped for %s: %s", db_path, exc)
|
||||
|
||||
connect_kwargs = dict(kwargs)
|
||||
connect_kwargs["check_same_thread"] = check_same_thread
|
||||
if isolation_level != "":
|
||||
|
||||
@@ -114,6 +114,27 @@ from praisonaiagents.tools import internet_search
|
||||
results = internet_search("AI news")
|
||||
```
|
||||
|
||||
### Parallel Search MCP
|
||||
|
||||
Install `praisonaiagents[parallel-search]` to use Parallel's free Search MCP
|
||||
without an API key. Select it for one call with `providers="parallel"`, or set
|
||||
`WEB_SEARCH_PROVIDER=parallel` for the existing provider-selection path:
|
||||
|
||||
```python
|
||||
from praisonaiagents.tools.web_search import search_web
|
||||
|
||||
results = search_web("recent agent framework releases", providers="parallel")
|
||||
```
|
||||
|
||||
Parallel is opt-in and does not change the automatic provider order. When you
|
||||
select it, your search query is sent to Parallel Search MCP for processing.
|
||||
Requests include a `praisonaiagents/<installed-version>` User-Agent so Parallel
|
||||
can measure aggregate project usage. This identifier is shared across
|
||||
installations and contains no user or installation ID. Anonymous access is
|
||||
intended for exploration and light use; see the
|
||||
[Search MCP documentation](https://docs.parallel.ai/integrations/mcp/search-mcp)
|
||||
for access limits.
|
||||
|
||||
### 2. Class-Based Approach (Complex Tools)
|
||||
|
||||
Best for tools that do multiple related things or need to remember information. Like a smart calculator that remembers your previous calculations and can do many different math operations.
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
This module provides a single `search_web` function that automatically tries
|
||||
multiple search providers in order, falling back to the next if one fails.
|
||||
|
||||
Search Provider Priority:
|
||||
Default Search Provider Priority:
|
||||
1. Tavily (requires TAVILY_API_KEY + tavily-python)
|
||||
2. Brave (requires BRAVE_API_KEY + requests)
|
||||
3. Exa (requires EXA_API_KEY + exa_py)
|
||||
@@ -11,6 +11,8 @@ Search Provider Priority:
|
||||
5. DuckDuckGo (requires ddgs package, no API key)
|
||||
6. SearxNG (requires requests + running SearxNG instance)
|
||||
|
||||
Parallel Search MCP is available when explicitly selected.
|
||||
|
||||
Usage:
|
||||
from praisonaiagents.tools import search_web
|
||||
|
||||
@@ -24,10 +26,19 @@ Usage:
|
||||
from typing import List, Dict, Any, Optional
|
||||
from praisonaiagents._logging import get_logger
|
||||
import os
|
||||
import json
|
||||
from importlib import util
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
PARALLEL_SEARCH_MCP_URL = "https://search.parallel.ai/mcp"
|
||||
PARALLEL_SEARCH_TIMEOUT_SECONDS = 30
|
||||
PARALLEL_MAX_RESPONSE_BYTES = 1_000_000
|
||||
PARALLEL_MAX_RESULTS = 10
|
||||
PARALLEL_MAX_TITLE_CHARS = 256
|
||||
PARALLEL_MAX_URL_CHARS = 2048
|
||||
PARALLEL_MAX_SNIPPET_CHARS = 1200
|
||||
|
||||
def _check_tavily() -> tuple[bool, Optional[str]]:
|
||||
"""Check if Tavily is available."""
|
||||
if util.find_spec("tavily") is None:
|
||||
@@ -72,6 +83,94 @@ def _check_searxng() -> tuple[bool, Optional[str]]:
|
||||
return False, "requests package not installed"
|
||||
return True, None
|
||||
|
||||
def _check_parallel() -> tuple[bool, Optional[str]]:
|
||||
"""Check if compatible Streamable HTTP MCP support is installed."""
|
||||
try:
|
||||
from praisonaiagents.mcp.mcp_http_stream import streamablehttp_client
|
||||
except ImportError:
|
||||
streamablehttp_client = None
|
||||
if streamablehttp_client is None:
|
||||
return False, "install praisonaiagents[parallel-search] for Streamable HTTP MCP support"
|
||||
from inspect import signature
|
||||
if "httpx_client_factory" not in signature(streamablehttp_client).parameters:
|
||||
return False, "install praisonaiagents[parallel-search] for bounded Streamable HTTP support"
|
||||
return True, None
|
||||
|
||||
def _truncate_parallel_text(value: Any, limit: int) -> str:
|
||||
text = str(value or "").strip()
|
||||
if len(text) > limit:
|
||||
return text[:limit - 1] + "…"
|
||||
return text
|
||||
|
||||
def _search_parallel(query: str, max_results: int = 5) -> List[Dict[str, Any]]:
|
||||
"""Search with the keyless Parallel Search MCP endpoint."""
|
||||
result_limit = max(0, min(int(max_results), PARALLEL_MAX_RESULTS))
|
||||
if result_limit == 0:
|
||||
return []
|
||||
|
||||
from importlib.metadata import PackageNotFoundError, version
|
||||
from praisonaiagents.mcp import MCP
|
||||
|
||||
# Keep this project-level so attribution aggregates across installations.
|
||||
# Fall back gracefully when dist metadata is unavailable (source checkouts,
|
||||
# vendored/frozen installs) instead of crashing the whole search path.
|
||||
try:
|
||||
package_version = version("praisonaiagents")
|
||||
except PackageNotFoundError:
|
||||
package_version = "unknown"
|
||||
headers = {"User-Agent": f"praisonaiagents/{package_version}"}
|
||||
with MCP(
|
||||
PARALLEL_SEARCH_MCP_URL,
|
||||
timeout=PARALLEL_SEARCH_TIMEOUT_SECONDS,
|
||||
allowed_tools=["web_search"],
|
||||
headers=headers,
|
||||
max_response_bytes=PARALLEL_MAX_RESPONSE_BYTES,
|
||||
) as client:
|
||||
search_tool = next(
|
||||
(tool for tool in client.get_tools() if getattr(tool, "__name__", None) == "web_search"),
|
||||
None,
|
||||
)
|
||||
if search_tool is None:
|
||||
raise RuntimeError("Parallel Search MCP did not expose web_search")
|
||||
response = search_tool(objective=query, search_queries=[query])
|
||||
|
||||
if not isinstance(response, str):
|
||||
raise ValueError("Parallel Search MCP returned a non-text response")
|
||||
if response.startswith("Error:"):
|
||||
raise RuntimeError(response)
|
||||
payload = json.loads(response)
|
||||
if not isinstance(payload, dict) or "results" not in payload:
|
||||
raise ValueError("Parallel Search MCP returned an invalid response shape")
|
||||
raw_results = payload["results"]
|
||||
if not isinstance(raw_results, list):
|
||||
raise ValueError("Parallel Search MCP returned an invalid results list")
|
||||
|
||||
results = []
|
||||
for result in raw_results:
|
||||
if not isinstance(result, dict):
|
||||
continue
|
||||
url = result.get("url")
|
||||
if not isinstance(url, str) or not url or len(url) > PARALLEL_MAX_URL_CHARS:
|
||||
continue
|
||||
excerpts = result.get("excerpts", [])
|
||||
if isinstance(excerpts, str):
|
||||
excerpts = [excerpts]
|
||||
if not isinstance(excerpts, list):
|
||||
excerpts = []
|
||||
snippet = " ".join(
|
||||
excerpt.strip() for excerpt in excerpts
|
||||
if isinstance(excerpt, str) and excerpt.strip()
|
||||
)
|
||||
results.append({
|
||||
"title": _truncate_parallel_text(result.get("title", ""), PARALLEL_MAX_TITLE_CHARS),
|
||||
"url": url,
|
||||
"snippet": _truncate_parallel_text(snippet, PARALLEL_MAX_SNIPPET_CHARS),
|
||||
"provider": "parallel",
|
||||
})
|
||||
if len(results) >= result_limit:
|
||||
break
|
||||
return results
|
||||
|
||||
def _search_tavily(query: str, max_results: int = 5, search_depth: str = "basic", raw_content: bool = False) -> List[Dict[str, Any]]:
|
||||
"""Search using Tavily."""
|
||||
from tavily import TavilyClient
|
||||
@@ -272,6 +371,10 @@ SEARCH_PROVIDERS = [
|
||||
("searxng", _check_searxng, _search_searxng),
|
||||
]
|
||||
|
||||
SELECTABLE_SEARCH_PROVIDERS = SEARCH_PROVIDERS + [
|
||||
("parallel", _check_parallel, _search_parallel),
|
||||
]
|
||||
|
||||
def search_web(
|
||||
query: str,
|
||||
max_results: int = 5,
|
||||
@@ -284,7 +387,8 @@ def search_web(
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Search the web using one or more providers with automatic fallback.
|
||||
|
||||
VALID PROVIDER NAMES: tavily, brave, exa, youdotcom, duckduckgo, searxng
|
||||
VALID PROVIDER NAMES: tavily, brave, exa, youdotcom, duckduckgo, searxng, parallel
|
||||
Parallel is opt-in and does not change the existing automatic provider order.
|
||||
|
||||
Args:
|
||||
query: Search query string
|
||||
@@ -293,7 +397,8 @@ def search_web(
|
||||
- Single string: "tavily" (uses just that provider)
|
||||
- Comma-separated: "tavily, brave" (tries in order, falls back on failure)
|
||||
- List: ["tavily", "brave"] (same as comma-separated)
|
||||
If not specified, tries all providers in order until one succeeds.
|
||||
If not specified, tries the default providers in order until one succeeds.
|
||||
Select "parallel" to use the keyless Parallel Search MCP provider.
|
||||
|
||||
Provider-specific options (only used by their respective provider):
|
||||
- searxng_url: [SearxNG] Custom instance URL
|
||||
@@ -308,7 +413,7 @@ def search_web(
|
||||
errors = []
|
||||
|
||||
# Valid provider names for reference
|
||||
valid_provider_names = [p[0] for p in SEARCH_PROVIDERS]
|
||||
valid_provider_names = [p[0] for p in SELECTABLE_SEARCH_PROVIDERS]
|
||||
|
||||
# Check for WEB_SEARCH_PROVIDER env var (set by CLI --web-provider flag)
|
||||
if providers is None:
|
||||
@@ -328,7 +433,7 @@ def search_web(
|
||||
for name in providers:
|
||||
name_lower = name.lower().strip()
|
||||
matched = False
|
||||
for pname, check_fn, search_fn in SEARCH_PROVIDERS:
|
||||
for pname, check_fn, search_fn in SELECTABLE_SEARCH_PROVIDERS:
|
||||
if pname == name_lower:
|
||||
provider_order.append((pname, check_fn, search_fn))
|
||||
matched = True
|
||||
@@ -398,7 +503,7 @@ def get_available_providers() -> List[Dict[str, Any]]:
|
||||
print(f"{p['name']}: {'✓' if p['available'] else '✗'} {p.get('reason', '')}")
|
||||
"""
|
||||
result = []
|
||||
for name, check_func, _ in SEARCH_PROVIDERS:
|
||||
for name, check_func, _ in SELECTABLE_SEARCH_PROVIDERS:
|
||||
is_available, error = check_func()
|
||||
result.append({
|
||||
"name": name,
|
||||
|
||||
@@ -7,48 +7,78 @@ ensuring no RuntimeError: "This event loop is already running" crashes.
|
||||
|
||||
import asyncio
|
||||
import concurrent.futures
|
||||
import contextvars
|
||||
import logging
|
||||
from typing import Any, Awaitable, TypeVar
|
||||
from typing import Any, Awaitable, Optional, TypeVar
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
T = TypeVar('T')
|
||||
|
||||
def run_coroutine_from_any_context(coro: Awaitable[T], timeout: float = 300) -> T:
|
||||
def run_coroutine_from_any_context(coro: Awaitable[T], timeout: Optional[float] = 300) -> T:
|
||||
"""
|
||||
Safely run a coroutine from sync context only.
|
||||
|
||||
Safely run a coroutine to completion from any context (sync OR inside a
|
||||
running event loop).
|
||||
|
||||
- No running loop: run it directly with ``asyncio.run`` (wrapped in
|
||||
``asyncio.wait_for`` when a timeout is set).
|
||||
- Running loop (FastAPI, Jupyter, a bot handler calling ``agent.chat()``):
|
||||
run it on a dedicated worker thread with its own loop so we never
|
||||
deadlock on the caller's loop. The caller's ``contextvars`` are copied
|
||||
into that thread so trace/session/approval context set by the caller is
|
||||
visible inside the coroutine, and the timeout is enforced *inside* the
|
||||
worker loop via ``asyncio.wait_for``. The caller also waits on the worker
|
||||
with the same ``timeout`` as a hard backstop, so a coroutine that ignores
|
||||
cancellation (e.g. a blocking C call that never yields) can still not pin
|
||||
the caller past ``timeout``.
|
||||
|
||||
Args:
|
||||
coro: The coroutine to execute
|
||||
timeout: Maximum execution time in seconds (default: 5 minutes)
|
||||
|
||||
timeout: Maximum execution time in seconds (default: 5 minutes).
|
||||
``None`` disables the timeout.
|
||||
|
||||
Returns:
|
||||
The result of the coroutine
|
||||
|
||||
|
||||
Raises:
|
||||
RuntimeError: If called from async context (event loop is running)
|
||||
TimeoutError: If the coroutine takes longer than timeout
|
||||
|
||||
TimeoutError: If the coroutine takes longer than ``timeout``
|
||||
|
||||
Examples:
|
||||
>>> async def my_async_function():
|
||||
... return "hello"
|
||||
>>> result = run_coroutine_from_any_context(my_async_function())
|
||||
>>> print(result) # "hello"
|
||||
"""
|
||||
def _wrapped() -> Awaitable[T]:
|
||||
return asyncio.wait_for(coro, timeout) if timeout is not None else coro
|
||||
|
||||
try:
|
||||
loop = asyncio.get_running_loop()
|
||||
# If we get here, event loop is running - this is not allowed
|
||||
raise RuntimeError(
|
||||
"run_coroutine_from_any_context() cannot be called from async context. "
|
||||
"Use 'await' instead when calling from async functions."
|
||||
)
|
||||
except RuntimeError as e:
|
||||
if "no running event loop" in str(e).lower():
|
||||
# No event loop — safe to use asyncio.run()
|
||||
return asyncio.run(coro)
|
||||
else:
|
||||
# Re-raise our own RuntimeError
|
||||
raise
|
||||
asyncio.get_running_loop()
|
||||
except RuntimeError:
|
||||
# No running event loop — safe to drive it directly on this thread.
|
||||
return asyncio.run(_wrapped())
|
||||
|
||||
# A loop is already running on this thread. Offload to a worker thread with
|
||||
# its own loop, carrying the caller's contextvars across so context set by
|
||||
# the caller (trace/session/approval) is visible inside the coroutine.
|
||||
ctx = contextvars.copy_context()
|
||||
executor = concurrent.futures.ThreadPoolExecutor(max_workers=1)
|
||||
future = executor.submit(ctx.run, asyncio.run, _wrapped())
|
||||
try:
|
||||
# The timeout is normally enforced *inside* the worker loop by
|
||||
# asyncio.wait_for. But a coroutine that ignores cancellation (a blocking
|
||||
# C call, a shielded/uncancellable await) can keep the worker alive past
|
||||
# the deadline, in which case that inner wait_for never fires. We
|
||||
# therefore also bound the caller's wait here so a stuck worker can never
|
||||
# pin the caller indefinitely.
|
||||
return future.result(timeout=timeout)
|
||||
except concurrent.futures.TimeoutError as exc:
|
||||
raise TimeoutError(
|
||||
f"Coroutine did not complete within {timeout}s"
|
||||
) from exc
|
||||
finally:
|
||||
# Never block on a stuck worker: don't wait for it to wind down.
|
||||
executor.shutdown(wait=False)
|
||||
|
||||
def is_async_context() -> bool:
|
||||
"""
|
||||
|
||||
@@ -617,6 +617,14 @@ class YAMLWorkflowParser:
|
||||
if _templates:
|
||||
agent_kwargs['templates'] = _templates
|
||||
agent = Agent(**agent_kwargs)
|
||||
# Preserve legacy YAML reflection fields for callers/tests that still
|
||||
# read the pre-consolidation attribute names.
|
||||
if reflect_llm is not None:
|
||||
agent._yaml_reflect_llm = reflect_llm
|
||||
if min_reflect is not None:
|
||||
agent._yaml_min_reflect = min_reflect
|
||||
if max_reflect is not None:
|
||||
agent._yaml_max_reflect = max_reflect
|
||||
|
||||
# Wire YAML max_rpm into a live RateLimiter when none is already set,
|
||||
# so `max_rpm` in YAML actually throttles requests. (Applies to
|
||||
|
||||
@@ -44,6 +44,10 @@ mcp = [
|
||||
"uvicorn>=0.34.0",
|
||||
"websockets>=12.0"
|
||||
]
|
||||
parallel-search = [
|
||||
"mcp>=1.30.0,<2",
|
||||
]
|
||||
|
||||
|
||||
memory = [
|
||||
"chromadb>=1.0.0",
|
||||
|
||||
@@ -119,6 +119,54 @@ class TestShortTermMemory:
|
||||
assert items[0].content == "Persistent memory"
|
||||
|
||||
|
||||
class TestAtomicWriteCleanup:
|
||||
"""Tests for atomic write staging-file cleanup (regression for #5491)."""
|
||||
|
||||
def test_serialization_error_leaves_no_temp_file(self):
|
||||
"""A non-serializable value must not leave a .*.tmp staging file behind."""
|
||||
from pathlib import Path
|
||||
from praisonaiagents.memory.file_memory import FileMemory
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
memory = FileMemory(base_path=tmpdir)
|
||||
target = memory.short_term_file
|
||||
|
||||
with pytest.raises(TypeError):
|
||||
memory._write_json(target, {"bad": object()})
|
||||
|
||||
leftovers = list(Path(target.parent).glob(f".{target.name}.*.tmp"))
|
||||
assert leftovers == [], f"leftover temp files: {leftovers}"
|
||||
|
||||
def test_serialization_error_preserves_existing_file(self):
|
||||
"""A failed write must not clobber the previously persisted file."""
|
||||
from praisonaiagents.memory.file_memory import FileMemory
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
memory = FileMemory(base_path=tmpdir)
|
||||
target = memory.short_term_file
|
||||
|
||||
assert memory._write_json(target, [{"content": "keep me"}]) is True
|
||||
original = target.read_text(encoding="utf-8")
|
||||
|
||||
with pytest.raises(TypeError):
|
||||
memory._write_json(target, {"bad": object()})
|
||||
|
||||
assert target.read_text(encoding="utf-8") == original
|
||||
|
||||
def test_successful_write_removes_temp_file(self):
|
||||
"""A successful atomic write must leave no staging file behind."""
|
||||
from pathlib import Path
|
||||
from praisonaiagents.memory.file_memory import FileMemory
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
memory = FileMemory(base_path=tmpdir)
|
||||
target = memory.short_term_file
|
||||
|
||||
assert memory._write_json(target, [{"content": "ok"}]) is True
|
||||
leftovers = list(Path(target.parent).glob(f".{target.name}.*.tmp"))
|
||||
assert leftovers == [], f"leftover temp files: {leftovers}"
|
||||
|
||||
|
||||
class TestLongTermMemory:
|
||||
"""Tests for long-term memory functionality."""
|
||||
|
||||
|
||||
@@ -0,0 +1,140 @@
|
||||
"""Regression tests for issue #5596.
|
||||
|
||||
A guardrail that blocks model output must never make Agent.chat/achat return
|
||||
``None`` (callers expect a ``str`` and crash on ``.strip()``). On a block the
|
||||
agent returns a safe, non-empty fallback string instead of leaking the rejected
|
||||
text.
|
||||
"""
|
||||
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from praisonaiagents import Agent, TaskOutput
|
||||
|
||||
|
||||
def _blocking_guardrail(task_output: TaskOutput):
|
||||
"""Reject every output, mimicking a forbidden-token guardrail."""
|
||||
return False, "contains forbidden token"
|
||||
|
||||
|
||||
def test_guardrail_blocked_message_is_safe_non_empty_string():
|
||||
agent = Agent(
|
||||
name="GuardAgent",
|
||||
instructions="You are a test agent",
|
||||
llm="gpt-4o-mini",
|
||||
guardrails=_blocking_guardrail,
|
||||
)
|
||||
message = agent._guardrail_blocked_message("contains forbidden token")
|
||||
assert isinstance(message, str)
|
||||
assert message.strip()
|
||||
# Fixed safe string (FR-001): the guardrail reason must NOT leak to the
|
||||
# caller, since an LLM validator's reason can echo the rejected output.
|
||||
assert "contains forbidden token" not in message
|
||||
|
||||
|
||||
def test_guardrail_blocked_message_without_reason():
|
||||
agent = Agent(
|
||||
name="GuardAgent",
|
||||
instructions="You are a test agent",
|
||||
llm="gpt-4o-mini",
|
||||
guardrails=_blocking_guardrail,
|
||||
)
|
||||
message = agent._guardrail_blocked_message()
|
||||
assert isinstance(message, str)
|
||||
assert message.strip()
|
||||
|
||||
|
||||
def test_chat_returns_safe_message_not_none_on_block():
|
||||
agent = Agent(
|
||||
name="GuardAgent",
|
||||
instructions="You are a test agent",
|
||||
llm="gpt-4o-mini",
|
||||
guardrails=_blocking_guardrail,
|
||||
)
|
||||
# retries=0 so the retry loop raises immediately without a network call.
|
||||
agent.max_guardrail_retries = 0
|
||||
# Force the custom-LLM branch and feed it forbidden output offline.
|
||||
agent._using_custom_llm = True
|
||||
agent.llm_instance = MagicMock()
|
||||
agent.llm_instance.get_response.return_value = "LEAKME secret"
|
||||
|
||||
result = agent.chat("say something")
|
||||
|
||||
assert result is not None
|
||||
assert isinstance(result, str)
|
||||
assert result.strip()
|
||||
assert "LEAKME" not in result
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_achat_returns_safe_message_not_none_on_block():
|
||||
agent = Agent(
|
||||
name="GuardAgent",
|
||||
instructions="You are a test agent",
|
||||
llm="gpt-4o-mini",
|
||||
guardrails=_blocking_guardrail,
|
||||
)
|
||||
agent.max_guardrail_retries = 0
|
||||
|
||||
async def _fake_get_response_async(**kwargs):
|
||||
return "LEAKME secret"
|
||||
|
||||
agent._using_custom_llm = True
|
||||
agent.llm_instance = MagicMock()
|
||||
agent.llm_instance.get_response_async = _fake_get_response_async
|
||||
|
||||
result = await agent.achat("say something")
|
||||
|
||||
assert result is not None
|
||||
assert isinstance(result, str)
|
||||
assert result.strip()
|
||||
assert "LEAKME" not in result
|
||||
|
||||
|
||||
def test_chat_does_not_leak_guardrail_reason():
|
||||
"""The fixed fallback must not echo the guardrail's rejection reason.
|
||||
|
||||
An LLM validator's reason can quote the blocked output, so the reason is
|
||||
logged (and exposed via ``last_guardrail_error``) but never returned.
|
||||
"""
|
||||
agent = Agent(
|
||||
name="GuardAgent",
|
||||
instructions="You are a test agent",
|
||||
llm="gpt-4o-mini",
|
||||
guardrails=_blocking_guardrail,
|
||||
)
|
||||
agent.max_guardrail_retries = 0
|
||||
agent._using_custom_llm = True
|
||||
agent.llm_instance = MagicMock()
|
||||
agent.llm_instance.get_response.return_value = "LEAKME secret"
|
||||
|
||||
result = agent.chat("say something")
|
||||
|
||||
assert isinstance(result, str) and result.strip()
|
||||
assert "LEAKME" not in result
|
||||
assert "contains forbidden token" not in result
|
||||
|
||||
|
||||
def test_chat_returns_safe_message_with_reflection_enabled():
|
||||
"""A guardrail block with self-reflection on still yields a safe string."""
|
||||
from praisonaiagents import ReflectionConfig
|
||||
|
||||
agent = Agent(
|
||||
name="GuardAgent",
|
||||
instructions="You are a test agent",
|
||||
llm="gpt-4o-mini",
|
||||
guardrails=_blocking_guardrail,
|
||||
reflection=ReflectionConfig(min_iterations=1, max_iterations=1),
|
||||
)
|
||||
agent.max_guardrail_retries = 0
|
||||
agent._using_custom_llm = True
|
||||
agent.llm_instance = MagicMock()
|
||||
agent.llm_instance.get_response.return_value = "LEAKME secret"
|
||||
|
||||
result = agent.chat("say something")
|
||||
|
||||
assert result is not None
|
||||
assert isinstance(result, str)
|
||||
assert result.strip()
|
||||
assert "LEAKME" not in result
|
||||
@@ -0,0 +1,136 @@
|
||||
"""
|
||||
Regression tests for cross-session memory recall (issue #5595).
|
||||
|
||||
Two Agent instances sharing the same memory store (e.g. the same user_id)
|
||||
must be able to recall facts written during an earlier turn. Previously the
|
||||
synchronous chat/run/start path never wrote the interaction to the memory
|
||||
store, so the second agent recalled nothing.
|
||||
|
||||
These tests drive the after-agent side-effect pipeline directly so they do
|
||||
not require an LLM or OPENAI_API_KEY.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent))
|
||||
|
||||
from praisonaiagents import Agent
|
||||
from praisonaiagents.memory import FileMemory
|
||||
|
||||
|
||||
class TestCrossSessionMemoryRecall(unittest.TestCase):
|
||||
"""MEM-001/002/003 from issue #5595."""
|
||||
|
||||
def _agent(self, user_id, base_path):
|
||||
memory = FileMemory(user_id=user_id, base_path=base_path)
|
||||
return Agent(
|
||||
name="Assistant",
|
||||
instructions="You are a helpful assistant.",
|
||||
memory=memory,
|
||||
)
|
||||
|
||||
def test_turn_persisted_and_recalled_across_instances(self):
|
||||
"""MEM-001: a fact stored in session A is recalled by session B."""
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
base = f"{tmpdir}/memory"
|
||||
|
||||
# Session A: simulate a completed turn storing a codename.
|
||||
agent_a = self._agent("e2e-memory-test", base)
|
||||
agent_a._after_agent_side_effects(
|
||||
"My codename is ORANGE-PANDA. ACK.",
|
||||
"Acknowledged. Your codename is ORANGE-PANDA.",
|
||||
)
|
||||
|
||||
# Session B: a fresh agent sharing the same user_id must recall it.
|
||||
agent_b = self._agent("e2e-memory-test", base)
|
||||
context = agent_b.get_memory_context(query="What is my codename?")
|
||||
self.assertIn("ORANGE-PANDA", context)
|
||||
|
||||
def test_different_user_id_is_isolated(self):
|
||||
"""MEM-002: a different user_id must not see another user's memory."""
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
base = f"{tmpdir}/memory"
|
||||
|
||||
agent_a = self._agent("user-a", base)
|
||||
agent_a._after_agent_side_effects(
|
||||
"My codename is ORANGE-PANDA.",
|
||||
"Acknowledged. Your codename is ORANGE-PANDA.",
|
||||
)
|
||||
|
||||
agent_b = self._agent("user-b", base)
|
||||
context = agent_b.get_memory_context(query="What is my codename?")
|
||||
self.assertNotIn("ORANGE-PANDA", context)
|
||||
|
||||
def test_empty_response_is_not_persisted(self):
|
||||
"""A turn with no response must not write an empty entry."""
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
base = f"{tmpdir}/memory"
|
||||
agent = self._agent("user-empty", base)
|
||||
agent._after_agent_side_effects("hello", "")
|
||||
self.assertEqual(
|
||||
agent._memory_instance.get_stats()["short_term_count"], 0
|
||||
)
|
||||
|
||||
def test_dict_memory_config_persists_and_recalls(self):
|
||||
"""MEM-004: the advertised ``memory={'user_id': uid}`` dict config —
|
||||
resolved by the Agent itself, not a preconstructed FileMemory — persists
|
||||
a turn that a second Agent with the same user_id recalls.
|
||||
|
||||
Guards the configuration-resolution + hook wiring that the direct-call
|
||||
tests above cannot (Greptile #3). Runs in a temp cwd so the default
|
||||
FileMemory base path is hermetic; no LLM/API key needed.
|
||||
"""
|
||||
cwd = os.getcwd()
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
os.chdir(tmpdir)
|
||||
try:
|
||||
uid = "dict-config-user"
|
||||
agent_a = Agent(name="A", instructions="x", memory={"user_id": uid})
|
||||
self.assertIsInstance(agent_a._memory_instance, FileMemory)
|
||||
agent_a._after_agent_side_effects(
|
||||
"My codename is BLUE-FOX.",
|
||||
"Acknowledged. Your codename is BLUE-FOX.",
|
||||
)
|
||||
|
||||
agent_b = Agent(name="B", instructions="x", memory={"user_id": uid})
|
||||
context = agent_b.get_memory_context(query="What is my codename?")
|
||||
self.assertIn("BLUE-FOX", context)
|
||||
finally:
|
||||
os.chdir(cwd)
|
||||
|
||||
def test_async_path_persists_without_blocking_loop(self):
|
||||
"""MEM-005: on the async after-agent path the blocking store write is
|
||||
offloaded to a thread (AGENTS.md §4.5), yet the turn is still persisted
|
||||
and recalled by a later Agent. Exercises the running-loop branch of
|
||||
``_persist_memory_turn``.
|
||||
"""
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
base = f"{tmpdir}/memory"
|
||||
|
||||
async def write_turn():
|
||||
agent_a = self._agent("async-user", base)
|
||||
agent_a._after_agent_side_effects(
|
||||
"My codename is GREEN-HAWK.",
|
||||
"Acknowledged. Your codename is GREEN-HAWK.",
|
||||
)
|
||||
# The write is scheduled via run_in_executor; yield until the
|
||||
# short-term store reflects it (bounded) rather than racing it.
|
||||
for _ in range(500):
|
||||
if agent_a._memory_instance.get_stats()["short_term_count"] > 0:
|
||||
break
|
||||
await asyncio.sleep(0.01)
|
||||
|
||||
asyncio.run(write_turn())
|
||||
|
||||
agent_b = self._agent("async-user", base)
|
||||
context = agent_b.get_memory_context(query="What is my codename?")
|
||||
self.assertIn("GREEN-HAWK", context)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,441 @@
|
||||
import importlib
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
from importlib.metadata import version
|
||||
import json
|
||||
from threading import Thread
|
||||
|
||||
import pytest
|
||||
|
||||
pytest.importorskip("mcp.client.streamable_http")
|
||||
|
||||
web_search_module = importlib.import_module("praisonaiagents.tools.web_search")
|
||||
|
||||
|
||||
class MCPHandler(BaseHTTPRequestHandler):
|
||||
protocol_version = "HTTP/1.1"
|
||||
|
||||
def do_POST(self):
|
||||
request_bytes = self.rfile.read(int(self.headers.get("Content-Length", "0")))
|
||||
request = json.loads(request_bytes)
|
||||
self.server.requests.append(request)
|
||||
self.server.header_snapshots.append({
|
||||
"user_agent": self.headers.get("User-Agent"),
|
||||
"accept": self.headers.get("Accept"),
|
||||
"content_type": self.headers.get("Content-Type"),
|
||||
"accept_encoding": self.headers.get("Accept-Encoding"),
|
||||
"authorization_present": self.headers.get("Authorization") is not None,
|
||||
})
|
||||
method = request.get("method")
|
||||
request_id = request.get("id")
|
||||
|
||||
if method == "notifications/initialized":
|
||||
self.send_response(202)
|
||||
self.send_header("Content-Length", "0")
|
||||
self.end_headers()
|
||||
return
|
||||
if method == "initialize":
|
||||
self._send_json({
|
||||
"jsonrpc": "2.0",
|
||||
"id": request_id,
|
||||
"result": {
|
||||
"protocolVersion": request["params"]["protocolVersion"],
|
||||
"capabilities": {"tools": {}},
|
||||
"serverInfo": {"name": "test-search", "version": "1.0"},
|
||||
},
|
||||
})
|
||||
return
|
||||
if method == "tools/list":
|
||||
description = "x" * 2000 if self.server.oversized_tools_list else "Search the web"
|
||||
self._send_json({
|
||||
"jsonrpc": "2.0",
|
||||
"id": request_id,
|
||||
"result": {
|
||||
"tools": [
|
||||
{
|
||||
"name": "web_search",
|
||||
"description": description,
|
||||
"inputSchema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"objective": {"type": "string"},
|
||||
"search_queries": {
|
||||
"type": "array",
|
||||
"items": {"type": "string"},
|
||||
},
|
||||
},
|
||||
"required": ["objective", "search_queries"],
|
||||
},
|
||||
},
|
||||
{
|
||||
"name": "web_fetch",
|
||||
"description": "Fetch a page",
|
||||
"inputSchema": {"type": "object", "properties": {}},
|
||||
},
|
||||
],
|
||||
},
|
||||
})
|
||||
return
|
||||
if method == "tools/call":
|
||||
self.server.tool_calls.append(request.get("params", {}))
|
||||
text = json.dumps({
|
||||
"results": [
|
||||
{
|
||||
"url": "https://example.com/one",
|
||||
"title": "First result",
|
||||
"excerpts": ["x" * 2000 if self.server.chunked_tool_call else "Useful evidence from the first result."],
|
||||
},
|
||||
{
|
||||
"url": "https://example.com/two",
|
||||
"title": "Second result",
|
||||
"excerpts": ["A second useful result."],
|
||||
},
|
||||
],
|
||||
})
|
||||
response = {
|
||||
"jsonrpc": "2.0",
|
||||
"id": request_id,
|
||||
"result": {"content": [{"type": "text", "text": text}], "isError": self.server.tool_error},
|
||||
}
|
||||
if self.server.chunked_tool_call:
|
||||
self._send_chunked_json(response)
|
||||
else:
|
||||
self._send_json(response)
|
||||
return
|
||||
|
||||
self._send_json({
|
||||
"jsonrpc": "2.0",
|
||||
"id": request_id,
|
||||
"error": {"code": -32601, "message": "Method not found"},
|
||||
})
|
||||
|
||||
def do_GET(self):
|
||||
self._send_chunked_json({"body": "x" * 2000})
|
||||
|
||||
def do_DELETE(self):
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Length", "0")
|
||||
self.end_headers()
|
||||
|
||||
def _send_json(self, body):
|
||||
response = json.dumps(body).encode()
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", "application/json")
|
||||
self.send_header("Content-Length", str(len(response)))
|
||||
self.end_headers()
|
||||
self.wfile.write(response)
|
||||
|
||||
def _send_chunked_json(self, body):
|
||||
response = json.dumps(body).encode()
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", "application/json")
|
||||
self.send_header("Transfer-Encoding", "chunked")
|
||||
self.end_headers()
|
||||
try:
|
||||
for offset in range(0, len(response), 64):
|
||||
chunk = response[offset:offset + 64]
|
||||
self.wfile.write(f"{len(chunk):x}\r\n".encode() + chunk + b"\r\n")
|
||||
self.wfile.flush()
|
||||
self.wfile.write(b"0\r\n\r\n")
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
def log_message(self, format, *args):
|
||||
return
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mcp_server():
|
||||
server = ThreadingHTTPServer(("127.0.0.1", 0), MCPHandler)
|
||||
server.daemon_threads = True
|
||||
server.requests = []
|
||||
server.tool_calls = []
|
||||
server.header_snapshots = []
|
||||
server.tool_error = False
|
||||
server.chunked_tool_call = False
|
||||
server.oversized_tools_list = False
|
||||
thread = Thread(target=server.serve_forever, daemon=True)
|
||||
thread.start()
|
||||
try:
|
||||
yield server
|
||||
finally:
|
||||
server.shutdown()
|
||||
server.server_close()
|
||||
thread.join(timeout=2)
|
||||
|
||||
|
||||
def _server_url(server):
|
||||
return f"http://127.0.0.1:{server.server_address[1]}/mcp"
|
||||
|
||||
|
||||
def test_parallel_is_explicit_and_does_not_change_default_order(monkeypatch):
|
||||
default_names = [name for name, _, _ in web_search_module.SEARCH_PROVIDERS]
|
||||
selectable_names = [name for name, _, _ in web_search_module.SELECTABLE_SEARCH_PROVIDERS]
|
||||
assert "parallel" not in default_names
|
||||
assert "parallel" in selectable_names
|
||||
|
||||
calls = []
|
||||
|
||||
def check_incumbent():
|
||||
return True, None
|
||||
|
||||
def search_incumbent(query, max_results):
|
||||
calls.append(query)
|
||||
return [{"title": "Existing", "url": "https://example.com", "snippet": "", "provider": "existing"}]
|
||||
|
||||
monkeypatch.setattr(web_search_module, "SEARCH_PROVIDERS", [("existing", check_incumbent, search_incumbent)])
|
||||
monkeypatch.delenv("WEB_SEARCH_PROVIDER", raising=False)
|
||||
result = web_search_module.search_web("default route")
|
||||
|
||||
assert calls == ["default route"]
|
||||
assert result[0]["provider"] == "existing"
|
||||
|
||||
|
||||
def test_parallel_provider_maps_results_and_never_adds_authorization(monkeypatch):
|
||||
mcp_module = importlib.import_module("praisonaiagents.mcp")
|
||||
observed = {}
|
||||
|
||||
class SearchTool:
|
||||
__name__ = "web_search"
|
||||
|
||||
def __call__(self, **arguments):
|
||||
observed["arguments"] = arguments
|
||||
return json.dumps({
|
||||
"results": [
|
||||
{"url": "https://example.com/one", "title": "T" * 300, "excerpts": ["S" * 1500]},
|
||||
{"url": "https://example.com/two", "title": "Second", "excerpts": ["Useful excerpt"]},
|
||||
{"url": "https://example.com/three", "title": "Third", "excerpts": []},
|
||||
],
|
||||
})
|
||||
|
||||
class FakeMCP:
|
||||
def __init__(self, url, **options):
|
||||
observed["url"] = url
|
||||
observed["options"] = options
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc_value, traceback):
|
||||
return False
|
||||
|
||||
def get_tools(self):
|
||||
return [SearchTool()]
|
||||
|
||||
monkeypatch.setattr(mcp_module, "MCP", FakeMCP)
|
||||
monkeypatch.delenv("PARALLEL_API_KEY", raising=False)
|
||||
query = "PraisonAI provider configuration"
|
||||
result = web_search_module.search_web(query, max_results=2, providers="parallel")
|
||||
|
||||
assert observed["url"] == web_search_module.PARALLEL_SEARCH_MCP_URL
|
||||
assert observed["options"]["allowed_tools"] == ["web_search"]
|
||||
assert observed["options"]["timeout"] == web_search_module.PARALLEL_SEARCH_TIMEOUT_SECONDS
|
||||
assert observed["options"]["max_response_bytes"] == web_search_module.PARALLEL_MAX_RESPONSE_BYTES
|
||||
assert observed["options"]["headers"] == {"User-Agent": f"praisonaiagents/{version('praisonaiagents')}"}
|
||||
assert "Authorization" not in observed["options"]["headers"]
|
||||
assert observed["arguments"] == {"objective": query, "search_queries": [query]}
|
||||
assert len(result) == 2
|
||||
assert result[0]["provider"] == "parallel"
|
||||
assert len(result[0]["title"]) == web_search_module.PARALLEL_MAX_TITLE_CHARS
|
||||
assert result[0]["title"].endswith("…")
|
||||
assert len(result[0]["snippet"]) == web_search_module.PARALLEL_MAX_SNIPPET_CHARS
|
||||
assert result[0]["snippet"].endswith("…")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("response", ["null", "[]", "{}", '{"other": []}', '{"results": {}}'])
|
||||
def test_parallel_rejects_malformed_result_shape(monkeypatch, response):
|
||||
mcp_module = importlib.import_module("praisonaiagents.mcp")
|
||||
|
||||
class SearchTool:
|
||||
__name__ = "web_search"
|
||||
|
||||
def __call__(self, **arguments):
|
||||
return response
|
||||
|
||||
class FakeMCP:
|
||||
def __init__(self, url, **options):
|
||||
pass
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc_value, traceback):
|
||||
return False
|
||||
|
||||
def get_tools(self):
|
||||
return [SearchTool()]
|
||||
|
||||
monkeypatch.setattr(mcp_module, "MCP", FakeMCP)
|
||||
|
||||
with pytest.raises(ValueError):
|
||||
web_search_module._search_parallel("search query")
|
||||
|
||||
|
||||
def test_parallel_accepts_valid_empty_results(monkeypatch):
|
||||
mcp_module = importlib.import_module("praisonaiagents.mcp")
|
||||
|
||||
class SearchTool:
|
||||
__name__ = "web_search"
|
||||
|
||||
def __call__(self, **arguments):
|
||||
return '{"results": []}'
|
||||
|
||||
class FakeMCP:
|
||||
def __init__(self, url, **options):
|
||||
pass
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc_value, traceback):
|
||||
return False
|
||||
|
||||
def get_tools(self):
|
||||
return [SearchTool()]
|
||||
|
||||
monkeypatch.setattr(mcp_module, "MCP", FakeMCP)
|
||||
|
||||
assert web_search_module._search_parallel("search query") == []
|
||||
|
||||
|
||||
def test_parallel_uses_real_mcp_caller_and_preserves_request_headers(mcp_server, monkeypatch):
|
||||
monkeypatch.setattr(web_search_module, "PARALLEL_SEARCH_MCP_URL", _server_url(mcp_server))
|
||||
monkeypatch.setenv("WEB_SEARCH_PROVIDER", "parallel")
|
||||
monkeypatch.delenv("PARALLEL_API_KEY", raising=False)
|
||||
monkeypatch.setenv("NO_PROXY", "127.0.0.1,localhost")
|
||||
query = "PraisonAI web search provider"
|
||||
|
||||
result = web_search_module.search_web(query, max_results=1)
|
||||
|
||||
assert result == [{
|
||||
"title": "First result",
|
||||
"url": "https://example.com/one",
|
||||
"snippet": "Useful evidence from the first result.",
|
||||
"provider": "parallel",
|
||||
}]
|
||||
assert len(mcp_server.tool_calls) == 1
|
||||
assert mcp_server.tool_calls[0]["name"] == "web_search"
|
||||
assert mcp_server.tool_calls[0]["arguments"] == {"objective": query, "search_queries": [query]}
|
||||
assert mcp_server.header_snapshots
|
||||
assert all(snapshot["user_agent"] == f"praisonaiagents/{version('praisonaiagents')}" for snapshot in mcp_server.header_snapshots)
|
||||
assert all("application/json" in snapshot["accept"] and "text/event-stream" in snapshot["accept"] for snapshot in mcp_server.header_snapshots)
|
||||
assert all(snapshot["content_type"] == "application/json" for snapshot in mcp_server.header_snapshots)
|
||||
assert all(snapshot["accept_encoding"] == "identity" for snapshot in mcp_server.header_snapshots)
|
||||
assert all(not snapshot["authorization_present"] for snapshot in mcp_server.header_snapshots)
|
||||
|
||||
|
||||
def test_parallel_bounded_mcp_client_routes_through_http_proxy(mcp_server, monkeypatch):
|
||||
monkeypatch.setattr(web_search_module, "PARALLEL_SEARCH_MCP_URL", "http://parallel.invalid/mcp")
|
||||
proxy_url = f"http://{mcp_server.server_address[0]}:{mcp_server.server_address[1]}"
|
||||
monkeypatch.setenv("HTTP_PROXY", proxy_url)
|
||||
monkeypatch.setenv("NO_PROXY", "")
|
||||
monkeypatch.setenv("no_proxy", "")
|
||||
monkeypatch.delenv("http_proxy", raising=False)
|
||||
monkeypatch.delenv("PARALLEL_API_KEY", raising=False)
|
||||
|
||||
result = web_search_module.search_web("proxy-routed search", providers="parallel", max_results=1)
|
||||
|
||||
assert result == [{
|
||||
"title": "First result",
|
||||
"url": "https://example.com/one",
|
||||
"snippet": "Useful evidence from the first result.",
|
||||
"provider": "parallel",
|
||||
}]
|
||||
assert len(mcp_server.tool_calls) == 1
|
||||
assert mcp_server.tool_calls[0]["name"] == "web_search"
|
||||
|
||||
|
||||
def test_bounded_mcp_client_routes_https_through_environment_proxy(monkeypatch):
|
||||
import asyncio
|
||||
import httpx
|
||||
from praisonaiagents.mcp.mcp_http_stream import _bounded_httpx_client_factory
|
||||
|
||||
class ConnectProxyHandler(BaseHTTPRequestHandler):
|
||||
def do_CONNECT(self):
|
||||
self.server.connect_requests.append(self.path)
|
||||
self.send_response(200, "Connection Established")
|
||||
self.end_headers()
|
||||
self.close_connection = True
|
||||
|
||||
def log_message(self, format, *args):
|
||||
return
|
||||
|
||||
proxy_server = ThreadingHTTPServer(("127.0.0.1", 0), ConnectProxyHandler)
|
||||
proxy_server.daemon_threads = True
|
||||
proxy_server.connect_requests = []
|
||||
proxy_thread = Thread(target=proxy_server.serve_forever, daemon=True)
|
||||
proxy_thread.start()
|
||||
proxy_url = f"http://127.0.0.1:{proxy_server.server_address[1]}"
|
||||
monkeypatch.setenv("HTTPS_PROXY", proxy_url)
|
||||
monkeypatch.setenv("NO_PROXY", "")
|
||||
monkeypatch.setenv("no_proxy", "")
|
||||
monkeypatch.delenv("https_proxy", raising=False)
|
||||
monkeypatch.delenv("ALL_PROXY", raising=False)
|
||||
monkeypatch.delenv("all_proxy", raising=False)
|
||||
|
||||
async def request_through_proxy():
|
||||
factory = _bounded_httpx_client_factory(512)
|
||||
async with factory(timeout=httpx.Timeout(2)) as client:
|
||||
with pytest.raises(httpx.TransportError):
|
||||
await client.get("https://parallel.invalid/mcp")
|
||||
|
||||
try:
|
||||
asyncio.run(request_through_proxy())
|
||||
finally:
|
||||
proxy_server.shutdown()
|
||||
proxy_server.server_close()
|
||||
proxy_thread.join(timeout=2)
|
||||
|
||||
assert proxy_server.connect_requests == ["parallel.invalid:443"]
|
||||
|
||||
|
||||
def test_parallel_uses_unknown_user_agent_when_distribution_metadata_is_missing(mcp_server, monkeypatch):
|
||||
metadata = importlib.import_module("importlib.metadata")
|
||||
installed_version = metadata.version
|
||||
|
||||
def missing_distribution(distribution):
|
||||
if distribution == "praisonaiagents":
|
||||
raise metadata.PackageNotFoundError(distribution)
|
||||
return installed_version(distribution)
|
||||
|
||||
monkeypatch.setattr(metadata, "version", missing_distribution)
|
||||
monkeypatch.setattr(web_search_module, "PARALLEL_SEARCH_MCP_URL", _server_url(mcp_server))
|
||||
monkeypatch.setenv("WEB_SEARCH_PROVIDER", "parallel")
|
||||
monkeypatch.delenv("PARALLEL_API_KEY", raising=False)
|
||||
monkeypatch.setenv("NO_PROXY", "127.0.0.1,localhost")
|
||||
|
||||
result = web_search_module.search_web("search without package metadata", max_results=1)
|
||||
|
||||
assert result[0]["provider"] == "parallel"
|
||||
assert mcp_server.header_snapshots
|
||||
assert all(snapshot["user_agent"] == "praisonaiagents/unknown" for snapshot in mcp_server.header_snapshots)
|
||||
|
||||
|
||||
def test_parallel_streaming_response_limit_is_enforced(mcp_server, monkeypatch):
|
||||
import asyncio
|
||||
from httpx import Timeout
|
||||
from praisonaiagents.mcp.mcp_http_stream import _bounded_httpx_client_factory
|
||||
|
||||
monkeypatch.setenv("NO_PROXY", "127.0.0.1,localhost")
|
||||
|
||||
async def read_oversized_response():
|
||||
factory = _bounded_httpx_client_factory(512)
|
||||
async with factory(timeout=Timeout(2)) as client:
|
||||
async with client.stream("GET", _server_url(mcp_server)) as response:
|
||||
async for _ in response.aiter_bytes():
|
||||
pass
|
||||
|
||||
with pytest.raises(ValueError, match="maximum of 512 bytes"):
|
||||
asyncio.run(read_oversized_response())
|
||||
|
||||
|
||||
def test_parallel_tool_error_is_not_reported_as_success(mcp_server, monkeypatch):
|
||||
monkeypatch.setattr(web_search_module, "PARALLEL_SEARCH_MCP_URL", _server_url(mcp_server))
|
||||
monkeypatch.setenv("NO_PROXY", "127.0.0.1,localhost")
|
||||
mcp_server.tool_error = True
|
||||
|
||||
result = web_search_module.search_web("failed search", providers="parallel")
|
||||
|
||||
assert len(mcp_server.tool_calls) == 1
|
||||
assert len(result) == 1
|
||||
assert "error" in result[0]
|
||||
assert "MCP tool web_search failed" in result[0]["error"]
|
||||
@@ -270,9 +270,11 @@ class TestAlwaysFailingValidatorIsBounded:
|
||||
except _CallCap as exc:
|
||||
pytest.fail(str(exc))
|
||||
|
||||
# chat() reports guardrail exhaustion by returning None after rolling
|
||||
# back history; the reason is on the agent for the caller to read.
|
||||
assert result is None
|
||||
# chat() reports guardrail exhaustion with a safe, non-empty fallback
|
||||
# string after rolling back history - never None (issue #5596) and
|
||||
# never the rejected answer. The reason is also on the agent to read.
|
||||
assert isinstance(result, str) and result.strip()
|
||||
assert "10001" not in result
|
||||
assert agent.guardrail_retry_count == 2, \
|
||||
f"must retry exactly max_guardrail_retries times, got {agent.guardrail_retry_count}"
|
||||
assert t.calls == 1 + 2, f"1 initial call + 2 retries, got {t.calls}"
|
||||
@@ -300,7 +302,8 @@ class TestAlwaysFailingValidatorIsBounded:
|
||||
def test_max_retries_zero_never_retries(self):
|
||||
agent, t = _agent(["10001", "SW1A 1AA"], rejects_first_answer, max_retries=0)
|
||||
result = agent.chat("what is the postcode?", stream=False)
|
||||
assert result is None
|
||||
assert isinstance(result, str) and result.strip()
|
||||
assert "10001" not in result
|
||||
assert t.calls == 1, f"max_guardrail_retries=0 must not call the model again (got {t.calls})"
|
||||
assert agent.guardrail_retry_count == 0
|
||||
|
||||
|
||||
@@ -0,0 +1,116 @@
|
||||
"""Unit tests for Agent.chat structured-output coercion.
|
||||
|
||||
Covers the fix for issue #5594: ``Agent.chat(output_pydantic=Model)`` must
|
||||
return a validated Pydantic instance (not a raw JSON string) on the live
|
||||
OpenAI/LiteLLM path. These tests exercise the ``_coerce_structured_output``
|
||||
helper directly so no live LLM is required.
|
||||
"""
|
||||
from types import SimpleNamespace
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
from praisonaiagents.agent.chat_mixin import ChatMixin
|
||||
|
||||
|
||||
class Pair(BaseModel):
|
||||
city: str
|
||||
country: str
|
||||
|
||||
|
||||
def _fake_completion(content):
|
||||
"""Build a minimal OpenAI-style response object for mocking."""
|
||||
message = SimpleNamespace(content=content, reasoning_content=None, tool_calls=None)
|
||||
return SimpleNamespace(choices=[SimpleNamespace(message=message)])
|
||||
|
||||
|
||||
def _mixin():
|
||||
m = ChatMixin()
|
||||
m.name = "TestAgent"
|
||||
return m
|
||||
|
||||
|
||||
def test_valid_json_string_returns_model():
|
||||
out = _mixin()._coerce_structured_output(
|
||||
'{"city": "Paris", "country": "France"}', Pair
|
||||
)
|
||||
assert isinstance(out, Pair)
|
||||
assert out.city == "Paris"
|
||||
assert out.country == "France"
|
||||
|
||||
|
||||
def test_markdown_fenced_json_returns_model():
|
||||
fenced = '```json\n{"city": "Paris", "country": "France"}\n```'
|
||||
out = _mixin()._coerce_structured_output(fenced, Pair)
|
||||
assert isinstance(out, Pair)
|
||||
assert out.city == "Paris"
|
||||
|
||||
|
||||
def test_prose_falls_back_to_raw_string():
|
||||
out = _mixin()._coerce_structured_output(
|
||||
"The capital of France is Paris.", Pair
|
||||
)
|
||||
assert isinstance(out, str)
|
||||
assert out == "The capital of France is Paris."
|
||||
|
||||
|
||||
def test_no_schema_returns_input_unchanged():
|
||||
out = _mixin()._coerce_structured_output("hello", None)
|
||||
assert out == "hello"
|
||||
|
||||
|
||||
def test_non_string_input_returns_unchanged():
|
||||
instance = Pair(city="Paris", country="France")
|
||||
out = _mixin()._coerce_structured_output(instance, Pair)
|
||||
assert out is instance
|
||||
|
||||
|
||||
def test_dict_schema_without_model_validate_json_returns_raw():
|
||||
# dict schemas (inline JSON schema) have no model_validate_json; must not crash
|
||||
out = _mixin()._coerce_structured_output('{"a": 1}', {"type": "object"})
|
||||
assert out == '{"a": 1}'
|
||||
|
||||
|
||||
# --- Public-path tests: chat()/achat() wiring and backward-compat guards ---
|
||||
|
||||
|
||||
def _agent(monkeypatch, content):
|
||||
"""Build a real Agent whose LLM call is mocked to return ``content``.
|
||||
|
||||
A placeholder API key lets the lazy OpenAI client construct; the network is
|
||||
never touched because ``_chat_completion`` is mocked.
|
||||
"""
|
||||
monkeypatch.setenv("OPENAI_API_KEY", "not-needed")
|
||||
from praisonaiagents import Agent
|
||||
|
||||
agent = Agent(name="TestAgent", instructions="test")
|
||||
monkeypatch.setattr(agent, "_chat_completion", lambda *a, **k: _fake_completion(content))
|
||||
return agent
|
||||
|
||||
|
||||
def test_chat_output_pydantic_returns_model(monkeypatch):
|
||||
# Public chat() path must return a validated Pydantic instance (issue #5594).
|
||||
agent = _agent(monkeypatch, '{"city": "Paris", "country": "France"}')
|
||||
out = agent.chat("capital of France?", output_pydantic=Pair)
|
||||
assert isinstance(out, Pair)
|
||||
assert out.city == "Paris"
|
||||
|
||||
|
||||
def test_chat_output_json_stays_string(monkeypatch):
|
||||
# Backward compat: output_json must keep returning a raw JSON string so that
|
||||
# downstream task/workflow JSON parsing (json.loads) keeps working.
|
||||
agent = _agent(monkeypatch, '{"city": "Paris", "country": "France"}')
|
||||
out = agent.chat("capital of France?", output_json=Pair)
|
||||
assert isinstance(out, str)
|
||||
assert '"Paris"' in out
|
||||
|
||||
|
||||
def test_after_agent_hook_receives_string_for_pydantic(monkeypatch):
|
||||
# A Pydantic response must not crash command/string hooks: the after-agent
|
||||
# hook payload truncates via response[:500], which requires a string.
|
||||
agent = _agent(monkeypatch, '{"city": "Paris", "country": "France"}')
|
||||
payload = agent._build_after_agent_input(
|
||||
prompt="p", response=Pair(city="Paris", country="France"), start_time=0.0
|
||||
)
|
||||
assert isinstance(payload.response, str)
|
||||
# to_dict() performs the [:500] slice that would otherwise raise on a model
|
||||
assert isinstance(payload.to_dict()["response"], str)
|
||||
@@ -4,7 +4,10 @@ Covers the dependency-free round-trip between PraisonAI eval models and the
|
||||
EvalPort spec dicts. A real agentic test (gated by RUN_LIVE_TESTS=1) runs a live
|
||||
Agent and exports its EvalReport as an EvalPort ResultSet.
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
from datetime import datetime
|
||||
|
||||
import pytest
|
||||
|
||||
@@ -17,6 +20,7 @@ from praisonaiagents.eval import (
|
||||
report_to_evalport,
|
||||
to_evalport,
|
||||
)
|
||||
from praisonaiagents.eval.evalport import EVALPORT_SPEC_VERSION, VERDICT_GRADER_ID
|
||||
|
||||
|
||||
class TestEvalPortImports:
|
||||
@@ -52,21 +56,28 @@ class TestSuiteConversion:
|
||||
|
||||
def test_to_evalport_shape(self):
|
||||
suite = to_evalport(self._pkg())
|
||||
assert suite["kind"] == "suite"
|
||||
assert suite["version"] == EVALPORT_SPEC_VERSION
|
||||
assert suite["id"] == "support_agent_eval"
|
||||
assert suite["name"] == "support_agent_eval"
|
||||
assert suite["version"] == "2.1.0"
|
||||
assert suite["thresholds"] == {"accuracy": 0.9}
|
||||
assert suite["seed"] == 7
|
||||
assert "evalport_version" in suite
|
||||
assert suite["description"] == "Support agent suite"
|
||||
native = suite["metadata"]["praisonai"]
|
||||
assert native == {"version": "2.1.0", "thresholds": {"accuracy": 0.9}, "seed": 7}
|
||||
|
||||
case = suite["cases"][0]
|
||||
grader_ids = [g["id"] for g in suite["graders"]]
|
||||
assert grader_ids == [VERDICT_GRADER_ID, "answer_correct"]
|
||||
for g in suite["graders"]:
|
||||
assert g["type"] == "custom" and g["params"]["handler"]
|
||||
|
||||
case = suite["test_cases"][0]
|
||||
assert case["id"] == "refund_policy"
|
||||
assert case["input"] == "What's your refund policy?"
|
||||
assert case["expected_output"] == "30 days, unused, receipt required"
|
||||
assert case["graders"] == ["answer_correct"]
|
||||
# Every case references the verdict grader (so the verdict result always
|
||||
# has a matching reference) plus its own criteria.
|
||||
assert case["graders"] == [VERDICT_GRADER_ID, "answer_correct"]
|
||||
assert case["metadata"]["tier"] == "gold"
|
||||
assert case["timeout_seconds"] == 15.0
|
||||
assert "timeout_seconds" not in case["metadata"]
|
||||
assert case["timeout_ms"] == 15000
|
||||
assert "praisonai" not in case["metadata"]
|
||||
|
||||
def test_from_evalport_shape(self):
|
||||
suite = to_evalport(self._pkg())
|
||||
@@ -106,9 +117,30 @@ class TestSuiteConversion:
|
||||
|
||||
def test_case_without_expected_or_criteria(self):
|
||||
pkg = EvalPackage(name="p", cases=[EvalCase(name="c", input="x")])
|
||||
case = to_evalport(pkg)["cases"][0]
|
||||
case = to_evalport(pkg)["test_cases"][0]
|
||||
assert "expected_output" not in case
|
||||
assert "graders" not in case
|
||||
# EvalPort requires >= 1 grader per test case: a case without criteria
|
||||
# still references the verdict grader.
|
||||
assert case["graders"] == [VERDICT_GRADER_ID]
|
||||
assert from_evalport(to_evalport(pkg)).cases[0].criteria == []
|
||||
|
||||
def test_empty_package_exports_valid_empty_suite(self):
|
||||
"""A valid package with no cases exports a spec-shaped empty suite."""
|
||||
suite = to_evalport(EvalPackage(name="empty"))
|
||||
assert suite["id"] == "empty"
|
||||
assert suite["test_cases"] == []
|
||||
assert _structural_suite_errors(suite) == []
|
||||
assert from_evalport(suite).cases == []
|
||||
|
||||
def test_nonpositive_timeout_round_trips(self):
|
||||
"""A zero/negative timeout is not spec-representable but must round-trip."""
|
||||
pkg = EvalPackage(
|
||||
name="p", cases=[EvalCase(name="c", input="x", timeout_seconds=0.0)]
|
||||
)
|
||||
case = to_evalport(pkg)["test_cases"][0]
|
||||
assert "timeout_ms" not in case # spec requires timeout_ms >= 1
|
||||
assert case["metadata"]["praisonai"]["timeout_seconds"] == 0.0
|
||||
assert from_evalport(to_evalport(pkg)).cases[0].timeout_seconds == 0.0
|
||||
|
||||
def test_metadata_timeout_key_does_not_corrupt_native_timeout(self):
|
||||
"""A user ``metadata['timeout_seconds']`` must not clobber the native one."""
|
||||
@@ -123,8 +155,8 @@ class TestSuiteConversion:
|
||||
)
|
||||
],
|
||||
)
|
||||
case = to_evalport(pkg)["cases"][0]
|
||||
assert case["timeout_seconds"] == 15.0
|
||||
case = to_evalport(pkg)["test_cases"][0]
|
||||
assert case["timeout_ms"] == 15000
|
||||
assert case["metadata"]["timeout_seconds"] == 999
|
||||
|
||||
round_tripped = from_evalport(to_evalport(pkg)).cases[0]
|
||||
@@ -141,6 +173,43 @@ class TestSuiteConversion:
|
||||
assert case.timeout_seconds == 45
|
||||
assert case.metadata["timeout_seconds"] == 45
|
||||
|
||||
def test_external_spec_suite_imports(self):
|
||||
"""A spec-shaped suite from another tool (test_cases, inline graders)."""
|
||||
external = {
|
||||
"version": "1.0.0",
|
||||
"id": "hub_suite",
|
||||
"test_cases": [
|
||||
{
|
||||
"id": "c1",
|
||||
"input": "hi",
|
||||
"graders": [{"id": "has_hi", "type": "contains", "params": {"substring": "hi"}}],
|
||||
"timeout_ms": 2500,
|
||||
}
|
||||
],
|
||||
}
|
||||
pkg = from_evalport(external)
|
||||
assert pkg.name == "hub_suite"
|
||||
assert pkg.version == "1.0.0" # spec version is not the package version
|
||||
assert pkg.cases[0].criteria == ["has_hi"]
|
||||
assert pkg.cases[0].timeout_seconds == 2.5
|
||||
|
||||
def test_legacy_suite_still_imports(self):
|
||||
"""Suites emitted by the pre-spec-alignment adapter still import."""
|
||||
legacy = {
|
||||
"evalport_version": "1.0",
|
||||
"kind": "suite",
|
||||
"name": "old",
|
||||
"version": "3.0.0",
|
||||
"cases": [{"id": "c1", "input": "hi", "timeout_seconds": 5.0}],
|
||||
"thresholds": {"accuracy": 0.8},
|
||||
"seed": 1,
|
||||
}
|
||||
pkg = from_evalport(legacy)
|
||||
assert pkg.version == "3.0.0"
|
||||
assert pkg.thresholds == {"accuracy": 0.8}
|
||||
assert pkg.seed == 1
|
||||
assert pkg.cases[0].timeout_seconds == 5.0
|
||||
|
||||
|
||||
class TestResultSetConversion:
|
||||
"""EvalReport -> EvalPort ResultSet."""
|
||||
@@ -173,33 +242,155 @@ class TestResultSetConversion:
|
||||
)
|
||||
|
||||
def test_result_set_shape(self):
|
||||
rs = report_to_evalport(self._report())
|
||||
assert rs["kind"] == "result_set"
|
||||
assert rs["suite_name"] == "support_agent_eval"
|
||||
rs = report_to_evalport(
|
||||
self._report(), run_id="run-1", started_at="2026-09-29T00:00:00+00:00"
|
||||
)
|
||||
assert rs["version"] == EVALPORT_SPEC_VERSION
|
||||
assert rs["suite_id"] == "support_agent_eval"
|
||||
assert rs["run_id"] == "run-1"
|
||||
assert rs["started_at"] == "2026-09-29T00:00:00+00:00"
|
||||
assert rs["runner"]["name"] == "praisonaiagents"
|
||||
assert rs["summary"]["total"] == 2
|
||||
assert rs["summary"]["passed"] == 1
|
||||
assert rs["summary"]["failed"] == 1
|
||||
assert rs["summary"]["pass_rate"] == 0.5
|
||||
assert rs["summary"]["average_score"] == 0.6
|
||||
assert rs["summary"]["thresholds_met"] == {"accuracy": False}
|
||||
assert rs["summary"]["avg_score"] == 0.6
|
||||
assert rs["metadata"]["praisonai"]["thresholds_met"] == {"accuracy": False}
|
||||
assert len(rs["results"]) == 2
|
||||
|
||||
def test_run_id_and_started_at_defaults(self):
|
||||
rs = report_to_evalport(self._report())
|
||||
assert isinstance(rs["run_id"], str) and rs["run_id"]
|
||||
assert datetime.fromisoformat(rs["started_at"]).tzinfo is not None
|
||||
assert report_to_evalport(self._report())["run_id"] != rs["run_id"]
|
||||
|
||||
def test_per_case_fields(self):
|
||||
rs = report_to_evalport(self._report())
|
||||
passed, failed = rs["results"]
|
||||
assert passed["case_id"] == "refund_policy"
|
||||
assert passed["test_case_id"] == "refund_policy"
|
||||
assert passed["passed"] is True
|
||||
assert passed["score"] == 0.95
|
||||
assert passed["latency_ms"] == 120.0
|
||||
# The verdict grader result comes first, followed by a per-criterion
|
||||
# grader result that matches the criterion grader referenced in the Suite.
|
||||
assert passed["grader_results"] == [
|
||||
{"grader_id": VERDICT_GRADER_ID, "type": "custom", "score": 0.95, "passed": True},
|
||||
{"grader_id": "answer_correct", "type": "custom", "score": 0.95, "passed": True},
|
||||
]
|
||||
assert passed["duration_ms"] == 120
|
||||
assert passed["actual_output"] == "30 days."
|
||||
assert passed["grader_scores"] == {"answer_correct": 0.95}
|
||||
assert "metadata" not in passed # criteria are graders now, not metadata
|
||||
assert "error" not in passed
|
||||
|
||||
assert failed["case_id"] == "shipping"
|
||||
assert failed["test_case_id"] == "shipping"
|
||||
assert failed["passed"] is False
|
||||
assert failed["error"] == "timeout"
|
||||
assert failed["grader_results"][0]["score"] == 0.25
|
||||
assert failed["grader_results"][0]["passed"] is False
|
||||
assert failed["error"] == {"type": "runner_error", "message": "timeout"}
|
||||
assert "actual_output" not in failed
|
||||
assert "grader_scores" not in failed
|
||||
assert "metadata" not in failed
|
||||
|
||||
def test_out_of_range_score_is_clamped_with_raw_kept(self):
|
||||
report = EvalReport(
|
||||
package_name="p", total_cases=1, passed_cases=1, failed_cases=0,
|
||||
average_score=7.5,
|
||||
results=[EvalResult(case_name="c", passed=True, score=7.5)],
|
||||
)
|
||||
rs = report_to_evalport(report)
|
||||
gr = rs["results"][0]["grader_results"][0]
|
||||
assert gr["score"] == 1.0
|
||||
assert gr["metadata"] == {"openeval.raw_score": 7.5}
|
||||
assert rs["summary"]["avg_score"] == 1.0
|
||||
assert rs["metadata"]["praisonai"]["average_score"] == 7.5
|
||||
|
||||
|
||||
def _structural_suite_errors(suite):
|
||||
"""Minimal EvalPort Suite checks that need no third-party packages."""
|
||||
errs = []
|
||||
if not re.match(r"^\d+\.\d+\.\d+(?:[-+].+)?$", suite.get("version", "")):
|
||||
errs.append("version")
|
||||
if not suite.get("id"):
|
||||
errs.append("id")
|
||||
allowed = {"$schema", "version", "id", "name", "description", "graders",
|
||||
"test_cases", "test_cases_file", "config", "metadata", "tags"}
|
||||
errs += [f"extra:{k}" for k in set(suite) - allowed]
|
||||
grader_ids = [g["id"] for g in suite.get("graders", [])]
|
||||
if len(grader_ids) != len(set(grader_ids)):
|
||||
errs.append("duplicate grader id")
|
||||
if "test_cases" not in suite: # spec allows an empty list, but the key is required
|
||||
errs.append("test_cases")
|
||||
for tc in suite.get("test_cases", []):
|
||||
if not tc.get("id") or not tc.get("input") or not tc.get("graders"):
|
||||
errs.append(f"test_case:{tc.get('id')}")
|
||||
errs += [f"dangling:{g}" for g in tc.get("graders", [])
|
||||
if isinstance(g, str) and g not in grader_ids]
|
||||
return errs
|
||||
|
||||
|
||||
def _structural_result_set_errors(rs):
|
||||
"""Minimal EvalPort ResultSet checks that need no third-party packages."""
|
||||
errs = []
|
||||
if not re.match(r"^\d+\.\d+\.\d+(?:[-+].+)?$", rs.get("version", "")):
|
||||
errs.append("version")
|
||||
errs += [k for k in ("suite_id", "run_id", "started_at") if not rs.get(k)]
|
||||
allowed = {"$schema", "version", "suite_id", "suite_version", "run_id", "started_at",
|
||||
"completed_at", "provider", "runner", "isolation", "group", "results",
|
||||
"summary", "metadata"}
|
||||
errs += [f"extra:{k}" for k in set(rs) - allowed]
|
||||
item_keys = {"test_case_id", "actual_output", "attempt", "completed_at",
|
||||
"grader_results", "passed", "duration_ms", "error", "metadata"}
|
||||
for r in rs.get("results", []):
|
||||
errs += [f"result extra:{k}" for k in set(r) - item_keys]
|
||||
if not r.get("test_case_id") or not isinstance(r.get("passed"), bool):
|
||||
errs.append("result")
|
||||
if not isinstance(r.get("duration_ms", 0), int):
|
||||
errs.append("duration_ms")
|
||||
for g in r.get("grader_results", None) or [None]:
|
||||
if not g or not g.get("grader_id") or not isinstance(g.get("passed"), bool):
|
||||
errs.append("grader_result")
|
||||
elif g["score"] is not None and not 0 <= g["score"] <= 1:
|
||||
errs.append("grader_result score")
|
||||
return errs
|
||||
|
||||
|
||||
class TestEvalPortSpecConformance:
|
||||
"""Emitted documents must pass the EvalPort validators."""
|
||||
|
||||
def _docs(self):
|
||||
pkg = TestSuiteConversion()._pkg()
|
||||
pkg.add_case(EvalCase(name="no_criteria", input="hello"))
|
||||
return to_evalport(pkg), report_to_evalport(TestResultSetConversion()._report())
|
||||
|
||||
def test_structurally_valid_without_sdk(self):
|
||||
suite, rs = self._docs()
|
||||
assert _structural_suite_errors(suite) == []
|
||||
assert _structural_result_set_errors(rs) == []
|
||||
json.dumps(suite)
|
||||
json.dumps(rs)
|
||||
|
||||
def test_criterion_grader_results_reference_suite_graders(self):
|
||||
"""Per-criterion grader results must match graders declared in the Suite.
|
||||
|
||||
The Suite's ``answer_correct`` grader is now surfaced as a grader_result
|
||||
in the ResultSet, so EvalPort consumers can diff that criterion across
|
||||
tools instead of only seeing the verdict grader.
|
||||
"""
|
||||
suite = to_evalport(TestSuiteConversion()._pkg())
|
||||
rs = report_to_evalport(TestResultSetConversion()._report())
|
||||
suite_grader_ids = {g["id"] for g in suite["graders"]}
|
||||
emitted = {
|
||||
g["grader_id"]
|
||||
for r in rs["results"]
|
||||
for g in r["grader_results"]
|
||||
}
|
||||
assert "answer_correct" in emitted
|
||||
assert emitted <= suite_grader_ids
|
||||
|
||||
def test_valid_with_evalport_sdk(self):
|
||||
validate = pytest.importorskip("openeval.validate")
|
||||
suite, rs = self._docs()
|
||||
suite_result = validate.validate_suite(suite)
|
||||
assert suite_result.valid, suite_result.errors
|
||||
rs_result = validate.validate_result_set(rs)
|
||||
assert rs_result.valid, rs_result.errors
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
@@ -244,7 +435,7 @@ class TestEvalPortRealAgent:
|
||||
)
|
||||
|
||||
result_set = report_to_evalport(report)
|
||||
assert result_set["kind"] == "result_set"
|
||||
assert result_set["suite_name"] == "math_eval"
|
||||
assert result_set["results"][0]["case_id"] == "math_addition"
|
||||
assert result_set["suite_id"] == "math_eval"
|
||||
assert result_set["results"][0]["test_case_id"] == "math_addition"
|
||||
assert result_set["results"][0]["actual_output"] == str(answer)
|
||||
assert _structural_result_set_errors(result_set) == []
|
||||
@@ -37,7 +37,7 @@ def test_add_returns_failure_when_embedding_raises():
|
||||
raise RuntimeError("403 model_not_found")
|
||||
|
||||
with patch("praisonaiagents.embedding.embedding", side_effect=boom):
|
||||
result = adapter.add("Paris is the capital of France.")
|
||||
result = adapter.add("Paris is the capital of France.", user_id="u1")
|
||||
|
||||
assert isinstance(result, AddResult)
|
||||
assert result.success is False
|
||||
@@ -54,7 +54,7 @@ def test_search_returns_empty_when_embedding_raises():
|
||||
raise RuntimeError("embedding down")
|
||||
|
||||
with patch("praisonaiagents.embedding.embedding", side_effect=boom):
|
||||
result = adapter.search("Paris")
|
||||
result = adapter.search("Paris", user_id="u1")
|
||||
|
||||
assert isinstance(result, SearchResult)
|
||||
assert result.results == []
|
||||
@@ -72,7 +72,7 @@ def test_add_uses_configurable_embedding_model(monkeypatch):
|
||||
return MagicMock(embeddings=[[0.1, 0.2, 0.3]])
|
||||
|
||||
with patch("praisonaiagents.embedding.embedding", side_effect=fake_embedding):
|
||||
result = adapter.add("hello")
|
||||
result = adapter.add("hello", user_id="u1")
|
||||
|
||||
assert captured["model"] == "custom-embed-model"
|
||||
assert result.success is True
|
||||
|
||||
@@ -0,0 +1,164 @@
|
||||
"""Thread-scoped SQLite cleanup must preserve the shared adapter (#5486).
|
||||
|
||||
`Memory.close_connections()` is documented as thread-scoped: it closes only the
|
||||
calling thread's SQLite connections and leaves the shared `SqliteMemoryAdapter`
|
||||
(and other threads' connections) intact. A previous adapter-shutdown block
|
||||
cleared `self.memory_adapter` unconditionally, so subsequent searches fell back
|
||||
to legacy SQL against `short_mem` / `long_mem` and raised
|
||||
``sqlite3.OperationalError: no such table``.
|
||||
|
||||
These tests pin the contract: after another thread (or the same thread) calls
|
||||
`close_connections()`, the shared adapter survives and searches keep working by
|
||||
lazily reopening the calling thread's connection.
|
||||
"""
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
|
||||
import pytest
|
||||
|
||||
from praisonaiagents.memory.memory import Memory
|
||||
from praisonaiagents.memory.adapters.sqlite_adapter import SqliteMemoryAdapter
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def store(tmp_path, monkeypatch):
|
||||
monkeypatch.setenv("PRAISONAI_HOME", str(tmp_path / "home"))
|
||||
monkeypatch.chdir(tmp_path)
|
||||
return Memory(config={"provider": "sqlite"})
|
||||
|
||||
|
||||
def _texts(results):
|
||||
items = results if isinstance(results, list) else results.get("results", [])
|
||||
return [r.get("text") or r.get("memory") or "" for r in items]
|
||||
|
||||
|
||||
def test_adapter_is_sqlite(store):
|
||||
"""Control: these tests only mean something for the SQLite adapter."""
|
||||
assert isinstance(store.memory_adapter, SqliteMemoryAdapter)
|
||||
|
||||
|
||||
def test_search_before_cleanup(store):
|
||||
"""Control: searching works before any cleanup happens."""
|
||||
store.store_short_term("shared record")
|
||||
assert "shared record" in _texts(store.search_short_term("shared record"))
|
||||
|
||||
|
||||
def test_short_term_search_survives_idle_worker_cleanup(store):
|
||||
"""An idle worker that never opened a connection must not break searches."""
|
||||
store.store_short_term("shared record")
|
||||
with ThreadPoolExecutor(max_workers=1) as pool:
|
||||
pool.submit(store.close_connections).result()
|
||||
|
||||
assert store.memory_adapter is not None
|
||||
assert "shared record" in _texts(store.search_short_term("shared record"))
|
||||
|
||||
|
||||
def test_long_term_search_survives_idle_worker_cleanup(store):
|
||||
store.store_long_term("shared long record")
|
||||
with ThreadPoolExecutor(max_workers=1) as pool:
|
||||
pool.submit(store.close_connections).result()
|
||||
|
||||
assert store.memory_adapter is not None
|
||||
assert "shared long record" in _texts(store.search_long_term("shared long record"))
|
||||
|
||||
|
||||
def test_short_term_search_survives_initialized_worker_cleanup(store):
|
||||
"""A worker that has opened its own connection cleans up only its own."""
|
||||
store.store_short_term("shared record")
|
||||
|
||||
def worker():
|
||||
store.store_short_term("worker record")
|
||||
store.close_connections()
|
||||
|
||||
with ThreadPoolExecutor(max_workers=1) as pool:
|
||||
pool.submit(worker).result()
|
||||
|
||||
assert store.memory_adapter is not None
|
||||
assert "shared record" in _texts(store.search_short_term("shared record"))
|
||||
assert "worker record" in _texts(store.search_short_term("worker record"))
|
||||
|
||||
|
||||
def test_same_thread_repeated_cleanup_and_reopen(store):
|
||||
"""Repeated same-thread cleanup must keep the adapter usable (lazy reopen)."""
|
||||
store.store_short_term("shared record")
|
||||
store.close_connections()
|
||||
store.close_connections()
|
||||
|
||||
assert store.memory_adapter is not None
|
||||
assert "shared record" in _texts(store.search_short_term("shared record"))
|
||||
|
||||
|
||||
def test_same_thread_long_term_cleanup_and_reopen(store):
|
||||
store.store_long_term("shared long record")
|
||||
store.close_connections()
|
||||
|
||||
assert store.memory_adapter is not None
|
||||
assert "shared long record" in _texts(store.search_long_term("shared long record"))
|
||||
|
||||
|
||||
def test_adapter_thread_close_is_idempotent(store):
|
||||
"""Adapter-level thread close tolerates being called with no open conns."""
|
||||
store.memory_adapter.close_thread_connections()
|
||||
store.memory_adapter.close_thread_connections()
|
||||
store.store_short_term("shared record")
|
||||
assert "shared record" in _texts(store.search_short_term("shared record"))
|
||||
|
||||
|
||||
def test_adapter_thread_close_closes_open_connection(store):
|
||||
"""Opening a connection then closing it directly must close the live conn
|
||||
and drop it from the shared registry (not just the no-op path)."""
|
||||
adapter = store.memory_adapter
|
||||
adapter.store_short_term("shared record")
|
||||
adapter.store_long_term("shared long record")
|
||||
|
||||
stm = adapter._local.stm_conn
|
||||
ltm = adapter._local.ltm_conn
|
||||
assert stm in adapter._all_connections
|
||||
assert ltm in adapter._all_connections
|
||||
|
||||
adapter.close_thread_connections()
|
||||
|
||||
# Thread-local refs cleared and registry no longer tracks the closed conns.
|
||||
assert getattr(adapter._local, "stm_conn", None) is None
|
||||
assert getattr(adapter._local, "ltm_conn", None) is None
|
||||
assert stm not in adapter._all_connections
|
||||
assert ltm not in adapter._all_connections
|
||||
|
||||
# The physical connections are closed (operations raise ProgrammingError).
|
||||
with pytest.raises(Exception):
|
||||
stm.execute("SELECT 1")
|
||||
with pytest.raises(Exception):
|
||||
ltm.execute("SELECT 1")
|
||||
|
||||
# Adapter remains usable via lazy reopen.
|
||||
assert "shared record" in _texts(adapter.search_short_term("shared record"))
|
||||
|
||||
|
||||
def test_adapter_thread_close_keeps_connection_tracked_on_failure(store):
|
||||
"""If a connection fails to close, it stays tracked for a later retry and
|
||||
the thread-local ref is preserved (the open-conn failure path)."""
|
||||
adapter = store.memory_adapter
|
||||
adapter.store_short_term("shared record")
|
||||
real = adapter._local.stm_conn
|
||||
|
||||
class _FlakyConn:
|
||||
def __init__(self, inner):
|
||||
self._inner = inner
|
||||
self.close_calls = 0
|
||||
|
||||
def close(self):
|
||||
self.close_calls += 1
|
||||
raise RuntimeError("boom")
|
||||
|
||||
flaky = _FlakyConn(real)
|
||||
# Swap the tracked real conn for a flaky wrapper in both registry and tls.
|
||||
with adapter._connection_lock:
|
||||
adapter._all_connections.discard(real)
|
||||
adapter._all_connections.add(flaky)
|
||||
adapter._local.stm_conn = flaky
|
||||
|
||||
adapter.close_thread_connections()
|
||||
|
||||
assert flaky.close_calls == 1
|
||||
# Failed close: connection stays tracked and the ref is NOT cleared.
|
||||
assert flaky in adapter._all_connections
|
||||
assert adapter._local.stm_conn is flaky
|
||||
@@ -692,3 +692,156 @@ class TestCallerEnforcement:
|
||||
)
|
||||
results = runner.execute_sync(HookEvent.BEFORE_LLM, inp)
|
||||
assert runner.is_blocked(results) is True
|
||||
|
||||
|
||||
class GatewayForensicsLikePlugin(Plugin):
|
||||
"""Mirrors the bundled GatewayForensicsPlugin: declares gateway hooks and
|
||||
defines matching observe-only methods that must actually fire."""
|
||||
|
||||
def __init__(self):
|
||||
self.events = []
|
||||
|
||||
@property
|
||||
def info(self):
|
||||
return PluginInfo(
|
||||
name="gwforensics",
|
||||
hooks=[PluginHook.GATEWAY_START, PluginHook.GATEWAY_STOP],
|
||||
)
|
||||
|
||||
def gateway_start(self, context):
|
||||
self.events.append(("start", context.get("bot_count")))
|
||||
|
||||
def gateway_stop(self, context):
|
||||
self.events.append(("stop", context.get("reason")))
|
||||
|
||||
|
||||
class AllRemainingLifecyclePlugin(Plugin):
|
||||
"""Declares every remaining lifecycle event the generic tail must bridge."""
|
||||
|
||||
def __init__(self):
|
||||
self.fired = []
|
||||
|
||||
@property
|
||||
def info(self):
|
||||
return PluginInfo(
|
||||
name="allremaining",
|
||||
hooks=[
|
||||
PluginHook.SCHEDULE_ADD,
|
||||
PluginHook.SCHEDULE_REMOVE,
|
||||
PluginHook.SCHEDULE_TRIGGER,
|
||||
PluginHook.BEFORE_COMPACTION,
|
||||
PluginHook.AFTER_COMPACTION,
|
||||
PluginHook.SUBAGENT_STOP,
|
||||
PluginHook.MODEL_FALLBACK,
|
||||
PluginHook.KANBAN_TASK_CREATED,
|
||||
],
|
||||
)
|
||||
|
||||
def schedule_add(self, context):
|
||||
self.fired.append("schedule_add")
|
||||
|
||||
def schedule_remove(self, context):
|
||||
self.fired.append("schedule_remove")
|
||||
|
||||
def schedule_trigger(self, context):
|
||||
self.fired.append("schedule_trigger")
|
||||
|
||||
def before_compaction(self, context):
|
||||
self.fired.append("before_compaction")
|
||||
|
||||
def after_compaction(self, context):
|
||||
self.fired.append("after_compaction")
|
||||
|
||||
def subagent_stop(self, context):
|
||||
self.fired.append("subagent_stop")
|
||||
|
||||
def model_fallback(self, context):
|
||||
self.fired.append("model_fallback")
|
||||
|
||||
def kanban_task_created(self, context):
|
||||
self.fired.append("kanban_task_created")
|
||||
|
||||
|
||||
class DeclaredButNotOverriddenPlugin(Plugin):
|
||||
"""Declares gateway hooks but does NOT override the base no-op methods."""
|
||||
|
||||
@property
|
||||
def info(self):
|
||||
return PluginInfo(
|
||||
name="declared_noop",
|
||||
hooks=[PluginHook.GATEWAY_START, PluginHook.GATEWAY_STOP],
|
||||
)
|
||||
|
||||
|
||||
class TestGenericLifecycleBridge:
|
||||
def test_gateway_hooks_wire_and_fire(self):
|
||||
reg = HookRegistry()
|
||||
mgr = PluginManager()
|
||||
plugin = GatewayForensicsLikePlugin()
|
||||
mgr.register(plugin)
|
||||
assert mgr.wire_into_hook_registry(reg) == 2
|
||||
assert reg.has_hooks(HookEvent.GATEWAY_START)
|
||||
assert reg.has_hooks(HookEvent.GATEWAY_STOP)
|
||||
|
||||
from praisonaiagents.hooks.events import GatewayStartInput, GatewayStopInput
|
||||
|
||||
runner = HookRunner(registry=reg, cwd=os.getcwd())
|
||||
runner.execute_sync(
|
||||
HookEvent.GATEWAY_START,
|
||||
GatewayStartInput(
|
||||
session_id="s", cwd=os.getcwd(),
|
||||
event_name=HookEvent.GATEWAY_START,
|
||||
timestamp=str(time.time()), bot_count=3,
|
||||
),
|
||||
)
|
||||
runner.execute_sync(
|
||||
HookEvent.GATEWAY_STOP,
|
||||
GatewayStopInput(
|
||||
session_id="s", cwd=os.getcwd(),
|
||||
event_name=HookEvent.GATEWAY_STOP,
|
||||
timestamp=str(time.time()), reason="sigterm",
|
||||
),
|
||||
)
|
||||
assert ("start", 3) in plugin.events
|
||||
assert ("stop", "sigterm") in plugin.events
|
||||
|
||||
def test_all_remaining_lifecycle_events_bridge(self):
|
||||
events = dict(_adapt_plugin_hooks(AllRemainingLifecyclePlugin()))
|
||||
for event in (
|
||||
HookEvent.SCHEDULE_ADD, HookEvent.SCHEDULE_REMOVE,
|
||||
HookEvent.SCHEDULE_TRIGGER, HookEvent.BEFORE_COMPACTION,
|
||||
HookEvent.AFTER_COMPACTION, HookEvent.SUBAGENT_STOP,
|
||||
HookEvent.MODEL_FALLBACK, HookEvent.KANBAN_TASK_CREATED,
|
||||
):
|
||||
assert event in events
|
||||
|
||||
def test_generic_hook_delivers_payload(self):
|
||||
plugin = AllRemainingLifecyclePlugin()
|
||||
events = dict(_adapt_plugin_hooks(plugin))
|
||||
|
||||
from praisonaiagents.hooks.events import SubagentStopInput
|
||||
|
||||
data = SubagentStopInput(
|
||||
session_id="s", cwd=os.getcwd(),
|
||||
event_name=HookEvent.SUBAGENT_STOP,
|
||||
timestamp=str(time.time()), task="t", success=True,
|
||||
)
|
||||
events[HookEvent.SUBAGENT_STOP](data)
|
||||
assert "subagent_stop" in plugin.fired
|
||||
|
||||
def test_declared_but_not_overridden_registers_nothing(self):
|
||||
# Base no-op methods must NOT be wired as dead hooks.
|
||||
assert list(_adapt_plugin_hooks(DeclaredButNotOverriddenPlugin())) == []
|
||||
|
||||
def test_undeclared_overridden_method_not_registered(self):
|
||||
# A plugin that defines gateway_start but never declares it in
|
||||
# PluginInfo.hooks must not be bridged (declaration-driven only).
|
||||
class _Undeclared(Plugin):
|
||||
@property
|
||||
def info(self):
|
||||
return PluginInfo(name="undeclared")
|
||||
|
||||
def gateway_start(self, context):
|
||||
pass
|
||||
|
||||
assert list(_adapt_plugin_hooks(_Undeclared())) == []
|
||||
@@ -0,0 +1,123 @@
|
||||
"""Closing and reopening a Session must preserve complete agent messages."""
|
||||
|
||||
from copy import deepcopy
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from praisonaiagents.session.api import AGENT_HISTORY_KEY, Session
|
||||
from praisonaiagents.session.store import DefaultSessionStore
|
||||
|
||||
|
||||
@pytest.mark.parametrize("history", [
|
||||
[{"role": "user", "content": "hello"}],
|
||||
[
|
||||
{"role": "assistant", "content": None, "tool_calls": [
|
||||
{"id": "call_1", "type": "function", "function": {
|
||||
"name": "lookup", "arguments": '{"query":"hello"}'}}
|
||||
]},
|
||||
{"role": "tool", "content": "found", "tool_call_id": "call_1",
|
||||
"name": "lookup"},
|
||||
{"role": "assistant", "content": "answer"},
|
||||
],
|
||||
[{"role": "assistant", "content": "answer", "reasoning_content": "why",
|
||||
"metadata": {"citations": [{"source": "local"}]}, "custom": False}],
|
||||
[{"role": "user", "content": [
|
||||
{"type": "text", "text": "describe"},
|
||||
{"type": "image_url", "image_url": {"url": "data:image/png;base64,AA=="}},
|
||||
], "name": "reader"}],
|
||||
], ids=["plain-control", "tool-turn", "reasoning-and-metadata", "multimodal"])
|
||||
def test_close_reopen_preserves_message_fields(tmp_path, monkeypatch, history):
|
||||
import praisonaiagents.agent as agent_module
|
||||
import praisonaiagents.session.store as store_module
|
||||
|
||||
class LocalAgent:
|
||||
def __init__(self, **kwargs):
|
||||
self.chat_history = []
|
||||
|
||||
monkeypatch.chdir(tmp_path)
|
||||
monkeypatch.setenv("PRAISONAI_HOME", str(tmp_path / "home"))
|
||||
monkeypatch.setattr(agent_module, "Agent", LocalAgent)
|
||||
session_dir = str(tmp_path / "sessions")
|
||||
store = DefaultSessionStore(session_dir=session_dir)
|
||||
monkeypatch.setattr(store_module, "_default_store", store)
|
||||
session = Session(session_id="parent")
|
||||
session._memory = SimpleNamespace(search_short_term=lambda **kwargs: [])
|
||||
agent = session.Agent(name="helper", memory=False)
|
||||
agent.chat_history = deepcopy(history)
|
||||
session.close()
|
||||
|
||||
fresh = DefaultSessionStore(session_dir=session_dir)
|
||||
monkeypatch.setattr(store_module, "_default_store", fresh)
|
||||
assert fresh.get_session("parent").metadata[AGENT_HISTORY_KEY][
|
||||
"helper:Assistant"
|
||||
] == history
|
||||
resumed = Session(session_id="parent").Agent(name="helper", memory=False)
|
||||
assert resumed.chat_history == history
|
||||
|
||||
|
||||
@pytest.mark.parametrize("builder", ["openai", "litellm"])
|
||||
def test_restored_extensions_do_not_reach_responses_input(tmp_path, monkeypatch, builder):
|
||||
import praisonaiagents.session.store as store_module
|
||||
from praisonaiagents.llm.openai_client import OpenAIClient
|
||||
from praisonaiagents.llm.llm import LLM
|
||||
|
||||
monkeypatch.chdir(tmp_path)
|
||||
monkeypatch.setenv("PRAISONAI_HOME", str(tmp_path / "home"))
|
||||
store = DefaultSessionStore(session_dir=str(tmp_path / "sessions"))
|
||||
monkeypatch.setattr(store_module, "_default_store", store)
|
||||
history = [
|
||||
{"role": "user", "content": "question", "name": "reader",
|
||||
"metadata": {"origin": "disk"}},
|
||||
{"role": "assistant", "content": "answer", "reasoning_content": "why",
|
||||
"metadata": {"citation": "local"}, "extension": False},
|
||||
{"role": "assistant", "content": None, "tool_calls": [{
|
||||
"id": "call_1", "type": "function",
|
||||
"function": {"name": "lookup", "arguments": "{}"},
|
||||
}]},
|
||||
{"role": "tool", "tool_call_id": "call_1", "content": "found"},
|
||||
]
|
||||
session = Session(session_id="parent")
|
||||
session._agents["helper:Assistant"] = {"agent": None, "chat_history": history}
|
||||
session.close()
|
||||
resumed = Session(session_id="parent")._restore_agent_chat_history("helper:Assistant")
|
||||
if builder == "openai":
|
||||
client = object.__new__(OpenAIClient)
|
||||
params = client._build_responses_input(resumed, model="gpt-4o")
|
||||
else:
|
||||
llm = object.__new__(LLM)
|
||||
llm.max_tokens = None
|
||||
llm.top_p = None
|
||||
llm.base_url = None
|
||||
llm.api_key = None
|
||||
llm.timeout = None
|
||||
llm._resolve_subscription_creds = lambda: None
|
||||
llm._resolve_openai_compatible_model = lambda: "gpt-4o"
|
||||
params = llm._build_responses_params(resumed)
|
||||
assert params["input"] == [
|
||||
{"role": "user", "content": "question"},
|
||||
{"role": "assistant", "content": "answer"},
|
||||
{"type": "function_call", "call_id": "call_1", "name": "lookup",
|
||||
"arguments": "{}"},
|
||||
{"type": "function_call_output", "call_id": "call_1", "output": "found"},
|
||||
]
|
||||
assert store.get_session("parent").metadata[AGENT_HISTORY_KEY][
|
||||
"helper:Assistant"
|
||||
] == history
|
||||
|
||||
|
||||
def test_legacy_defaults_and_non_message_filter(tmp_path, monkeypatch):
|
||||
import praisonaiagents.session.store as store_module
|
||||
|
||||
monkeypatch.chdir(tmp_path)
|
||||
monkeypatch.setenv("PRAISONAI_HOME", str(tmp_path / "home"))
|
||||
store = DefaultSessionStore(session_dir=str(tmp_path / "sessions"))
|
||||
monkeypatch.setattr(store_module, "_default_store", store)
|
||||
session = Session(session_id="parent")
|
||||
session._agents["helper:Assistant"] = {
|
||||
"agent": None, "chat_history": [None, {"name": "helper"}],
|
||||
}
|
||||
session.close()
|
||||
assert store.get_session("parent").metadata[AGENT_HISTORY_KEY][
|
||||
"helper:Assistant"
|
||||
] == [{"role": "user", "content": "", "name": "helper"}]
|
||||
@@ -0,0 +1,91 @@
|
||||
"""Concurrent Session closes must merge different agent histories under lock."""
|
||||
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from threading import Event
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from praisonaiagents.session.api import AGENT_HISTORY_KEY, Session
|
||||
from praisonaiagents.session.store import DefaultSessionStore
|
||||
|
||||
|
||||
@pytest.mark.parametrize("existing", [{}, {"peer:Assistant": [
|
||||
{"role": "user", "content": "peer"},
|
||||
]}])
|
||||
def test_two_closes_preserve_both_agent_histories(tmp_path, monkeypatch, existing):
|
||||
monkeypatch.chdir(tmp_path)
|
||||
monkeypatch.setenv("PRAISONAI_HOME", str(tmp_path / "home"))
|
||||
directory = str(tmp_path / "sessions")
|
||||
first = DefaultSessionStore(session_dir=directory)
|
||||
second = DefaultSessionStore(session_dir=directory)
|
||||
first.update_session_metadata("parent", **{AGENT_HISTORY_KEY: existing})
|
||||
a, b = Session(session_id="parent"), Session(session_id="parent")
|
||||
monkeypatch.setattr(a, "_get_session_store", lambda: first)
|
||||
monkeypatch.setattr(b, "_get_session_store", lambda: second)
|
||||
ah = [{"role": "user", "content": "a"}]
|
||||
bh = [{"role": "user", "content": "b"}]
|
||||
a._agents["a:Assistant"] = {"agent": None, "chat_history": ah}
|
||||
b._agents["b:Assistant"] = {"agent": None, "chat_history": bh}
|
||||
ready, completed = Event(), Event()
|
||||
original = first._modify_session_locked
|
||||
|
||||
def pause_before_lock(*args, **kwargs):
|
||||
ready.set()
|
||||
assert completed.wait(10), "second writer did not complete"
|
||||
return original(*args, **kwargs)
|
||||
|
||||
monkeypatch.setattr(first, "_modify_session_locked", pause_before_lock)
|
||||
|
||||
def close_second():
|
||||
assert ready.wait(10), "first writer did not reach write boundary"
|
||||
try:
|
||||
b.close()
|
||||
finally:
|
||||
completed.set()
|
||||
|
||||
with ThreadPoolExecutor(max_workers=2) as pool:
|
||||
fa, fb = pool.submit(a.close), pool.submit(close_second)
|
||||
fa.result(timeout=15)
|
||||
fb.result(timeout=15)
|
||||
fresh = DefaultSessionStore(session_dir=directory)
|
||||
assert fresh.get_session("parent").metadata[AGENT_HISTORY_KEY] == {
|
||||
**existing, "a:Assistant": ah, "b:Assistant": bh,
|
||||
}
|
||||
|
||||
|
||||
def test_metadata_map_defaults_do_not_overwrite_current_entries(tmp_path):
|
||||
store = DefaultSessionStore(session_dir=str(tmp_path))
|
||||
store.update_session_metadata("parent", marker="keep", **{AGENT_HISTORY_KEY: {
|
||||
"current": [], "peer": ["present"],
|
||||
}})
|
||||
assert store.merge_session_metadata_map(
|
||||
"parent", AGENT_HISTORY_KEY, {"new": ["new"]},
|
||||
defaults={"current": ["legacy"], "legacy-only": ["old"]},
|
||||
)
|
||||
data = DefaultSessionStore(session_dir=str(tmp_path)).get_session("parent")
|
||||
assert data.metadata[AGENT_HISTORY_KEY] == {
|
||||
"current": [], "peer": ["present"], "new": ["new"],
|
||||
"legacy-only": ["old"],
|
||||
}
|
||||
assert data.metadata["marker"] == "keep"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("failure", ["read", "write"])
|
||||
def test_metadata_map_failure_preserves_disk(tmp_path, monkeypatch, failure):
|
||||
import praisonaiagents.session.store as store_module
|
||||
|
||||
store = DefaultSessionStore(session_dir=str(tmp_path))
|
||||
store.update_session_metadata("parent", **{AGENT_HISTORY_KEY: {"peer": ["old"]}})
|
||||
path = Path(store._get_session_path("parent"))
|
||||
before = path.read_bytes()
|
||||
|
||||
def fail(*args, **kwargs):
|
||||
raise OSError("injected I/O failure")
|
||||
|
||||
if failure == "read":
|
||||
monkeypatch.setattr(store, "_load_session_from_disk", fail)
|
||||
else:
|
||||
monkeypatch.setattr(store_module.os, "replace", fail)
|
||||
assert not store.merge_session_metadata_map("parent", AGENT_HISTORY_KEY, {"new": []})
|
||||
assert path.read_bytes() == before
|
||||
@@ -0,0 +1,52 @@
|
||||
"""Default imports must not overwrite a session created after the first check."""
|
||||
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from threading import Event
|
||||
|
||||
import pytest
|
||||
|
||||
from praisonaiagents.session.store import DefaultSessionStore
|
||||
|
||||
|
||||
@pytest.mark.parametrize("overwrite", [False, True])
|
||||
def test_import_rechecks_new_peer_session_under_write_lock(tmp_path, monkeypatch, overwrite):
|
||||
directory = str(tmp_path / "sessions")
|
||||
importer = DefaultSessionStore(session_dir=directory)
|
||||
peer = DefaultSessionStore(session_dir=directory)
|
||||
ready, completed = Event(), Event()
|
||||
original = importer._save_imported_session
|
||||
|
||||
def pause_before_write(*args, **kwargs):
|
||||
ready.set()
|
||||
assert completed.wait(10), "peer did not finish creating its session"
|
||||
return original(*args, **kwargs)
|
||||
|
||||
monkeypatch.setattr(importer, "_save_imported_session", pause_before_write)
|
||||
payload = {"version": 1, "sessions": [{
|
||||
"session_id": "shared", "messages": [{
|
||||
"role": "user", "content": "imported", "timestamp": 1,
|
||||
}],
|
||||
}]}
|
||||
|
||||
def create_peer():
|
||||
assert ready.wait(10), "import did not reach write boundary"
|
||||
try:
|
||||
assert peer.add_message("shared", "user", "peer-created")
|
||||
finally:
|
||||
completed.set()
|
||||
|
||||
with ThreadPoolExecutor(max_workers=2) as pool:
|
||||
future = pool.submit(create_peer)
|
||||
report = importer.import_sessions(payload, overwrite=overwrite)
|
||||
future.result(timeout=15)
|
||||
fresh = DefaultSessionStore(session_dir=directory)
|
||||
if overwrite:
|
||||
assert report.imported == 1
|
||||
assert not report.skipped
|
||||
assert fresh.get_chat_history("shared") == [{"role": "user", "content": "imported"}]
|
||||
else:
|
||||
assert report.imported == 0
|
||||
assert report.skipped == [{
|
||||
"session_id": "shared", "reason": "already exists (use overwrite)",
|
||||
}]
|
||||
assert fresh.get_chat_history("shared") == [{"role": "user", "content": "peer-created"}]
|
||||
@@ -0,0 +1,187 @@
|
||||
"""Portable imports refresh the indexed store's content and routing views."""
|
||||
|
||||
import pytest
|
||||
import threading
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
|
||||
from praisonaiagents.session.sqlite_store import SqliteSessionStore
|
||||
from praisonaiagents.session.store import DefaultSessionStore
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def make_store(tmp_path):
|
||||
stores = []
|
||||
|
||||
def create(kind, **kwargs):
|
||||
store = kind(session_dir=str(tmp_path / str(len(stores))), **kwargs)
|
||||
stores.append(store)
|
||||
return store
|
||||
|
||||
yield create
|
||||
for store in stores:
|
||||
conn = getattr(store, "_conn", None)
|
||||
if conn is not None:
|
||||
conn.close()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("kind", [DefaultSessionStore, SqliteSessionStore])
|
||||
@pytest.mark.parametrize("warm", [False, True])
|
||||
@pytest.mark.parametrize("overwrite", [False, True])
|
||||
@pytest.mark.parametrize("reset_live_fields", [False, True])
|
||||
def test_import_refreshes_content_and_routes(make_store, kind, warm, overwrite, reset_live_fields):
|
||||
source = make_store(DefaultSessionStore)
|
||||
messages = [{"role": "user", "content": f"narwhal imported turn {index}"} for index in range(5)]
|
||||
assert source.set_chat_history("session", messages)
|
||||
assert source.set_gateway_info("session", gateway_session_id="new-gateway", agent_id="new-agent")
|
||||
payload = source.export_all()
|
||||
|
||||
destination = make_store(kind, active_window=2)
|
||||
if overwrite:
|
||||
assert destination.add_message("session", "user", "pelican old turn")
|
||||
assert destination.set_gateway_info("session", gateway_session_id="old-gateway", agent_id="old-agent")
|
||||
if warm:
|
||||
destination.search("pelican")
|
||||
destination.get_by_gateway_session("old-gateway")
|
||||
|
||||
report = destination.import_sessions(payload, overwrite=overwrite, reset_live_fields=reset_live_fields)
|
||||
assert report.imported == 1
|
||||
assert report.skipped == []
|
||||
session = destination.get_session("session")
|
||||
assert [message.content for message in session.messages] == [message["content"] for message in messages]
|
||||
assert destination.get_by_gateway_session("old-gateway") is None
|
||||
assert destination.list_sessions_by_gateway_agent("old-agent") == []
|
||||
found = destination.get_by_gateway_session("new-gateway")
|
||||
if reset_live_fields:
|
||||
assert found is None
|
||||
assert destination.list_sessions_by_gateway_agent("new-agent") == []
|
||||
else:
|
||||
assert found is not None and found.session_id == "session"
|
||||
assert destination.list_sessions_by_gateway_agent("new-agent") == ["session"]
|
||||
assert destination.search("pelican") == []
|
||||
assert [hit.session_id for hit in destination.search("narwhal")] == ["session"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("write_failure", [False, True])
|
||||
def test_unsaved_import_keeps_previous_index(make_store, monkeypatch, write_failure):
|
||||
source = make_store(DefaultSessionStore)
|
||||
assert source.add_message("session", "user", "narwhal replacement")
|
||||
destination = make_store(SqliteSessionStore)
|
||||
assert destination.add_message("session", "user", "pelican original")
|
||||
assert destination.set_gateway_info("session", gateway_session_id="gateway", agent_id="agent")
|
||||
assert destination.search("pelican")
|
||||
if write_failure:
|
||||
monkeypatch.setattr(destination, "_atomic_write_json", lambda *args: False)
|
||||
|
||||
report = destination.import_sessions(source.export_all(), overwrite=write_failure)
|
||||
assert report.imported == 0
|
||||
assert len(report.skipped) == 1
|
||||
assert report.skipped[0]["reason"] == ("write failed" if write_failure else "already exists (use overwrite)")
|
||||
assert [hit.session_id for hit in destination.search("pelican")] == ["session"]
|
||||
assert destination.search("narwhal") == []
|
||||
assert destination.get_by_gateway_session("gateway").session_id == "session"
|
||||
assert destination.list_sessions_by_gateway_agent("agent") == ["session"]
|
||||
|
||||
|
||||
def test_warm_import_indexes_archived_turns(make_store):
|
||||
source = make_store(DefaultSessionStore, active_window=3)
|
||||
assert source.add_message("session", "user", "pelican archived marker")
|
||||
for index in range(8):
|
||||
assert source.add_message("session", "user", f"later turn {index}")
|
||||
assert any("pelican" in message.content for message in source.get_session("session").archived_messages)
|
||||
destination = make_store(SqliteSessionStore, active_window=2)
|
||||
assert destination.search("pelican") == []
|
||||
assert destination.import_sessions(source.export_all()).imported == 1
|
||||
hits = destination.search("pelican")
|
||||
assert [hit.session_id for hit in hits] == ["session"]
|
||||
assert any(message["archived"] for message in hits[0].messages)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("separate_store", [False, True])
|
||||
def test_write_between_import_save_and_index_keeps_newer_content(make_store, monkeypatch, separate_store):
|
||||
source = make_store(DefaultSessionStore)
|
||||
assert source.add_message("session", "user", "pelican imported turn")
|
||||
destination = make_store(SqliteSessionStore)
|
||||
destination.search("pelican")
|
||||
writer = destination
|
||||
if separate_store:
|
||||
writer = SqliteSessionStore(session_dir=destination.session_dir, db_path=destination.db_path)
|
||||
writer.search("pelican")
|
||||
|
||||
saved = threading.Event()
|
||||
resume = threading.Event()
|
||||
original = DefaultSessionStore._save_imported_session
|
||||
|
||||
def delayed_save(store, session, **kwargs):
|
||||
result = original(store, session, **kwargs)
|
||||
saved.set()
|
||||
assert resume.wait(5), "concurrent writer did not finish"
|
||||
return result
|
||||
|
||||
monkeypatch.setattr(DefaultSessionStore, "_save_imported_session", delayed_save)
|
||||
try:
|
||||
with ThreadPoolExecutor(max_workers=1) as executor:
|
||||
importing = executor.submit(destination.import_sessions, source.export_all())
|
||||
try:
|
||||
assert saved.wait(5), "import did not persist the JSON"
|
||||
assert writer.set_chat_history("session", [{"role": "user", "content": "narwhal newer turn"}])
|
||||
assert writer.set_gateway_info("session", gateway_session_id="new-gateway", agent_id="new-agent")
|
||||
assert writer.search("narwhal")
|
||||
finally:
|
||||
resume.set()
|
||||
assert importing.result(timeout=5).imported == 1
|
||||
destination.invalidate_cache()
|
||||
assert destination.get_session("session").messages[0].content == "narwhal newer turn"
|
||||
assert [hit.session_id for hit in destination.search("narwhal")] == ["session"]
|
||||
assert destination.search("pelican") == []
|
||||
assert destination.get_by_gateway_session("new-gateway").session_id == "session"
|
||||
assert destination.list_sessions_by_gateway_agent("new-agent") == ["session"]
|
||||
finally:
|
||||
if separate_store and writer._conn is not None:
|
||||
writer._conn.close()
|
||||
|
||||
|
||||
def test_post_import_refresh_failure_does_not_report_durable_write_failure(make_store, monkeypatch):
|
||||
source = make_store(DefaultSessionStore)
|
||||
assert source.add_message("session", "user", "durable imported note")
|
||||
destination = make_store(SqliteSessionStore)
|
||||
|
||||
def unavailable(*args):
|
||||
raise OSError("injected index refresh read failure")
|
||||
|
||||
monkeypatch.setattr(destination, "_load_session_from_disk", unavailable)
|
||||
report = destination.import_sessions(source.export_all())
|
||||
assert report.imported == 1
|
||||
assert report.skipped == []
|
||||
reader = DefaultSessionStore(session_dir=destination.session_dir)
|
||||
assert reader.get_session("session").messages[0].content == "durable imported note"
|
||||
|
||||
|
||||
def test_post_import_read_failure_does_not_restore_old_routes(make_store, monkeypatch):
|
||||
source = make_store(DefaultSessionStore)
|
||||
assert source.add_message("session", "user", "narwhal replacement")
|
||||
destination = make_store(SqliteSessionStore)
|
||||
assert destination.add_message("session", "user", "pelican original")
|
||||
assert destination.set_gateway_info("session", gateway_session_id="old-gateway", agent_id="old-agent")
|
||||
assert destination.get_by_gateway_session("old-gateway").session_id == "session"
|
||||
assert destination.search("pelican")
|
||||
original = destination._load_session_from_disk
|
||||
|
||||
def unavailable(*args):
|
||||
raise OSError("injected post-save read failure")
|
||||
|
||||
monkeypatch.setattr(destination, "_load_session_from_disk", unavailable)
|
||||
report = destination.import_sessions(source.export_all(), overwrite=True)
|
||||
assert report.imported == 1
|
||||
assert report.skipped == []
|
||||
assert destination.get_by_gateway_session("old-gateway") is None
|
||||
assert destination.list_sessions_by_gateway_agent("old-agent") == []
|
||||
monkeypatch.setattr(destination, "_load_session_from_disk", original)
|
||||
destination.invalidate_cache()
|
||||
saved = destination.get_session("session")
|
||||
assert saved.gateway_session_id is None
|
||||
assert saved.agent_id is None
|
||||
assert saved.messages[0].content == "narwhal replacement"
|
||||
assert destination.search("narwhal") == []
|
||||
assert destination.add_message("session", "user", "later update")
|
||||
assert [hit.session_id for hit in destination.search("narwhal")] == ["session"]
|
||||
assert destination.get_by_gateway_session("old-gateway") is None
|
||||
@@ -0,0 +1,179 @@
|
||||
"""Portable session operations use the SQLite transcript persistence backend."""
|
||||
|
||||
import json
|
||||
import tracemalloc
|
||||
|
||||
import pytest
|
||||
|
||||
from praisonaiagents.session.store import DefaultSessionStore
|
||||
from praisonaiagents.session.sqlite_transcript_store import SqliteTranscriptStore
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def stores(tmp_path):
|
||||
opened = []
|
||||
|
||||
def make(name, backend=SqliteTranscriptStore, **kwargs):
|
||||
store = backend(session_dir=str(tmp_path / name), **kwargs)
|
||||
opened.append(store)
|
||||
return store
|
||||
|
||||
yield make
|
||||
for store in opened:
|
||||
conn = getattr(store, "_conn", None)
|
||||
if conn is not None:
|
||||
conn.close()
|
||||
|
||||
|
||||
def test_export_all_reads_database_not_json_sidecars(stores):
|
||||
store = stores("source")
|
||||
assert store.add_message("one", "user", "First")
|
||||
assert store.add_message("two", "assistant", "Second")
|
||||
from pathlib import Path
|
||||
(Path(store.session_dir) / "unrelated.json").write_text(
|
||||
json.dumps({"session_id": "sidecar", "messages": []}), encoding="utf-8"
|
||||
)
|
||||
payload = store.export_all()
|
||||
assert payload["version"] == store.PORTABLE_VERSION
|
||||
assert {record["session_id"] for record in payload["sessions"]} == {"one", "two"}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("key", ["lineage_id", "root_session_id", "thread_id"])
|
||||
def test_single_export_includes_database_lineage(stores, key):
|
||||
store = stores("source")
|
||||
for session_id in ("first", "continuation", "unrelated"):
|
||||
assert store.add_message(session_id, "user", session_id)
|
||||
assert store.update_session_metadata(session_id, **{key: "shared" if session_id != "unrelated" else "other"})
|
||||
payload = store.export_session("continuation")
|
||||
assert {record["session_id"] for record in payload["sessions"]} == {"first", "continuation"}
|
||||
assert [record["session_id"] for record in store.export_session("continuation", include_lineage=False)["sessions"]] == ["continuation"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("source_backend", [DefaultSessionStore, SqliteTranscriptStore])
|
||||
@pytest.mark.parametrize("target_backend", [DefaultSessionStore, SqliteTranscriptStore])
|
||||
def test_cross_backend_restore_survives_reopen_without_window_truncation(stores, source_backend, target_backend):
|
||||
source = stores("source", source_backend)
|
||||
for index in range(6):
|
||||
assert source.add_message("reference", "user", f"message-{index}")
|
||||
assert source.set_gateway_info("reference", gateway_session_id="live", agent_id="agent")
|
||||
payload = source.export_all()
|
||||
assert len(payload["sessions"]) == 1
|
||||
target = stores("target", target_backend, active_window=2, retention="truncate")
|
||||
# An existing SQLite row prevents legacy JSON migration from masking a
|
||||
# restore incorrectly written to a sidecar file instead of the database.
|
||||
assert target.add_message("existing", "user", "Existing history")
|
||||
assert target.import_sessions(payload).imported == 1
|
||||
assert target.session_exists("reference")
|
||||
target.invalidate_cache()
|
||||
reopened = stores("target", target_backend, active_window=2, retention="truncate")
|
||||
for current in (target, reopened):
|
||||
assert [message["content"] for message in current.get_chat_history("reference")] == [f"message-{index}" for index in range(6)]
|
||||
session = current.get_session("reference")
|
||||
assert session.gateway_session_id is None
|
||||
assert session.agent_id is None
|
||||
if target_backend is SqliteTranscriptStore:
|
||||
from pathlib import Path
|
||||
assert list(Path(target.session_dir).glob("*.json")) == []
|
||||
|
||||
|
||||
def test_import_sqlite_write_failure_is_reported_and_preserves_existing_row(stores):
|
||||
source = stores("source", DefaultSessionStore)
|
||||
assert source.add_message("reference", "user", "New history")
|
||||
target = stores("target")
|
||||
assert target.add_message("reference", "user", "Original history")
|
||||
target._connect().execute("PRAGMA query_only=ON")
|
||||
report = target.import_sessions(source.export_all(), overwrite=True)
|
||||
assert report.imported == 0
|
||||
assert report.skipped_count == 1
|
||||
assert "write failed" in report.skipped[0]["reason"]
|
||||
target.invalidate_cache()
|
||||
assert target.get_chat_history("reference")[0]["content"] == "Original history"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("reset_live_fields", [True, False])
|
||||
def test_restore_preserves_archived_history_and_tool_calls(stores, reset_live_fields):
|
||||
tool_calls = [{"id": "call-1", "type": "function", "function": {"name": "lookup", "arguments": "{}"}}]
|
||||
payload = {"sessions": [{
|
||||
"session_id": "reference",
|
||||
"gateway_session_id": "live",
|
||||
"agent_id": "agent",
|
||||
"metadata": {"gateway_session_id": "live", "agent_id": "agent", "lineage_id": "shared"},
|
||||
"archived_messages": [{"role": "user", "content": "Archived", "timestamp": 1}],
|
||||
"messages": [{"role": "assistant", "content": "Tool call", "tool_calls": tool_calls}],
|
||||
}]}
|
||||
target = stores("target", active_window=1, retention="truncate")
|
||||
assert target.add_message("existing", "user", "Existing")
|
||||
assert target.import_sessions(payload, reset_live_fields=reset_live_fields).imported == 1
|
||||
record = stores("target").export_session("reference")["sessions"][0]
|
||||
assert record["messages"][0]["tool_calls"] == tool_calls
|
||||
assert record["archived_messages"][0]["content"] == "Archived"
|
||||
assert record["metadata"]["lineage_id"] == "shared"
|
||||
if reset_live_fields:
|
||||
assert record.get("gateway_session_id") is None
|
||||
assert record.get("agent_id") is None
|
||||
assert "gateway_session_id" not in record["metadata"]
|
||||
assert "agent_id" not in record["metadata"]
|
||||
else:
|
||||
assert record["gateway_session_id"] == "live"
|
||||
assert record["agent_id"] == "agent"
|
||||
assert record["metadata"]["gateway_session_id"] == "live"
|
||||
assert record["metadata"]["agent_id"] == "agent"
|
||||
|
||||
|
||||
def test_sqlite_restore_keeps_import_guards(stores):
|
||||
source = stores("source", DefaultSessionStore)
|
||||
assert source.add_message("reference", "user", "Original")
|
||||
payload = source.export_all()
|
||||
payload["sessions"].append(dict(payload["sessions"][0]))
|
||||
target = stores("target")
|
||||
report = target.import_sessions(payload)
|
||||
assert report.imported == 1
|
||||
assert "duplicate" in report.skipped[0]["reason"]
|
||||
assert target.import_sessions(payload).imported == 0
|
||||
assert target.import_sessions(payload, overwrite=True).imported == 1
|
||||
assert target.export_session("missing")["sessions"] == []
|
||||
payload["version"] = target.PORTABLE_VERSION + 1
|
||||
assert target.import_sessions(payload, overwrite=True).imported == 0
|
||||
assert target.get_chat_history("reference")[0]["content"] == "Original"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("overwrite", [False, True])
|
||||
def test_import_does_not_clobber_peer_created_after_initial_check(stores, monkeypatch, overwrite):
|
||||
target = stores("target")
|
||||
peer = stores("target")
|
||||
original = target._save_imported_session
|
||||
|
||||
def create_peer_before_write(*args, **kwargs):
|
||||
assert peer.add_message("reference", "user", "peer-created")
|
||||
return original(*args, **kwargs)
|
||||
|
||||
monkeypatch.setattr(target, "_save_imported_session", create_peer_before_write)
|
||||
payload = {"sessions": [{"session_id": "reference", "messages": [{"role": "user", "content": "imported"}]}]}
|
||||
report = target.import_sessions(payload, overwrite=overwrite)
|
||||
assert report.imported == int(overwrite)
|
||||
assert target.get_chat_history("reference")[0]["content"] == ("imported" if overwrite else "peer-created")
|
||||
if overwrite:
|
||||
assert target._cache["reference"].messages[0].content == "imported"
|
||||
else:
|
||||
assert report.skipped[0]["reason"] == "already exists (use overwrite)"
|
||||
|
||||
|
||||
def test_lineage_export_does_not_materialize_unrelated_transcripts(stores):
|
||||
store = stores("source")
|
||||
for sid in ("root", "continuation"):
|
||||
assert store.add_message(sid, "user", sid)
|
||||
assert store.update_session_metadata(sid, lineage_id="shared")
|
||||
for index in range(24):
|
||||
sid = f"unrelated-{index}"
|
||||
assert store.add_message(sid, "user", "x" * 256_000)
|
||||
assert store.update_session_metadata(sid, lineage_id="other", value=float("nan"))
|
||||
tracemalloc.start()
|
||||
try:
|
||||
payload = store.export_session("continuation")
|
||||
_, peak = tracemalloc.get_traced_memory()
|
||||
finally:
|
||||
tracemalloc.stop()
|
||||
assert {record["session_id"] for record in payload["sessions"]} == {"root", "continuation"}
|
||||
# The unrelated transcripts total over 6 MB. A single-record parse can
|
||||
# allocate its input and decoded content, but must not retain all rows.
|
||||
assert peak < 2_000_000
|
||||
@@ -61,6 +61,35 @@ class TestSqliteTranscriptStore:
|
||||
history = s2.get_chat_history("s1")
|
||||
assert history == [{"role": "user", "content": "persist me"}]
|
||||
|
||||
def test_boots_on_malformed_db(self, tmp_dir):
|
||||
"""A malformed transcript DB recovers instead of taking the store down.
|
||||
|
||||
Before Issue #5387's guard, opening a corrupt ``sessions.db`` raised an
|
||||
uncaught ``sqlite3.DatabaseError`` on the first ``CREATE TABLE``. The
|
||||
guard forensically backs up the bad bytes and lets a fresh, usable store
|
||||
open so durable session handling keeps working.
|
||||
"""
|
||||
db = os.path.join(tmp_dir, "sessions.db")
|
||||
s1 = SqliteTranscriptStore(session_dir=tmp_dir, db_path=db)
|
||||
s1.add_message("s1", "user", "before corruption")
|
||||
s1._conn.close() # release the file/WAL before corrupting on disk
|
||||
|
||||
# Corrupt the on-disk database so quick_check fails and it is unreadable.
|
||||
for sidecar in (db + "-wal", db + "-shm"):
|
||||
if os.path.exists(sidecar):
|
||||
os.remove(sidecar)
|
||||
with open(db, "r+b") as fh:
|
||||
fh.write(b"not a sqlite database" + b"\x00" * 300)
|
||||
|
||||
# A new store must open without raising and be usable.
|
||||
s2 = SqliteTranscriptStore(session_dir=tmp_dir, db_path=db)
|
||||
assert s2.add_message("s2", "user", "after recovery")
|
||||
assert s2.get_chat_history("s2") == [
|
||||
{"role": "user", "content": "after recovery"}
|
||||
]
|
||||
# The malformed bytes were preserved for forensics.
|
||||
assert any(f.startswith("sessions.db.corrupt-") for f in os.listdir(tmp_dir))
|
||||
|
||||
def test_session_exists_and_delete(self, tmp_dir):
|
||||
store = SqliteTranscriptStore(session_dir=tmp_dir)
|
||||
assert not store.session_exists("s1")
|
||||
@@ -175,6 +204,57 @@ class TestSqliteTranscriptStore:
|
||||
{"role": "user", "content": "legacy transcript"}
|
||||
]
|
||||
|
||||
def test_migration_skips_invalid_utf8_legacy_file(self, tmp_dir):
|
||||
"""A legacy file with invalid UTF-8 bytes is skipped (not fatal), and
|
||||
valid transcripts still migrate; the bad bytes are preserved."""
|
||||
legacy = DefaultSessionStore(session_dir=tmp_dir)
|
||||
legacy.add_message("old", "user", "legacy transcript")
|
||||
|
||||
bad_path = os.path.join(tmp_dir, "invalid.json")
|
||||
with open(bad_path, "wb") as handle:
|
||||
handle.write(b"\xff\xfe")
|
||||
|
||||
store = SqliteTranscriptStore(session_dir=tmp_dir)
|
||||
assert store.session_exists("old")
|
||||
assert store.get_chat_history("old") == [
|
||||
{"role": "user", "content": "legacy transcript"}
|
||||
]
|
||||
|
||||
with open(bad_path, "rb") as handle:
|
||||
assert handle.read() == b"\xff\xfe"
|
||||
|
||||
def test_migration_continues_after_invalid_utf8_ordered_first(self, tmp_dir):
|
||||
"""A bad file processed before valid ones must not abort later migrations.
|
||||
|
||||
Migration sorts filenames, so ``aaa_invalid.json`` is deterministically
|
||||
processed before the ``zeta`` session's JSON file — guaranteeing the
|
||||
bad-first scenario is actually exercised.
|
||||
"""
|
||||
legacy = DefaultSessionStore(session_dir=tmp_dir)
|
||||
legacy.add_message("zeta", "user", "valid transcript")
|
||||
|
||||
bad_path = os.path.join(tmp_dir, "aaa_invalid.json")
|
||||
with open(bad_path, "wb") as handle:
|
||||
handle.write(b"\xff\xfe")
|
||||
|
||||
import json as _json
|
||||
|
||||
json_files = sorted(f for f in os.listdir(tmp_dir) if f.endswith(".json"))
|
||||
assert json_files[0] == "aaa_invalid.json"
|
||||
valid_file = next(
|
||||
f for f in json_files
|
||||
if f != "aaa_invalid.json" and "zeta" in _json.dumps(
|
||||
_json.load(open(os.path.join(tmp_dir, f)))
|
||||
)
|
||||
)
|
||||
assert "aaa_invalid.json" < valid_file
|
||||
|
||||
store = SqliteTranscriptStore(session_dir=tmp_dir)
|
||||
assert store.session_exists("zeta")
|
||||
assert store.get_chat_history("zeta") == [
|
||||
{"role": "user", "content": "valid transcript"}
|
||||
]
|
||||
|
||||
|
||||
class TestTranscriptArchivedRecall:
|
||||
"""Issue #5031: transcript store must recall compacted (archived) turns."""
|
||||
|
||||
@@ -0,0 +1,76 @@
|
||||
"""The public title length cap applies to generated and fallback titles."""
|
||||
|
||||
import pytest
|
||||
|
||||
from praisonaiagents.session.title import generate_title, generate_title_async
|
||||
|
||||
|
||||
@pytest.mark.parametrize("max_length", [0, 1, 2, 3, 4, 60])
|
||||
@pytest.mark.parametrize("user_msg", ["", "?!.", "A long user request about debugging code"])
|
||||
@pytest.mark.asyncio
|
||||
async def test_sync_fallback_in_running_loop_respects_length(user_msg, max_length):
|
||||
title = generate_title(user_msg, "Response", max_length=max_length)
|
||||
assert len(title) <= max_length
|
||||
if max_length == 60:
|
||||
assert title
|
||||
|
||||
|
||||
@pytest.mark.parametrize("max_length", [0, 1, 2, 3, 4, 60])
|
||||
@pytest.mark.parametrize("response", ["A long generated title about Python", None])
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_generated_and_fallback_titles_respect_length(monkeypatch, max_length, response):
|
||||
import praisonaiagents.llm as llm_module
|
||||
|
||||
class LocalLLM:
|
||||
def __init__(self, **kwargs):
|
||||
pass
|
||||
|
||||
async def get_response_async(self, **kwargs):
|
||||
return response
|
||||
|
||||
monkeypatch.setattr(llm_module, "LLM", LocalLLM)
|
||||
title = await generate_title_async("", "Response", llm_model="test", max_length=max_length)
|
||||
assert len(title) <= max_length
|
||||
if max_length == 60:
|
||||
assert title == (response or "Chat Session")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("max_length", [0, -1])
|
||||
@pytest.mark.asyncio
|
||||
async def test_nonpositive_cap_skips_model_initialization(monkeypatch, max_length):
|
||||
import praisonaiagents.llm as llm_module
|
||||
|
||||
calls = []
|
||||
|
||||
class UnexpectedLLM:
|
||||
def __init__(self, **kwargs):
|
||||
calls.append(kwargs)
|
||||
|
||||
async def get_response_async(self, **kwargs):
|
||||
return "Unneeded title"
|
||||
|
||||
monkeypatch.setattr(llm_module, "LLM", UnexpectedLLM)
|
||||
assert await generate_title_async("Question", "Response", max_length=max_length) == ""
|
||||
assert generate_title("Question", "Response", max_length=max_length) == ""
|
||||
assert calls == []
|
||||
|
||||
|
||||
@pytest.mark.parametrize("user_msg", ["", "?!."])
|
||||
@pytest.mark.asyncio
|
||||
async def test_three_character_fallback_keeps_readable_prefix(user_msg):
|
||||
assert generate_title(user_msg, "Response", max_length=3) == "Cha"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_three_character_generated_title_keeps_readable_prefix(monkeypatch):
|
||||
import praisonaiagents.llm as llm_module
|
||||
|
||||
class LocalLLM:
|
||||
def __init__(self, **kwargs):
|
||||
pass
|
||||
|
||||
async def get_response_async(self, **kwargs):
|
||||
return "Python debugging"
|
||||
|
||||
monkeypatch.setattr(llm_module, "LLM", LocalLLM)
|
||||
assert await generate_title_async("Question", "Response", max_length=3) == "Pyt"
|
||||
@@ -270,22 +270,28 @@ def test_dispatch_skill_review_inline_runs_synchronously():
|
||||
|
||||
|
||||
def test_dispatch_skill_review_background_runs_off_path():
|
||||
import threading
|
||||
import time
|
||||
|
||||
agent = Agent(instructions="x", self_improve="background")
|
||||
done = []
|
||||
# Gate the review body so the assertion is on ordering, not wall-clock:
|
||||
# the caller must return *before* the review is released, proving the turn
|
||||
# ran off-path. This is deterministic — it never depends on how fast the
|
||||
# runner schedules the background thread (AGENTS.md §4.6: no timing-based
|
||||
# tests), unlike an absolute ``elapsed < 0.1`` bound that flaked on slow CI.
|
||||
release = threading.Event()
|
||||
|
||||
def slow_review(p, r, t):
|
||||
time.sleep(0.15)
|
||||
def gated_review(p, r, t):
|
||||
release.wait(5.0)
|
||||
done.append((p, r, t))
|
||||
|
||||
start = time.time()
|
||||
agent._dispatch_skill_review(slow_review, "p", "r", ["shell"])
|
||||
elapsed = time.time() - start
|
||||
# Caller is not blocked by the review turn.
|
||||
assert elapsed < 0.1
|
||||
# But the review still runs on the background runner.
|
||||
for _ in range(50):
|
||||
agent._dispatch_skill_review(gated_review, "p", "r", ["shell"])
|
||||
# Caller returned while the review is still blocked → it did not run inline.
|
||||
assert done == []
|
||||
# Release it and confirm the review still runs on the background runner.
|
||||
release.set()
|
||||
for _ in range(250):
|
||||
if done:
|
||||
break
|
||||
time.sleep(0.02)
|
||||
|
||||
@@ -11,6 +11,8 @@ Tests cover:
|
||||
import threading
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from praisonaiagents.storage.backends import FileBackend, SQLiteBackend, get_backend
|
||||
from praisonaiagents.storage.protocols import StorageBackendProtocol
|
||||
|
||||
@@ -33,7 +35,33 @@ class TestFileBackend:
|
||||
|
||||
loaded = backend.load("test_key")
|
||||
assert loaded == data
|
||||
|
||||
|
||||
def test_file_backend_load_invalid_utf8_returns_none(self, tmp_path, caplog):
|
||||
"""Undecodable bytes should warn and return None, not raise (Issue #5566)."""
|
||||
import logging
|
||||
|
||||
backend = FileBackend(storage_dir=str(tmp_path))
|
||||
|
||||
for raw in (b"\xff", b"\xe4\xb8", b"\xed\xa0\x80"):
|
||||
backend.save("bad", {"message": "original"})
|
||||
file_path = Path(tmp_path) / "bad.json"
|
||||
file_path.write_bytes(raw)
|
||||
|
||||
with caplog.at_level(logging.WARNING, logger="praisonaiagents.storage.backends"):
|
||||
caplog.clear()
|
||||
assert backend.load("bad") is None
|
||||
# A warning is emitted identifying the failed key
|
||||
assert any(
|
||||
rec.levelno == logging.WARNING and "Failed to load bad" in rec.getMessage()
|
||||
for rec in caplog.records
|
||||
)
|
||||
# Original bytes are preserved (load is non-destructive)
|
||||
assert file_path.read_bytes() == raw
|
||||
|
||||
# Valid neighboring keys still load
|
||||
backend.save("good", {"key": "value"})
|
||||
assert backend.load("good") == {"key": "value"}
|
||||
|
||||
def test_file_backend_exists(self, tmp_path):
|
||||
"""Test exists check."""
|
||||
backend = FileBackend(storage_dir=str(tmp_path))
|
||||
@@ -115,6 +143,29 @@ class TestFileBackend:
|
||||
backend = FileBackend(storage_dir=str(tmp_path))
|
||||
assert isinstance(backend, StorageBackendProtocol)
|
||||
|
||||
@pytest.mark.parametrize("pretty", [True, False])
|
||||
def test_file_backend_save_cleans_tmp_on_serialization_error(self, tmp_path, pretty):
|
||||
"""Serialization failures must not leave partial .tmp staging files."""
|
||||
backend = FileBackend(storage_dir=str(tmp_path), pretty=pretty)
|
||||
backend.save("good", {"value": 1})
|
||||
|
||||
circular = {}
|
||||
circular["self"] = circular
|
||||
with pytest.raises(ValueError):
|
||||
backend.save("circular", circular)
|
||||
|
||||
with pytest.raises(TypeError):
|
||||
backend.save("tuple_key", {("a", "b"): 1})
|
||||
|
||||
with pytest.raises((ValueError, TypeError)):
|
||||
backend.save("good", circular)
|
||||
|
||||
tmp_files = list(tmp_path.glob("*.tmp"))
|
||||
assert tmp_files == []
|
||||
assert backend.load("good") == {"value": 1}
|
||||
assert not backend.exists("circular")
|
||||
assert not backend.exists("tuple_key")
|
||||
|
||||
|
||||
class TestSQLiteBackend:
|
||||
"""Tests for SQLiteBackend."""
|
||||
@@ -248,6 +299,68 @@ class TestSQLiteBackend:
|
||||
loaded = backend2.load("key1")
|
||||
assert loaded["data"] == 1
|
||||
|
||||
@pytest.mark.parametrize("table_name", ["select", "123", "9table", "praison_storage"])
|
||||
def test_sqlite_backend_keyword_and_digit_table_names(self, tmp_path, table_name):
|
||||
"""Validator-accepted names that are SQL keywords or start with a digit
|
||||
must work across schema creation and all CRUD operations (Issue #5569)."""
|
||||
db_path = tmp_path / "test.db"
|
||||
backend = SQLiteBackend(db_path=str(db_path), table_name=table_name)
|
||||
|
||||
# save / upsert
|
||||
backend.save("key1", {"version": 1})
|
||||
backend.save("key1", {"version": 2})
|
||||
assert backend.load("key1")["version"] == 2
|
||||
|
||||
# exists
|
||||
assert backend.exists("key1")
|
||||
assert not backend.exists("missing")
|
||||
|
||||
# list_keys (full + prefix)
|
||||
backend.save("key2", {"data": 2})
|
||||
backend.save("other", {"data": 3})
|
||||
assert backend.list_keys() == ["key1", "key2", "other"]
|
||||
assert backend.list_keys(prefix="key") == ["key1", "key2"]
|
||||
|
||||
# delete
|
||||
assert backend.delete("other") is True
|
||||
assert not backend.exists("other")
|
||||
|
||||
# clear
|
||||
assert backend.clear() == 2
|
||||
assert backend.list_keys() == []
|
||||
|
||||
def test_sqlite_backend_reopen_with_auto_create_false(self, tmp_path):
|
||||
"""A keyword table name persists and reopens with auto_create=False."""
|
||||
db_path = tmp_path / "test.db"
|
||||
backend = SQLiteBackend(db_path=str(db_path), table_name="select")
|
||||
backend.save("key1", {"data": 1})
|
||||
backend.close()
|
||||
|
||||
backend2 = SQLiteBackend(
|
||||
db_path=str(db_path), table_name="select", auto_create=False
|
||||
)
|
||||
assert backend2.load("key1")["data"] == 1
|
||||
|
||||
def test_sqlite_backend_separate_tables_same_db(self, tmp_path):
|
||||
"""Quoted identifiers keep distinct tables isolated in one DB file."""
|
||||
db_path = tmp_path / "test.db"
|
||||
t1 = SQLiteBackend(db_path=str(db_path), table_name="select")
|
||||
t2 = SQLiteBackend(db_path=str(db_path), table_name="praison_storage")
|
||||
|
||||
t1.save("shared", {"from": "select"})
|
||||
t2.save("shared", {"from": "praison_storage"})
|
||||
|
||||
assert t1.load("shared")["from"] == "select"
|
||||
assert t2.load("shared")["from"] == "praison_storage"
|
||||
|
||||
def test_sqlite_backend_invalid_table_name_rejected(self, tmp_path):
|
||||
"""Names with disallowed characters are still rejected by the validator."""
|
||||
db_path = tmp_path / "test.db"
|
||||
with pytest.raises(ValueError):
|
||||
SQLiteBackend(db_path=str(db_path), table_name="bad name")
|
||||
with pytest.raises(ValueError):
|
||||
SQLiteBackend(db_path=str(db_path), table_name='drop";')
|
||||
|
||||
|
||||
class TestGetBackend:
|
||||
"""Tests for get_backend factory function."""
|
||||
|
||||
@@ -0,0 +1,30 @@
|
||||
"""Enumeration and clearing honor the complete configured filename suffix."""
|
||||
|
||||
import pytest
|
||||
|
||||
from praisonaiagents.storage.backends import FileBackend
|
||||
|
||||
|
||||
@pytest.mark.parametrize("suffix", [".data.json", "record", ".json", ""])
|
||||
@pytest.mark.parametrize("operation", ["list", "clear"])
|
||||
def test_configured_suffix_selects_saved_records(tmp_path, suffix, operation):
|
||||
store = FileBackend(storage_dir=str(tmp_path), suffix=suffix)
|
||||
for key in ("session_one", "session_two", "other"):
|
||||
store.save(key, {"key": key})
|
||||
unrelated = tmp_path / "unrelated.txt"
|
||||
unrelated.write_text("keep", encoding="utf-8")
|
||||
directory = tmp_path / ("directory" + suffix)
|
||||
directory.mkdir()
|
||||
assert store.exists("session_one")
|
||||
assert store.load("session_one") == {"key": "session_one"}
|
||||
if operation == "list":
|
||||
assert store.list_keys() == ["other", "session_one", "session_two"]
|
||||
assert store.list_keys("session_") == ["session_one", "session_two"]
|
||||
for key in store.list_keys():
|
||||
assert store.load(key) == {"key": key}
|
||||
else:
|
||||
assert store.clear() == 3
|
||||
assert not store.exists("session_one")
|
||||
assert store.list_keys() == []
|
||||
assert unrelated.read_text(encoding="utf-8") == "keep"
|
||||
assert directory.is_dir()
|
||||
@@ -0,0 +1,41 @@
|
||||
"""Retention must include sessions beyond the listing display limit."""
|
||||
|
||||
import os
|
||||
import time
|
||||
|
||||
import pytest
|
||||
|
||||
from praisonaiagents.storage.base import cleanup_old_sessions, list_json_sessions
|
||||
|
||||
|
||||
@pytest.mark.parametrize("boundary", ["age", "size"])
|
||||
@pytest.mark.parametrize("suffix", [".json", ".jsonl"])
|
||||
def test_cleanup_includes_oldest_session_after_ten_thousand_newer_files(tmp_path, boundary, suffix):
|
||||
oldest = tmp_path / ("oldest" + suffix)
|
||||
oldest.write_bytes(b"x")
|
||||
now = time.time()
|
||||
old_time = now - 90 * 24 * 60 * 60
|
||||
os.utime(oldest, (old_time, old_time))
|
||||
for index in range(10000):
|
||||
(tmp_path / f"new-{index:05d}{suffix}").write_bytes(b"")
|
||||
neighbor = tmp_path / "unrelated.txt"
|
||||
neighbor.write_bytes(b"keep")
|
||||
|
||||
if boundary == "age":
|
||||
deleted = cleanup_old_sessions(tmp_path, suffix=suffix, max_age_days=30, max_size_mb=100)
|
||||
else:
|
||||
# Only the oldest file contributes bytes to the quota; none exceed age.
|
||||
deleted = cleanup_old_sessions(tmp_path, suffix=suffix, max_age_days=365, max_size_mb=0)
|
||||
|
||||
assert deleted == 1
|
||||
assert not oldest.exists()
|
||||
assert len(list(tmp_path.glob("new-*" + suffix))) == 10000
|
||||
assert neighbor.read_bytes() == b"keep"
|
||||
|
||||
|
||||
def test_listing_keeps_default_limit_and_supports_unbounded_listing(tmp_path):
|
||||
for index in range(51):
|
||||
(tmp_path / f"session-{index:02d}.json").write_text("{}", encoding="utf-8")
|
||||
assert len(list_json_sessions(tmp_path)) == 50
|
||||
assert len(list_json_sessions(tmp_path, limit=3)) == 3
|
||||
assert len(list_json_sessions(tmp_path, limit=None)) == 51
|
||||
@@ -0,0 +1,54 @@
|
||||
"""Session counts use the UTF-8 writer format, independent of host locale."""
|
||||
|
||||
import builtins
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from praisonaiagents.storage.base import BaseJSONStore, list_json_sessions
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def legacy_text_locale(monkeypatch, tmp_path):
|
||||
original_open = builtins.open
|
||||
|
||||
def locale_open(file, mode="r", *args, **kwargs):
|
||||
if Path(file).parent == tmp_path and mode == "r" and not kwargs.get("encoding"):
|
||||
kwargs["encoding"] = "cp1252"
|
||||
return original_open(file, mode, *args, **kwargs)
|
||||
|
||||
# Exercise real decoding with a deterministic Windows-style default locale.
|
||||
monkeypatch.setattr(builtins, "open", locale_open)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("field", ["iterations", "messages", "items", "jsonl"])
|
||||
def test_utf8_session_counts_survive_legacy_locale(tmp_path, legacy_text_locale, field):
|
||||
values = ["丁 café 😀", "second"]
|
||||
suffix = ".jsonl" if field == "jsonl" else ".json"
|
||||
path = tmp_path / ("session" + suffix)
|
||||
if field == "jsonl":
|
||||
path.write_text("".join(json.dumps({"value": value}, ensure_ascii=False) + "\n" for value in values), encoding="utf-8")
|
||||
else:
|
||||
BaseJSONStore(path).save({field: values})
|
||||
before = path.read_bytes()
|
||||
|
||||
sessions = list_json_sessions(tmp_path, suffix=suffix)
|
||||
|
||||
assert len(sessions) == 1
|
||||
assert sessions[0].item_count == 2
|
||||
assert sessions[0].size_bytes == len(before)
|
||||
assert path.read_bytes() == before
|
||||
|
||||
|
||||
@pytest.mark.parametrize("suffix", [".json", ".jsonl"])
|
||||
@pytest.mark.parametrize("valid", [True, False])
|
||||
def test_ascii_and_unreadable_files_keep_existing_behavior(tmp_path, legacy_text_locale, suffix, valid):
|
||||
path = tmp_path / ("session" + suffix)
|
||||
content = b'{"items":[1,2]}' if suffix == ".json" else b'{"value":1}\n{"value":2}\n'
|
||||
path.write_bytes(content if valid else b"\x81")
|
||||
before = path.read_bytes()
|
||||
sessions = list_json_sessions(tmp_path, suffix=suffix)
|
||||
assert len(sessions) == 1
|
||||
assert sessions[0].item_count == (2 if valid else 0)
|
||||
assert path.read_bytes() == before
|
||||
@@ -0,0 +1,52 @@
|
||||
"""A failed backend constructor releases its real SQLite connection."""
|
||||
|
||||
import sqlite3
|
||||
|
||||
import pytest
|
||||
|
||||
from praisonaiagents.storage import sqlite as connector
|
||||
from praisonaiagents.storage.backends import SQLiteBackend
|
||||
|
||||
|
||||
@pytest.mark.parametrize("failure", ["reserved_name", "incompatible_schema"])
|
||||
def test_constructor_closes_connection_after_schema_failure(tmp_path, monkeypatch, failure):
|
||||
path = str(tmp_path / "store.db")
|
||||
with sqlite3.connect(path) as seed:
|
||||
seed.execute("CREATE TABLE preserved (value TEXT)")
|
||||
seed.execute("INSERT INTO preserved VALUES ('original')")
|
||||
if failure == "incompatible_schema":
|
||||
seed.execute("CREATE TABLE praison_storage (wrong_column TEXT)")
|
||||
seed.close()
|
||||
opened = []
|
||||
original = connector.connect
|
||||
|
||||
def connect(*args, **kwargs):
|
||||
conn = original(*args, **kwargs)
|
||||
opened.append(conn)
|
||||
return conn
|
||||
|
||||
monkeypatch.setattr(connector, "connect", connect)
|
||||
try:
|
||||
name = "sqlite_private" if failure == "reserved_name" else "praison_storage"
|
||||
with pytest.raises(sqlite3.OperationalError) as error:
|
||||
SQLiteBackend(db_path=path, table_name=name)
|
||||
expected = "reserved" if failure == "reserved_name" else "no such column"
|
||||
assert expected in str(error.value)
|
||||
assert len(opened) == 1
|
||||
with pytest.raises(sqlite3.ProgrammingError, match="closed database"):
|
||||
opened[0].execute("SELECT 1")
|
||||
with sqlite3.connect(path) as check:
|
||||
assert check.execute("SELECT value FROM preserved").fetchall() == [("original",)]
|
||||
check.close()
|
||||
finally:
|
||||
for conn in opened:
|
||||
conn.close()
|
||||
|
||||
|
||||
def test_successful_constructor_keeps_connection_usable(tmp_path):
|
||||
backend = SQLiteBackend(db_path=str(tmp_path / "store.db"))
|
||||
try:
|
||||
backend.save("key", {"value": 1})
|
||||
assert backend.load("key") == {"value": 1}
|
||||
finally:
|
||||
backend.close()
|
||||
@@ -173,3 +173,255 @@ def test_lazy_export_from_storage_package():
|
||||
from praisonaiagents.storage import sqlite_connect
|
||||
|
||||
assert sqlite_connect is connect
|
||||
|
||||
|
||||
# ── corruption detection / repair / backup (Issue #5387) ──────────────
|
||||
|
||||
|
||||
def _corrupt_file(path: str) -> None:
|
||||
"""Overwrite a database's header/pages with garbage so it is malformed."""
|
||||
with open(path, "r+b") as fh:
|
||||
fh.write(b"this is not a valid sqlite header" + b"\x00" * 200)
|
||||
|
||||
|
||||
def _make_db(path: str, rows: int = 5) -> None:
|
||||
import sqlite3
|
||||
|
||||
conn = sqlite3.connect(path)
|
||||
conn.execute("CREATE TABLE t (x INTEGER)")
|
||||
conn.executemany("INSERT INTO t VALUES (?)", [(i,) for i in range(rows)])
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
|
||||
def test_quick_check_reports_ok_for_healthy_db(tmp_path):
|
||||
from praisonaiagents.storage.sqlite import quick_check
|
||||
import sqlite3
|
||||
|
||||
db = str(tmp_path / "ok.db")
|
||||
_make_db(db)
|
||||
conn = sqlite3.connect(db)
|
||||
try:
|
||||
assert quick_check(conn) is True
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def test_quick_check_false_on_malformed_db(tmp_path):
|
||||
from praisonaiagents.storage.sqlite import quick_check
|
||||
import sqlite3
|
||||
|
||||
db = str(tmp_path / "bad.db")
|
||||
_make_db(db)
|
||||
_corrupt_file(db)
|
||||
conn = sqlite3.connect(db)
|
||||
try:
|
||||
assert quick_check(conn) is False
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def test_guard_noop_for_healthy_db(tmp_path):
|
||||
"""A healthy database is not backed up or altered when guarded."""
|
||||
db = str(tmp_path / "healthy.db")
|
||||
_make_db(db, rows=3)
|
||||
conn = connect(db, guard=True)
|
||||
try:
|
||||
assert conn.execute("SELECT COUNT(*) FROM t").fetchone()[0] == 3
|
||||
finally:
|
||||
conn.close()
|
||||
# No forensic copy created for a healthy file.
|
||||
assert not any(p.name.startswith("healthy.db.corrupt-") for p in tmp_path.iterdir())
|
||||
|
||||
|
||||
def test_guard_backs_up_malformed_db(tmp_path):
|
||||
"""A malformed file is forensically backed up before repair."""
|
||||
db = str(tmp_path / "state.db")
|
||||
_make_db(db)
|
||||
_corrupt_file(db)
|
||||
conn = connect(db, guard=True, repair=False)
|
||||
conn.close()
|
||||
backups = [p for p in tmp_path.iterdir() if p.name.startswith("state.db.corrupt-")]
|
||||
assert backups, "expected a forensic backup of the malformed file"
|
||||
|
||||
|
||||
def test_guard_never_raises_on_malformed_open(tmp_path):
|
||||
"""A malformed file must not surface an uncaught DatabaseError on open.
|
||||
|
||||
This is the core of Issue #5387: today a malformed file crashes the durable
|
||||
path. With the guard, ``connect`` returns a usable connection object (the
|
||||
malformed original is backed up first), never propagating the error.
|
||||
"""
|
||||
db = str(tmp_path / "state.db")
|
||||
_make_db(db)
|
||||
_corrupt_file(db)
|
||||
conn = connect(db, guard=True) # must not raise
|
||||
try:
|
||||
assert conn is not None
|
||||
finally:
|
||||
conn.close()
|
||||
backups = [p for p in tmp_path.iterdir() if p.name.startswith("state.db.corrupt-")]
|
||||
assert backups, "malformed file should have been forensically backed up"
|
||||
|
||||
|
||||
def test_repair_promotes_only_verified_snapshot(tmp_path):
|
||||
"""A repair that cannot rebuild an intact file never touches the original.
|
||||
|
||||
An unreadable ("file is not a database") original cannot be recovered by the
|
||||
online-backup API, so the canonical file must be left exactly as-is (only the
|
||||
forensic copy is added) rather than replaced by a half-written snapshot.
|
||||
"""
|
||||
from praisonaiagents.storage.sqlite import _repair_via_online_backup
|
||||
|
||||
db = str(tmp_path / "state.db")
|
||||
_make_db(db)
|
||||
_corrupt_file(db)
|
||||
with open(db, "rb") as fh:
|
||||
before = fh.read()
|
||||
assert _repair_via_online_backup(db) is False
|
||||
with open(db, "rb") as fh:
|
||||
after = fh.read()
|
||||
assert before == after # canonical bytes untouched by a failed repair
|
||||
assert not os.path.exists(db + ".repair-tmp") # snapshot cleaned up
|
||||
|
||||
|
||||
def test_repair_budget_is_bounded(tmp_path):
|
||||
"""A permanently-bad file exhausts its repair budget and stops retrying."""
|
||||
from praisonaiagents.storage.sqlite import (
|
||||
_repair_budget_ok,
|
||||
_record_repair_attempt,
|
||||
_MAX_REPAIR_ATTEMPTS,
|
||||
)
|
||||
|
||||
db = str(tmp_path / "state.db")
|
||||
_make_db(db)
|
||||
assert _repair_budget_ok(db) is True
|
||||
for _ in range(_MAX_REPAIR_ATTEMPTS):
|
||||
_record_repair_attempt(db)
|
||||
assert _repair_budget_ok(db) is False
|
||||
|
||||
|
||||
def test_forensic_backup_retention_capped(tmp_path):
|
||||
from praisonaiagents.storage.sqlite import _forensic_backup, _MAX_FORENSIC_BACKUPS
|
||||
import os
|
||||
import time
|
||||
|
||||
db = str(tmp_path / "state.db")
|
||||
for _ in range(_MAX_FORENSIC_BACKUPS + 3):
|
||||
_make_db(db)
|
||||
_corrupt_file(db)
|
||||
assert _forensic_backup(db) is not None
|
||||
os.remove(db) # start clean for the next iteration's fresh DB
|
||||
time.sleep(0.002)
|
||||
copies = [
|
||||
p
|
||||
for p in tmp_path.iterdir()
|
||||
if p.name.startswith("state.db.corrupt-")
|
||||
and not (p.name.endswith("-wal") or p.name.endswith("-shm"))
|
||||
]
|
||||
assert len(copies) <= _MAX_FORENSIC_BACKUPS
|
||||
|
||||
|
||||
def test_guard_off_by_default_leaves_malformed_untouched(tmp_path):
|
||||
"""Without guard, connect() does not back up or repair (unchanged behaviour)."""
|
||||
db = str(tmp_path / "state.db")
|
||||
_make_db(db)
|
||||
_corrupt_file(db)
|
||||
conn = connect(db) # guard defaults to False
|
||||
conn.close()
|
||||
assert not any(p.name.startswith("state.db.corrupt-") for p in tmp_path.iterdir())
|
||||
|
||||
|
||||
def test_guard_quarantines_unrepairable_so_store_can_boot(tmp_path):
|
||||
"""An unrepairable malformed file is cleared so a fresh DB opens usable.
|
||||
|
||||
This is the durable-path fix for Issue #5387: after the guard runs, the
|
||||
connection must be usable for schema creation (``CREATE TABLE``) instead of
|
||||
raising ``file is not a database`` on first query. The malformed bytes are
|
||||
preserved in a forensic copy.
|
||||
"""
|
||||
db = str(tmp_path / "state.db")
|
||||
_make_db(db)
|
||||
_corrupt_file(db)
|
||||
conn = connect(db, guard=True)
|
||||
try:
|
||||
# A fresh, empty database must be usable — this raised before the fix.
|
||||
conn.execute("CREATE TABLE s (session_id TEXT PRIMARY KEY)")
|
||||
conn.execute("INSERT INTO s VALUES ('a')")
|
||||
conn.commit()
|
||||
assert conn.execute("SELECT COUNT(*) FROM s").fetchone()[0] == 1
|
||||
finally:
|
||||
conn.close()
|
||||
backups = [p for p in tmp_path.iterdir() if p.name.startswith("state.db.corrupt-")]
|
||||
assert backups, "malformed bytes must be preserved before quarantine"
|
||||
|
||||
|
||||
def test_guard_leaves_locked_healthy_db_untouched(tmp_path, monkeypatch):
|
||||
"""A transient lock on a healthy DB is not mistaken for corruption."""
|
||||
from praisonaiagents.storage import sqlite as sqlite_mod
|
||||
|
||||
db = str(tmp_path / "state.db")
|
||||
_make_db(db, rows=4)
|
||||
|
||||
import sqlite3
|
||||
|
||||
real_connect = sqlite3.connect
|
||||
|
||||
class _LockingConn:
|
||||
def __init__(self, conn):
|
||||
self._conn = conn
|
||||
|
||||
def execute(self, sql, *a, **k):
|
||||
if isinstance(sql, str) and "quick_check" in sql:
|
||||
raise sqlite3.OperationalError("database is locked")
|
||||
return self._conn.execute(sql, *a, **k)
|
||||
|
||||
def __getattr__(self, name):
|
||||
return getattr(self._conn, name)
|
||||
|
||||
def _locking_connect(target, *args, **kwargs):
|
||||
conn = real_connect(target, *args, **kwargs)
|
||||
if str(target) == db:
|
||||
return _LockingConn(conn)
|
||||
return conn
|
||||
|
||||
monkeypatch.setattr(sqlite3, "connect", _locking_connect)
|
||||
# Guard must NOT back up or quarantine a merely-locked healthy file.
|
||||
sqlite_mod._guard_integrity(db, repair=True, backup=True)
|
||||
monkeypatch.undo()
|
||||
|
||||
assert not any(p.name.startswith("state.db.corrupt-") for p in tmp_path.iterdir())
|
||||
# The original healthy data is intact.
|
||||
conn = connect(db)
|
||||
try:
|
||||
assert conn.execute("SELECT COUNT(*) FROM t").fetchone()[0] == 4
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def test_guard_skips_repair_when_backup_fails(tmp_path, monkeypatch):
|
||||
"""If the forensic backup cannot be taken, repair must not run.
|
||||
|
||||
Repair's atomic promotion would otherwise destroy the only copy of the
|
||||
malformed bytes, so a failed backup must leave the canonical file untouched.
|
||||
"""
|
||||
from praisonaiagents.storage import sqlite as sqlite_mod
|
||||
|
||||
db = str(tmp_path / "state.db")
|
||||
_make_db(db)
|
||||
_corrupt_file(db)
|
||||
with open(db, "rb") as fh:
|
||||
before = fh.read()
|
||||
|
||||
monkeypatch.setattr(sqlite_mod, "_forensic_backup", lambda path: None)
|
||||
repaired = {"called": False}
|
||||
monkeypatch.setattr(
|
||||
sqlite_mod,
|
||||
"_repair_via_online_backup",
|
||||
lambda path: repaired.__setitem__("called", True) or True,
|
||||
)
|
||||
sqlite_mod._guard_integrity(db, repair=True, backup=True)
|
||||
|
||||
assert repaired["called"] is False, "repair must not run when backup failed"
|
||||
with open(db, "rb") as fh:
|
||||
assert fh.read() == before # canonical malformed bytes preserved
|
||||
@@ -59,6 +59,109 @@ class TestContextCompactorExists:
|
||||
assert result.compacted_tokens < result.original_tokens
|
||||
|
||||
|
||||
class TestCompactionMaxTokensResolution:
|
||||
"""When max_context_tokens is unset, trigger must size to the model window."""
|
||||
|
||||
def _make_agent(self, model):
|
||||
from praisonaiagents.agent.agent import Agent
|
||||
return Agent(name="t", instructions="t", llm=model)
|
||||
|
||||
def test_explicit_max_context_tokens_wins(self):
|
||||
from praisonaiagents.config.feature_configs import ExecutionConfig
|
||||
agent = self._make_agent("gpt-4o")
|
||||
cfg = ExecutionConfig(context_compaction=True, max_context_tokens=8000)
|
||||
assert agent._resolve_compaction_max_tokens(cfg) == 8000
|
||||
|
||||
def test_unset_derives_from_large_model_window(self):
|
||||
from praisonaiagents.config.feature_configs import ExecutionConfig
|
||||
agent = self._make_agent("gpt-4o") # 128k window
|
||||
cfg = ExecutionConfig(context_compaction=True) # max_context_tokens unset
|
||||
resolved = agent._resolve_compaction_max_tokens(cfg)
|
||||
# Must be far above the old flat 8000 for a large-context model.
|
||||
assert resolved > 8000
|
||||
|
||||
def test_unset_does_not_trigger_within_window(self):
|
||||
from praisonaiagents.config.feature_configs import ExecutionConfig
|
||||
from praisonaiagents.compaction import ContextCompactor
|
||||
agent = self._make_agent("gpt-4o") # 128k window
|
||||
cfg = ExecutionConfig(context_compaction=True)
|
||||
resolved = agent._resolve_compaction_max_tokens(cfg)
|
||||
# ~30k tokens of conversation should fit comfortably in a 128k model.
|
||||
msgs = [{"role": "user", "content": "A" * 4 * 30000}]
|
||||
compactor = ContextCompactor(max_tokens=resolved)
|
||||
assert compactor.needs_compaction(msgs) is False
|
||||
|
||||
def test_unset_triggers_when_budget_approached(self):
|
||||
from praisonaiagents.config.feature_configs import ExecutionConfig
|
||||
from praisonaiagents.compaction import ContextCompactor
|
||||
agent = self._make_agent("gpt-4o")
|
||||
cfg = ExecutionConfig(context_compaction=True)
|
||||
resolved = agent._resolve_compaction_max_tokens(cfg)
|
||||
# Well beyond the working budget must trigger compaction. Repeated
|
||||
# characters compress heavily under tiktoken (~8 chars/token), so use
|
||||
# 8× not the heuristic 4× when sizing synthetic overflow content.
|
||||
msgs = [{"role": "user", "content": "A" * 8 * (resolved + 50000)}]
|
||||
compactor = ContextCompactor(max_tokens=resolved)
|
||||
assert compactor.needs_compaction(msgs) is True
|
||||
|
||||
|
||||
class TestCompactionEntryPointWiring:
|
||||
"""Both sync and async entry points must pass the resolved model-aware
|
||||
budget to ContextCompactor when max_context_tokens is unset.
|
||||
|
||||
These guard against a regression where an entry point stops forwarding the
|
||||
resolved budget (e.g. reverting to a flat default), which the resolver-only
|
||||
tests above would not catch.
|
||||
"""
|
||||
|
||||
def _make_agent(self, model):
|
||||
from praisonaiagents.agent.agent import Agent
|
||||
return Agent(name="t", instructions="t", llm=model)
|
||||
|
||||
def _capture_max_tokens(self):
|
||||
captured = {}
|
||||
|
||||
class _FakeCompactor:
|
||||
def __init__(self, max_tokens=None, strategy=None, llm_summarize_fn=None):
|
||||
captured["max_tokens"] = max_tokens
|
||||
|
||||
def needs_compaction(self, messages):
|
||||
return False # short-circuit; we only assert the wired budget
|
||||
|
||||
return captured, _FakeCompactor
|
||||
|
||||
def test_sync_entry_point_wires_resolved_budget(self):
|
||||
from praisonaiagents.config.feature_configs import ExecutionConfig
|
||||
from praisonaiagents.hooks.types import HookEvent
|
||||
|
||||
agent = self._make_agent("gpt-4o") # 128k window
|
||||
agent.execution = ExecutionConfig(context_compaction=True) # unset limit
|
||||
expected = agent._resolve_compaction_max_tokens(agent.execution)
|
||||
assert expected > 8000 # model-aware, not the old flat default
|
||||
|
||||
captured, fake = self._capture_max_tokens()
|
||||
with patch("praisonaiagents.compaction.ContextCompactor", fake):
|
||||
agent._apply_context_compaction([{"role": "user", "content": "hi"}], HookEvent)
|
||||
assert captured["max_tokens"] == expected
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_entry_point_wires_resolved_budget(self):
|
||||
from praisonaiagents.config.feature_configs import ExecutionConfig
|
||||
from praisonaiagents.hooks.types import HookEvent
|
||||
|
||||
agent = self._make_agent("gpt-4o") # 128k window
|
||||
agent.execution = ExecutionConfig(context_compaction=True) # unset limit
|
||||
expected = agent._resolve_compaction_max_tokens(agent.execution)
|
||||
assert expected > 8000
|
||||
|
||||
captured, fake = self._capture_max_tokens()
|
||||
with patch("praisonaiagents.compaction.ContextCompactor", fake):
|
||||
await agent._apply_context_compaction_async(
|
||||
[{"role": "user", "content": "hi"}], HookEvent
|
||||
)
|
||||
assert captured["max_tokens"] == expected
|
||||
|
||||
|
||||
class TestCompactionHookEvents:
|
||||
"""BEFORE_COMPACTION and AFTER_COMPACTION must be defined in HookEvent."""
|
||||
|
||||
|
||||
@@ -0,0 +1,162 @@
|
||||
"""Tests for the canonical session-address grammar (Issue #5383)."""
|
||||
|
||||
import pytest
|
||||
|
||||
from praisonaiagents.gateway import (
|
||||
GATEWAY_METHODS,
|
||||
RESERVED_NAMES,
|
||||
SHORT_ID_LEN,
|
||||
OperatorScope,
|
||||
SessionRef,
|
||||
build_session_path,
|
||||
derive_short_id,
|
||||
parse_session_path,
|
||||
resolve_required_scope,
|
||||
slugify,
|
||||
)
|
||||
|
||||
|
||||
def test_derive_short_id_from_uuid_tail():
|
||||
key = "agent:assistant:a3f9c2e1-7b04-4c8e-9f21-1234abcd5678"
|
||||
short = derive_short_id(key)
|
||||
assert short == "1234abcd5678"[-SHORT_ID_LEN:]
|
||||
assert len(short) == SHORT_ID_LEN
|
||||
|
||||
|
||||
def test_derive_short_id_none_without_hex_tail():
|
||||
assert derive_short_id("report.js") is None
|
||||
assert derive_short_id("") is None
|
||||
assert derive_short_id("main") is None
|
||||
|
||||
|
||||
def test_derive_short_id_none_when_hex_not_at_end():
|
||||
# A hex run followed by non-hex chars (a file-like key) must NOT be
|
||||
# shortened, so it can round-trip losslessly via the literal escape hatch.
|
||||
assert derive_short_id("report-deadbeef.js") is None
|
||||
assert derive_short_id("deadbeef.txt") is None
|
||||
|
||||
|
||||
def test_file_like_key_with_hex_run_roundtrips():
|
||||
# Regression: ``report-deadbeef.js`` must not collapse to ``deadbeef``.
|
||||
key = "report-deadbeef.js"
|
||||
path = build_session_path("bot", key)
|
||||
assert path == "chat/bot/!report-deadbeef.js"
|
||||
ref = parse_session_path(path)
|
||||
assert ref == SessionRef(agent_id="bot", literal_key=key)
|
||||
|
||||
|
||||
def test_slugify():
|
||||
assert slugify("Quarterly Report") == "quarterly-report"
|
||||
assert slugify(" Hello, World!! ") == "hello-world"
|
||||
assert slugify("") is None
|
||||
assert slugify(" ") is None
|
||||
assert slugify("---") is None
|
||||
|
||||
|
||||
def test_slugify_truncates():
|
||||
long = "a" * 100
|
||||
out = slugify(long, max_len=10)
|
||||
assert out is not None and len(out) <= 10
|
||||
|
||||
|
||||
def test_build_path_hides_whole_uuid():
|
||||
key = "agent:assistant:a3f9c2e1-7b04-4c8e-9f21-000000000000"
|
||||
path = build_session_path("assistant", key, "Quarterly report")
|
||||
assert path is not None
|
||||
assert path.startswith("chat/assistant/quarterly-report-")
|
||||
# The whole UUID never appears whole in the URL.
|
||||
assert "a3f9c2e1-7b04-4c8e-9f21-000000000000" not in path
|
||||
|
||||
|
||||
def test_build_path_without_display_name():
|
||||
key = "agent:bot:aaaabbbbcccc"
|
||||
path = build_session_path("bot", key)
|
||||
assert path == "chat/bot/aaaabbbb"[: len("chat/bot/")] + derive_short_id(key)
|
||||
|
||||
|
||||
def test_build_path_custom_namespace():
|
||||
path = build_session_path("bot", "x-11112222", namespace="thread")
|
||||
assert path is not None and path.startswith("thread/bot/")
|
||||
|
||||
|
||||
def test_build_path_empty_inputs():
|
||||
assert build_session_path("", "key") is None
|
||||
assert build_session_path("agent", "") is None
|
||||
|
||||
|
||||
def test_roundtrip_slug_short():
|
||||
key = "agent:assistant:a3f9c2e1-7b04-4c8e-9f21-1234abcd5678"
|
||||
path = build_session_path("assistant", key, "Quarterly report")
|
||||
ref = parse_session_path(path)
|
||||
assert ref == SessionRef(
|
||||
agent_id="assistant",
|
||||
short_id=derive_short_id(key),
|
||||
slug_hint="quarterly-report",
|
||||
)
|
||||
|
||||
|
||||
def test_parse_no_namespace():
|
||||
ref = parse_session_path("assistant/quarterly-report-1234abcd")
|
||||
assert ref.agent_id == "assistant"
|
||||
assert ref.short_id == "1234abcd"
|
||||
assert ref.slug_hint == "quarterly-report"
|
||||
|
||||
|
||||
def test_parse_reserved_sentinel():
|
||||
for name in RESERVED_NAMES:
|
||||
ref = parse_session_path(f"chat/assistant/{name}")
|
||||
assert ref == SessionRef(agent_id="assistant", slug_hint=name)
|
||||
|
||||
|
||||
def test_parse_short_id_only():
|
||||
ref = parse_session_path("chat/assistant/deadbeef")
|
||||
assert ref.short_id == "deadbeef"
|
||||
assert ref.slug_hint is None
|
||||
|
||||
|
||||
def test_literal_key_escape_hatch_roundtrips():
|
||||
path = build_session_path("bot", "report.js")
|
||||
assert path == "chat/bot/!report.js"
|
||||
ref = parse_session_path(path)
|
||||
assert ref == SessionRef(agent_id="bot", literal_key="report.js")
|
||||
|
||||
|
||||
def test_file_like_key_not_a_static_asset():
|
||||
# A key that looks like an asset is escaped behind the ``!`` hatch, so a
|
||||
# router never mistakes the final segment for ``report.js`` on disk.
|
||||
path = build_session_path("bot", "weird/key.js")
|
||||
ref = parse_session_path(path)
|
||||
assert ref.literal_key == "weird/key.js"
|
||||
|
||||
|
||||
def test_slug_hint_only_when_no_short_id():
|
||||
ref = parse_session_path("chat/assistant/my-label")
|
||||
# "my-label" has no 8-hex short id, so it becomes a slug hint the resolver
|
||||
# matches against session labels.
|
||||
assert ref.short_id is None
|
||||
assert ref.slug_hint == "my-label"
|
||||
|
||||
|
||||
def test_parse_invalid():
|
||||
assert parse_session_path("") is None
|
||||
assert parse_session_path("just-one-segment") is None
|
||||
assert parse_session_path("/") is None
|
||||
|
||||
|
||||
def test_parse_rejects_surplus_segments():
|
||||
# A canonical address is at most ``<ns>/<agent>/<ref>``; a longer path is
|
||||
# malformed and must be rejected rather than silently using its last two
|
||||
# segments (which would resolve to the wrong agent).
|
||||
assert parse_session_path("chat/other/assistant/deadbeef") is None
|
||||
assert parse_session_path("a/b/c/d/e") is None
|
||||
|
||||
|
||||
def test_agent_id_with_special_chars_roundtrips():
|
||||
path = build_session_path("team/assistant", "x-11112222")
|
||||
ref = parse_session_path(path)
|
||||
assert ref.agent_id == "team/assistant"
|
||||
|
||||
|
||||
def test_session_resolve_registered_read():
|
||||
assert "session.resolve" in GATEWAY_METHODS
|
||||
assert resolve_required_scope("session.resolve") == OperatorScope.READ
|
||||
@@ -636,3 +636,83 @@ class TestAttachmentConfig:
|
||||
assert m.gateway.attachments.max_attachments == 3
|
||||
assert m.gateway.attachments.max_attachment_bytes == 2048
|
||||
assert m.gateway.attachments.allowed_types == ["image/"]
|
||||
|
||||
|
||||
class TestDrainWatchdogParity:
|
||||
"""Drain/watchdog GatewayConfig ↔ YAML parity and round-trip (#5362)."""
|
||||
|
||||
def test_python_fields_default_off(self):
|
||||
from praisonaiagents.gateway import GatewayConfig
|
||||
|
||||
gc = GatewayConfig()
|
||||
assert gc.drain_timeout is None
|
||||
assert gc.watchdog is False
|
||||
assert gc.watchdog_timeout is None
|
||||
|
||||
def test_watchdog_timeout_survives_round_trip(self):
|
||||
"""to_dict → from_dict must not silently drop watchdog_timeout."""
|
||||
from praisonaiagents.gateway.config import (
|
||||
GatewayConfig,
|
||||
MultiChannelGatewayConfig,
|
||||
)
|
||||
|
||||
gc = GatewayConfig(watchdog=True, watchdog_timeout=60.0)
|
||||
round_tripped = MultiChannelGatewayConfig.from_dict(
|
||||
{"gateway": gc.to_dict()}
|
||||
).gateway
|
||||
assert round_tripped.watchdog is True
|
||||
assert round_tripped.watchdog_timeout == 60.0
|
||||
|
||||
def test_watchdog_timeout_scalar_key_parsed(self):
|
||||
from praisonaiagents.gateway.config import MultiChannelGatewayConfig
|
||||
|
||||
m = MultiChannelGatewayConfig.from_dict(
|
||||
{"gateway": {"watchdog": True, "watchdog_timeout": "15"}}
|
||||
)
|
||||
assert m.gateway.watchdog is True
|
||||
assert m.gateway.watchdog_timeout == 15.0
|
||||
|
||||
def test_watchdog_block_still_derives_budget(self):
|
||||
from praisonaiagents.gateway.config import MultiChannelGatewayConfig
|
||||
|
||||
m = MultiChannelGatewayConfig.from_dict(
|
||||
{
|
||||
"gateway": {
|
||||
"watchdog": {
|
||||
"enabled": True,
|
||||
"liveness_interval": 5.0,
|
||||
"liveness_strikes": 4,
|
||||
}
|
||||
}
|
||||
}
|
||||
)
|
||||
assert m.gateway.watchdog is True
|
||||
assert m.gateway.watchdog_timeout == 20.0
|
||||
|
||||
def test_scalar_watchdog_timeout_non_positive_is_none(self):
|
||||
from praisonaiagents.gateway.config import MultiChannelGatewayConfig
|
||||
|
||||
m = MultiChannelGatewayConfig.from_dict(
|
||||
{"gateway": {"watchdog": True, "watchdog_timeout": 0}}
|
||||
)
|
||||
assert m.gateway.watchdog_timeout is None
|
||||
|
||||
def test_drain_timeout_round_trip(self):
|
||||
from praisonaiagents.gateway.config import (
|
||||
GatewayConfig,
|
||||
MultiChannelGatewayConfig,
|
||||
)
|
||||
|
||||
gc = GatewayConfig(drain_timeout=20.0)
|
||||
round_tripped = MultiChannelGatewayConfig.from_dict(
|
||||
{"gateway": gc.to_dict()}
|
||||
).gateway
|
||||
assert round_tripped.drain_timeout == 20.0
|
||||
|
||||
def test_scalar_watchdog_false_parsed(self):
|
||||
from praisonaiagents.gateway.config import MultiChannelGatewayConfig
|
||||
|
||||
m = MultiChannelGatewayConfig.from_dict(
|
||||
{"gateway": {"watchdog": False}}
|
||||
)
|
||||
assert m.gateway.watchdog is False
|
||||
@@ -0,0 +1,194 @@
|
||||
"""Unit tests for trusted-proxy ingress attribution (Issue #5312).
|
||||
|
||||
Covers the pure, core-side contract: :func:`parse_forwarded_for`,
|
||||
:func:`resolve_ingress_attribution`, and the :class:`IngressAttribution`
|
||||
dataclass. No I/O — the trust decision must fail closed for anything
|
||||
proxy-shaped but unattributable, and resolve the real client only across
|
||||
operator-declared trusted hops.
|
||||
"""
|
||||
|
||||
from praisonaiagents.gateway import (
|
||||
IngressAttribution,
|
||||
parse_forwarded_for,
|
||||
resolve_ingress_attribution,
|
||||
)
|
||||
|
||||
|
||||
def test_parse_forwarded_for_splits_and_trims():
|
||||
assert parse_forwarded_for("1.2.3.4, 10.0.0.1 , 10.0.0.2") == [
|
||||
"1.2.3.4",
|
||||
"10.0.0.1",
|
||||
"10.0.0.2",
|
||||
]
|
||||
|
||||
|
||||
def test_parse_forwarded_for_empty():
|
||||
assert parse_forwarded_for(None) == []
|
||||
assert parse_forwarded_for("") == []
|
||||
assert parse_forwarded_for(" , ") == []
|
||||
|
||||
|
||||
def test_direct_local_no_headers():
|
||||
attr = resolve_ingress_attribution(peer_ip="127.0.0.1")
|
||||
assert attr == IngressAttribution(
|
||||
trust="direct-local", client_ip="127.0.0.1", via_proxy=False, fail_closed=False
|
||||
)
|
||||
|
||||
|
||||
def test_direct_remote_no_headers():
|
||||
attr = resolve_ingress_attribution(peer_ip="203.0.113.9")
|
||||
assert attr.trust == "direct-remote"
|
||||
assert attr.client_ip == "203.0.113.9"
|
||||
assert attr.via_proxy is False
|
||||
assert attr.fail_closed is False
|
||||
|
||||
|
||||
def test_unattributable_proxy_fails_closed():
|
||||
# Directly-reachable gateway: an attacker sets X-Forwarded-For but the peer
|
||||
# is not a declared trusted proxy -> never trust the header, key on peer.
|
||||
attr = resolve_ingress_attribution(
|
||||
peer_ip="203.0.113.9",
|
||||
forwarded_for=["1.1.1.1"],
|
||||
trusted_proxies=[],
|
||||
)
|
||||
assert attr.trust == "unattributable-proxy"
|
||||
assert attr.client_ip == "203.0.113.9"
|
||||
assert attr.via_proxy is True
|
||||
assert attr.fail_closed is True
|
||||
|
||||
|
||||
def test_untrusted_peer_with_trusted_set_still_fails_closed():
|
||||
attr = resolve_ingress_attribution(
|
||||
peer_ip="203.0.113.9",
|
||||
forwarded_for=["1.1.1.1"],
|
||||
trusted_proxies=["10.0.0.0/8"],
|
||||
)
|
||||
assert attr.trust == "unattributable-proxy"
|
||||
assert attr.fail_closed is True
|
||||
assert attr.client_ip == "203.0.113.9"
|
||||
|
||||
|
||||
def test_trusted_proxy_resolves_real_client():
|
||||
attr = resolve_ingress_attribution(
|
||||
peer_ip="10.0.0.5",
|
||||
forwarded_for=["203.0.113.9", "10.0.0.5"],
|
||||
trusted_proxies=["10.0.0.0/8"],
|
||||
)
|
||||
assert attr.trust == "trusted-proxy"
|
||||
assert attr.client_ip == "203.0.113.9"
|
||||
assert attr.via_proxy is True
|
||||
assert attr.fail_closed is False
|
||||
|
||||
|
||||
def test_trusted_proxy_peels_chain_to_first_untrusted():
|
||||
# Two trusted hops appended; the real client is the first untrusted address
|
||||
# walking right->left.
|
||||
attr = resolve_ingress_attribution(
|
||||
peer_ip="10.0.0.1",
|
||||
forwarded_for=["198.51.100.7", "10.0.0.2", "10.0.0.1"],
|
||||
trusted_proxies=["10.0.0.0/8"],
|
||||
)
|
||||
assert attr.client_ip == "198.51.100.7"
|
||||
assert attr.trust == "trusted-proxy"
|
||||
|
||||
|
||||
def test_spoofed_prefix_not_peeled_past_untrusted_hop():
|
||||
# An attacker prepends a spoofed trusted-looking address; because the first
|
||||
# untrusted hop stops the walk, the spoof cannot masquerade as the client.
|
||||
attr = resolve_ingress_attribution(
|
||||
peer_ip="10.0.0.1",
|
||||
forwarded_for=["10.0.0.9", "198.51.100.7", "10.0.0.1"],
|
||||
trusted_proxies=["10.0.0.0/8"],
|
||||
)
|
||||
assert attr.client_ip == "198.51.100.7"
|
||||
|
||||
|
||||
def test_bare_ip_trusted_entry():
|
||||
attr = resolve_ingress_attribution(
|
||||
peer_ip="192.168.1.10",
|
||||
forwarded_for=["203.0.113.9", "192.168.1.10"],
|
||||
trusted_proxies=["192.168.1.10"],
|
||||
)
|
||||
assert attr.trust == "trusted-proxy"
|
||||
assert attr.client_ip == "203.0.113.9"
|
||||
|
||||
|
||||
def test_tunnel_classification_for_loopback_trusted_hop():
|
||||
# A loopback sidecar tunnel (Tailscale/Cloudflare) declared trusted.
|
||||
attr = resolve_ingress_attribution(
|
||||
peer_ip="127.0.0.1",
|
||||
forwarded_for=["203.0.113.9"],
|
||||
trusted_proxies=["127.0.0.1"],
|
||||
real_ip=None,
|
||||
)
|
||||
assert attr.trust == "tunnel"
|
||||
assert attr.client_ip == "203.0.113.9"
|
||||
assert attr.fail_closed is False
|
||||
|
||||
|
||||
def test_real_ip_used_when_chain_all_trusted():
|
||||
attr = resolve_ingress_attribution(
|
||||
peer_ip="10.0.0.5",
|
||||
forwarded_for=["10.0.0.6", "10.0.0.5"],
|
||||
real_ip="203.0.113.9",
|
||||
trusted_proxies=["10.0.0.0/8"],
|
||||
)
|
||||
assert attr.client_ip == "203.0.113.9"
|
||||
|
||||
|
||||
def test_real_ip_only_with_trusted_peer():
|
||||
attr = resolve_ingress_attribution(
|
||||
peer_ip="10.0.0.5",
|
||||
forwarded_for=[],
|
||||
real_ip="203.0.113.9",
|
||||
trusted_proxies=["10.0.0.0/8"],
|
||||
)
|
||||
assert attr.trust == "trusted-proxy"
|
||||
assert attr.client_ip == "203.0.113.9"
|
||||
|
||||
|
||||
def test_real_ip_only_untrusted_peer_fails_closed():
|
||||
attr = resolve_ingress_attribution(
|
||||
peer_ip="203.0.113.9",
|
||||
forwarded_for=[],
|
||||
real_ip="1.1.1.1",
|
||||
trusted_proxies=["10.0.0.0/8"],
|
||||
)
|
||||
assert attr.fail_closed is True
|
||||
assert attr.client_ip == "203.0.113.9"
|
||||
|
||||
|
||||
def test_malformed_client_hop_fails_closed():
|
||||
# A trusted proxy forwards a non-IP token (e.g. "unknown" or an injected
|
||||
# label). It must NOT become the per-IP subject — fail closed to the peer.
|
||||
attr = resolve_ingress_attribution(
|
||||
peer_ip="10.0.0.5",
|
||||
forwarded_for=["unknown", "10.0.0.5"],
|
||||
trusted_proxies=["10.0.0.0/8"],
|
||||
)
|
||||
assert attr.trust == "unattributable-proxy"
|
||||
assert attr.fail_closed is True
|
||||
assert attr.client_ip == "10.0.0.5"
|
||||
|
||||
|
||||
def test_malformed_real_ip_fallback_fails_closed():
|
||||
# Chain entirely trusted; a garbage X-Real-IP must not key policy.
|
||||
attr = resolve_ingress_attribution(
|
||||
peer_ip="10.0.0.5",
|
||||
forwarded_for=["10.0.0.6", "10.0.0.5"],
|
||||
real_ip="not-an-ip",
|
||||
trusted_proxies=["10.0.0.0/8"],
|
||||
)
|
||||
assert attr.fail_closed is True
|
||||
assert attr.client_ip == "10.0.0.5"
|
||||
|
||||
|
||||
def test_ipv6_client_hop_is_valid():
|
||||
attr = resolve_ingress_attribution(
|
||||
peer_ip="10.0.0.5",
|
||||
forwarded_for=["2001:db8::1", "10.0.0.5"],
|
||||
trusted_proxies=["10.0.0.0/8"],
|
||||
)
|
||||
assert attr.trust == "trusted-proxy"
|
||||
assert attr.client_ip == "2001:db8::1"
|
||||
assert attr.fail_closed is False
|
||||
@@ -8,12 +8,17 @@ do not kill the interpreter.
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
|
||||
import pytest
|
||||
|
||||
from praisonaiagents.gateway import LoopWatchdog, LoopWatchdogPolicy
|
||||
from praisonaiagents.gateway import (
|
||||
LoopWatchdog,
|
||||
LoopWatchdogPolicy,
|
||||
StartupWatchdog,
|
||||
)
|
||||
from praisonaiagents.gateway.protocols import GATEWAY_RESTART_EXIT_CODE
|
||||
|
||||
|
||||
@@ -381,3 +386,218 @@ def test_non_positive_deadline_is_noop():
|
||||
assert wd.deadline_armed is False
|
||||
wd.arm_deadline(-5)
|
||||
assert wd.deadline_armed is False
|
||||
|
||||
|
||||
# ── Startup-phase watchdog (Issue #5265) ──
|
||||
|
||||
|
||||
def test_startup_validation():
|
||||
with pytest.raises(ValueError):
|
||||
StartupWatchdog(on_expire="boom")
|
||||
with pytest.raises(ValueError):
|
||||
StartupWatchdog(max_extensions=-1)
|
||||
|
||||
|
||||
def test_startup_defaults_and_initial_state():
|
||||
sw = StartupWatchdog()
|
||||
assert sw.on_expire == "dump_and_exit"
|
||||
assert sw.exit_code == GATEWAY_RESTART_EXIT_CODE
|
||||
assert sw.armed is False
|
||||
assert sw.expired is False
|
||||
assert sw.confirmed is False
|
||||
|
||||
|
||||
def test_startup_non_positive_deadline_is_noop():
|
||||
sw = StartupWatchdog()
|
||||
sw.arm(deadline_s=0)
|
||||
assert sw.armed is False
|
||||
sw.arm(deadline_s=-1)
|
||||
assert sw.armed is False
|
||||
sw.arm(deadline_s=float("nan"))
|
||||
assert sw.armed is False
|
||||
|
||||
|
||||
def test_startup_confirm_when_not_armed_is_safe():
|
||||
sw = StartupWatchdog()
|
||||
sw.confirm_loop_live() # must not raise
|
||||
assert sw.confirmed is True
|
||||
assert sw.expired is False
|
||||
|
||||
|
||||
def test_startup_progress_when_not_armed_is_safe():
|
||||
sw = StartupWatchdog()
|
||||
sw.report_startup_progress("noop") # must not raise
|
||||
assert sw.armed is False
|
||||
|
||||
|
||||
def test_startup_confirm_before_deadline_no_exit():
|
||||
"""A startup that confirms the loop live before the deadline never exits."""
|
||||
sw = StartupWatchdog(on_expire="dump_and_exit")
|
||||
sw.arm(deadline_s=5.0)
|
||||
assert sw.armed is True
|
||||
sw.confirm_loop_live() # loop came up promptly
|
||||
assert sw.armed is False
|
||||
assert sw.confirmed is True
|
||||
assert sw.expired is False
|
||||
|
||||
|
||||
def test_startup_arm_is_idempotent():
|
||||
sw = StartupWatchdog(on_expire="dump_only")
|
||||
sw.arm(deadline_s=5.0)
|
||||
first = sw._thread
|
||||
sw.arm(deadline_s=5.0) # no-op while armed
|
||||
assert sw._thread is first
|
||||
sw.confirm_loop_live()
|
||||
|
||||
|
||||
def test_startup_deadline_expires_and_dumps(tmp_path):
|
||||
"""A wedged startup expires, dumps stacks, records expiry (dump_only)."""
|
||||
dump = tmp_path / "startup.txt"
|
||||
sw = StartupWatchdog(on_expire="dump_only", dump_file=str(dump))
|
||||
sw.arm(deadline_s=0.05)
|
||||
thread = sw._thread
|
||||
assert thread is not None
|
||||
thread.join(timeout=5.0)
|
||||
assert thread.is_alive() is False
|
||||
assert sw.expired is True
|
||||
assert dump.exists()
|
||||
assert "startup did not confirm" in dump.read_text()
|
||||
|
||||
|
||||
def test_startup_progress_extends_deadline():
|
||||
"""Progress reports keep a slow-but-alive startup from being tripped."""
|
||||
sw = StartupWatchdog(on_expire="dump_only", max_extensions=10)
|
||||
sw.arm(deadline_s=0.1)
|
||||
# Nudge the deadline several times across a span longer than the base
|
||||
# deadline; a live startup reporting progress must not expire.
|
||||
for _ in range(4):
|
||||
time.sleep(0.05)
|
||||
sw.report_startup_progress("phase")
|
||||
assert sw.expired is False
|
||||
assert sw.armed is True
|
||||
sw.confirm_loop_live()
|
||||
assert sw.expired is False
|
||||
|
||||
|
||||
def test_startup_extensions_are_bounded():
|
||||
"""Once extensions are exhausted, a still-wedged phase trips the deadline."""
|
||||
sw = StartupWatchdog(on_expire="dump_only", max_extensions=2)
|
||||
sw.arm(deadline_s=0.05)
|
||||
sw.report_startup_progress("a")
|
||||
sw.report_startup_progress("b")
|
||||
# Third report is refused (bound reached); the phase then wedges.
|
||||
sw.report_startup_progress("c")
|
||||
thread = sw._thread
|
||||
assert thread is not None
|
||||
thread.join(timeout=5.0)
|
||||
assert sw.expired is True
|
||||
# Budget accounting is done by the watchdog thread (so batched reports never
|
||||
# lose the resets they paid for); it is exact once the thread has drained.
|
||||
assert sw._extensions_used == 2
|
||||
|
||||
|
||||
def test_startup_batched_progress_does_not_waste_budget():
|
||||
"""Reports batched before a wake each still buy a real deadline reset.
|
||||
|
||||
Regression for the review finding "progress reports lose extensions": with
|
||||
per-call accounting, several reports landing between two wakes would each
|
||||
spend budget but only reset the window once, letting a later legitimate
|
||||
phase exhaust the budget early. With thread-side draining, a single slow
|
||||
boot that stays under the extension bound must survive well past its base
|
||||
deadline.
|
||||
"""
|
||||
sw = StartupWatchdog(on_expire="dump_only", max_extensions=6)
|
||||
sw.arm(deadline_s=0.08)
|
||||
# Fire a burst of reports (likely batched) then keep reporting across a
|
||||
# span far longer than the base deadline; must not expire while affordable.
|
||||
sw.report_startup_progress("burst-1")
|
||||
sw.report_startup_progress("burst-2")
|
||||
for _ in range(4):
|
||||
time.sleep(0.05)
|
||||
sw.report_startup_progress("phase")
|
||||
assert sw.expired is False
|
||||
assert sw.armed is True
|
||||
sw.confirm_loop_live()
|
||||
assert sw.expired is False
|
||||
|
||||
|
||||
def test_startup_confirm_suppresses_in_flight_exit():
|
||||
"""A confirm racing an in-flight expiry must not call os._exit."""
|
||||
sw = StartupWatchdog(on_expire="dump_and_exit")
|
||||
stop = threading.Event()
|
||||
stop.set() # simulate confirm_loop_live() having raced in
|
||||
exited = []
|
||||
original_exit = os._exit
|
||||
os._exit = lambda code: exited.append(code)
|
||||
try:
|
||||
sw._on_expire(stop) # must observe its own stop flag and stay inert
|
||||
finally:
|
||||
os._exit = original_exit
|
||||
assert exited == []
|
||||
assert sw.expired is False
|
||||
|
||||
|
||||
def test_startup_expiry_takes_active_exit_path():
|
||||
"""The active exit path (stop NOT set) must dump and call os._exit.
|
||||
|
||||
The in-flight-suppression test only exercises the first stop check; this one
|
||||
covers the real wedge path so a regression that silently stopped exiting
|
||||
would be caught (review: "exit race remains untested").
|
||||
"""
|
||||
sw = StartupWatchdog(on_expire="dump_and_exit", exit_code=42)
|
||||
sw._dump_stacks_on_expire = True
|
||||
stop = threading.Event() # deliberately NOT set: a genuine wedge
|
||||
dumped = []
|
||||
exited = []
|
||||
sw._dump_stacks = lambda: dumped.append(True)
|
||||
original_exit = os._exit
|
||||
os._exit = lambda code: exited.append(code)
|
||||
try:
|
||||
sw._on_expire(stop)
|
||||
finally:
|
||||
os._exit = original_exit
|
||||
assert dumped == [True]
|
||||
assert exited == [42]
|
||||
assert sw.expired is True
|
||||
|
||||
|
||||
def test_startup_confirm_during_flush_suppresses_exit():
|
||||
"""A confirm landing while the pre-exit flush blocks must cancel the exit.
|
||||
|
||||
Simulates a full-pipe stderr.flush() that blocks long enough for
|
||||
confirm_loop_live() to set the stop event; the final re-check before
|
||||
os._exit must then honour it (review: "confirmation cannot prevent late
|
||||
exit").
|
||||
"""
|
||||
sw = StartupWatchdog(on_expire="dump_and_exit")
|
||||
sw._dump_stacks_on_expire = False
|
||||
stop = threading.Event() # NOT set initially: wedge path is taken
|
||||
|
||||
exited = []
|
||||
original_exit = os._exit
|
||||
original_flush = sys.stderr.flush
|
||||
|
||||
def blocking_flush():
|
||||
# The confirm races in "while" the flush is in progress.
|
||||
stop.set()
|
||||
|
||||
os._exit = lambda code: exited.append(code)
|
||||
sys.stderr.flush = blocking_flush
|
||||
try:
|
||||
sw._on_expire(stop)
|
||||
finally:
|
||||
os._exit = original_exit
|
||||
sys.stderr.flush = original_flush
|
||||
assert exited == [] # final re-check honoured the racing confirm
|
||||
assert sw.expired is True
|
||||
|
||||
|
||||
def test_startup_dump_stacks_can_be_disabled(tmp_path):
|
||||
dump = tmp_path / "nodump.txt"
|
||||
sw = StartupWatchdog(on_expire="dump_only", dump_file=str(dump))
|
||||
sw.arm(deadline_s=0.05, dump_stacks=False)
|
||||
thread = sw._thread
|
||||
assert thread is not None
|
||||
thread.join(timeout=5.0)
|
||||
assert sw.expired is True
|
||||
assert dump.exists() is False
|
||||
@@ -0,0 +1,160 @@
|
||||
"""Tests for the pure notification policy in praisonaiagents.push.policy."""
|
||||
|
||||
from datetime import datetime, time, timezone, timedelta
|
||||
|
||||
import pytest
|
||||
|
||||
from praisonaiagents.push import (
|
||||
DEFAULT_PREFERENCE,
|
||||
DetailLevel,
|
||||
NotificationCategory,
|
||||
NotificationDecision,
|
||||
NotificationPreference,
|
||||
evaluate,
|
||||
)
|
||||
|
||||
|
||||
class TestExports:
|
||||
def test_symbols_importable_from_push(self):
|
||||
assert NotificationCategory.APPROVAL_REQUESTED.value == "approval_requested"
|
||||
assert DetailLevel.PRIVATE.value == "private"
|
||||
assert isinstance(DEFAULT_PREFERENCE, NotificationPreference)
|
||||
|
||||
|
||||
class TestCategoryFilter:
|
||||
def test_unsubscribed_category_suppressed(self):
|
||||
prefs = NotificationPreference(categories={NotificationCategory.AGENT_FINISHED})
|
||||
decision = evaluate(NotificationCategory.AGENT_QUESTION, prefs)
|
||||
assert decision.send is False
|
||||
assert decision.reason == "category_not_subscribed"
|
||||
assert decision.redacted_content is None
|
||||
|
||||
def test_subscribed_category_sent(self):
|
||||
prefs = NotificationPreference(categories={NotificationCategory.AGENT_FINISHED})
|
||||
decision = evaluate(
|
||||
NotificationCategory.AGENT_FINISHED, prefs, content="transcript"
|
||||
)
|
||||
assert decision.send is True
|
||||
assert decision.reason == "ok"
|
||||
|
||||
def test_default_preference_used_when_none(self):
|
||||
# DEFAULT_PREFERENCE opts into approval + question, not agent_finished.
|
||||
assert evaluate(NotificationCategory.APPROVAL_REQUESTED).send is True
|
||||
assert evaluate(NotificationCategory.AGENT_FINISHED).send is False
|
||||
|
||||
|
||||
class TestDetailLevelRedaction:
|
||||
def _prefs(self, detail):
|
||||
return NotificationPreference(
|
||||
categories={NotificationCategory.AGENT_FINISHED}, detail=detail
|
||||
)
|
||||
|
||||
def test_private_hides_everything(self):
|
||||
d = evaluate(
|
||||
NotificationCategory.AGENT_FINISHED,
|
||||
self._prefs(DetailLevel.PRIVATE),
|
||||
content="secret transcript",
|
||||
identity="session-42",
|
||||
)
|
||||
assert d.redacted_content == "You have new activity"
|
||||
|
||||
def test_identified_shows_identity_not_content(self):
|
||||
d = evaluate(
|
||||
NotificationCategory.AGENT_FINISHED,
|
||||
self._prefs(DetailLevel.IDENTIFIED),
|
||||
content="secret transcript",
|
||||
identity="session-42",
|
||||
)
|
||||
assert d.redacted_content == "Activity in session-42"
|
||||
assert "secret" not in d.redacted_content
|
||||
|
||||
def test_detailed_includes_content(self):
|
||||
d = evaluate(
|
||||
NotificationCategory.AGENT_FINISHED,
|
||||
self._prefs(DetailLevel.DETAILED),
|
||||
content="full transcript tail",
|
||||
identity="session-42",
|
||||
)
|
||||
assert d.redacted_content == "full transcript tail"
|
||||
|
||||
def test_identified_without_identity_falls_back(self):
|
||||
d = evaluate(
|
||||
NotificationCategory.AGENT_FINISHED,
|
||||
self._prefs(DetailLevel.IDENTIFIED),
|
||||
content="x",
|
||||
)
|
||||
assert d.redacted_content == "You have new activity"
|
||||
|
||||
|
||||
class TestQuietHours:
|
||||
def _prefs(self, **kw):
|
||||
base = dict(
|
||||
categories={NotificationCategory.AGENT_FINISHED,
|
||||
NotificationCategory.APPROVAL_REQUESTED},
|
||||
quiet_hours=(time(22, 0), time(7, 30)),
|
||||
timezone="UTC",
|
||||
)
|
||||
base.update(kw)
|
||||
return NotificationPreference(**base)
|
||||
|
||||
def test_suppressed_inside_wrapping_window(self):
|
||||
now = datetime(2026, 1, 1, 23, 0, tzinfo=timezone.utc)
|
||||
d = evaluate(NotificationCategory.AGENT_FINISHED, self._prefs(), now=now)
|
||||
assert d.send is False
|
||||
assert d.reason == "quiet_hours"
|
||||
|
||||
def test_suppressed_after_midnight_before_end(self):
|
||||
now = datetime(2026, 1, 1, 6, 0, tzinfo=timezone.utc)
|
||||
d = evaluate(NotificationCategory.AGENT_FINISHED, self._prefs(), now=now)
|
||||
assert d.send is False
|
||||
|
||||
def test_sent_outside_window(self):
|
||||
now = datetime(2026, 1, 1, 12, 0, tzinfo=timezone.utc)
|
||||
d = evaluate(NotificationCategory.AGENT_FINISHED, self._prefs(), now=now)
|
||||
assert d.send is True
|
||||
|
||||
def test_urgent_overrides_quiet_hours(self):
|
||||
now = datetime(2026, 1, 1, 23, 0, tzinfo=timezone.utc)
|
||||
d = evaluate(
|
||||
NotificationCategory.APPROVAL_REQUESTED,
|
||||
self._prefs(),
|
||||
now=now,
|
||||
is_urgent=True,
|
||||
)
|
||||
assert d.send is True
|
||||
assert d.reason == "urgent_override"
|
||||
|
||||
def test_urgent_does_not_override_when_disabled(self):
|
||||
now = datetime(2026, 1, 1, 23, 0, tzinfo=timezone.utc)
|
||||
d = evaluate(
|
||||
NotificationCategory.APPROVAL_REQUESTED,
|
||||
self._prefs(urgent_overrides_quiet=False),
|
||||
now=now,
|
||||
is_urgent=True,
|
||||
)
|
||||
assert d.send is False
|
||||
assert d.reason == "quiet_hours"
|
||||
|
||||
def test_timezone_applied(self):
|
||||
# 23:00 UTC is 08:00 in Asia/Tokyo (+09:00) — outside the quiet window.
|
||||
now = datetime(2026, 1, 1, 23, 0, tzinfo=timezone.utc)
|
||||
d = evaluate(
|
||||
NotificationCategory.AGENT_FINISHED,
|
||||
self._prefs(timezone="Asia/Tokyo"),
|
||||
now=now,
|
||||
)
|
||||
assert d.send is True
|
||||
|
||||
def test_no_quiet_hours_always_considered(self):
|
||||
prefs = NotificationPreference(
|
||||
categories={NotificationCategory.AGENT_FINISHED}
|
||||
)
|
||||
now = datetime(2026, 1, 1, 3, 0, tzinfo=timezone.utc)
|
||||
assert evaluate(NotificationCategory.AGENT_FINISHED, prefs, now=now).send
|
||||
|
||||
|
||||
class TestPreferenceHelpers:
|
||||
def test_allows(self):
|
||||
prefs = NotificationPreference(categories={NotificationCategory.AGENT_QUESTION})
|
||||
assert prefs.allows(NotificationCategory.AGENT_QUESTION)
|
||||
assert not prefs.allows(NotificationCategory.AGENT_FINISHED)
|
||||
@@ -308,6 +308,7 @@ def test_a_hosted_backend_reclaims_its_instance_when_collected():
|
||||
instance kept billing until the provider's own idle timer noticed, which
|
||||
docker and flyio do not have."""
|
||||
import gc
|
||||
import time
|
||||
|
||||
from praisonai.integrations.compute_managed_agent import ComputeManagedAgent
|
||||
|
||||
@@ -338,6 +339,12 @@ def test_a_hosted_backend_reclaims_its_instance_when_collected():
|
||||
|
||||
del backend
|
||||
gc.collect()
|
||||
# During live GC the finalizer reclaims fire-and-forget on the shared
|
||||
# AsyncBridge's background loop, so the shutdown lands a beat after
|
||||
# gc.collect() returns. Wait (bounded) for it rather than racing it.
|
||||
deadline = time.monotonic() + 5.0
|
||||
while not released and time.monotonic() < deadline:
|
||||
time.sleep(0.01)
|
||||
assert released == ["inst-1"], "the instance outlived the backend that owned it"
|
||||
|
||||
|
||||
@@ -361,6 +368,7 @@ def test_release_reclaims_even_on_an_event_loop_thread():
|
||||
ever runs, silently leaking the instance -- so _release must bridge the loop
|
||||
the way SharedCompute does, not call asyncio.run() directly."""
|
||||
import asyncio
|
||||
import time
|
||||
|
||||
from praisonai.integrations.compute_managed_agent import _release
|
||||
|
||||
@@ -374,4 +382,11 @@ def test_release_reclaims_even_on_an_event_loop_thread():
|
||||
_release(Fake(), "inst-loop", "docker")
|
||||
|
||||
asyncio.run(collect_on_the_loop())
|
||||
# _release routes through the shared AsyncBridge's background loop
|
||||
# fire-and-forget, so the shutdown lands a beat after asyncio.run()
|
||||
# returns. Wait (bounded) for it rather than racing it — the assertion
|
||||
# still fails if _release ever leaks (never schedules the shutdown).
|
||||
deadline = time.monotonic() + 5.0
|
||||
while not released and time.monotonic() < deadline:
|
||||
time.sleep(0.01)
|
||||
assert released == ["inst-loop"], "an instance leaked when GC ran on a live loop"
|
||||
@@ -0,0 +1,250 @@
|
||||
"""
|
||||
Unit tests for the scheduler misfire / missed-run policy.
|
||||
|
||||
Tests:
|
||||
- scheduled_instant() returns the occurrence a due job is firing for
|
||||
- is_misfire() is False without a grace window (backward-compatible)
|
||||
- A recurring occurrence older than misfire_grace_seconds is recorded as
|
||||
``missed`` and not claimed/fired (interval + cron)
|
||||
- A recovery within the grace window still fires once (coalesce)
|
||||
- A first run (never fired) always fires, never recorded missed
|
||||
- misfire_grace_seconds round-trips through to_dict / from_dict and is omitted
|
||||
when unset
|
||||
- The default store (ConfigYamlScheduleStore) applies the same policy
|
||||
"""
|
||||
|
||||
import os
|
||||
import tempfile
|
||||
import time
|
||||
|
||||
import pytest
|
||||
|
||||
from praisonaiagents.scheduler.models import Schedule, ScheduleJob
|
||||
from praisonaiagents.scheduler.store import FileScheduleStore
|
||||
from praisonaiagents.scheduler.config_store import ConfigYamlScheduleStore
|
||||
from praisonaiagents.scheduler.due import scheduled_instant, is_misfire
|
||||
|
||||
|
||||
def _interval_job(name="job", every=60, grace=None, last_run_at=None):
|
||||
job = ScheduleJob(
|
||||
name=name,
|
||||
schedule=Schedule(kind="every", every_seconds=every),
|
||||
message="hi",
|
||||
misfire_grace_seconds=grace,
|
||||
)
|
||||
if last_run_at is not None:
|
||||
job.last_run_at = last_run_at
|
||||
return job
|
||||
|
||||
|
||||
class TestScheduledInstant:
|
||||
def test_interval_missed_slot(self):
|
||||
# last run at t0; the first overdue slot is t0 + one interval — its age
|
||||
# measures how long the job has been overdue.
|
||||
t0 = 1000.0
|
||||
job = _interval_job(every=60, last_run_at=t0)
|
||||
instant = scheduled_instant(job, t0 + 60 * 5 + 3)
|
||||
assert instant == t0 + 60
|
||||
|
||||
def test_interval_first_run_has_no_instant(self):
|
||||
job = _interval_job(every=60) # never run
|
||||
assert scheduled_instant(job, time.time()) is None
|
||||
|
||||
def test_interval_not_yet_elapsed(self):
|
||||
t0 = 1000.0
|
||||
job = _interval_job(every=60, last_run_at=t0)
|
||||
assert scheduled_instant(job, t0 + 30) is None
|
||||
|
||||
|
||||
class TestIsMisfire:
|
||||
def test_no_grace_is_never_misfire(self):
|
||||
t0 = 1000.0
|
||||
job = _interval_job(every=60, last_run_at=t0, grace=None)
|
||||
# Even a very old slot is not a misfire without a grace window.
|
||||
assert is_misfire(job, t0 + 60 * 100) is False
|
||||
|
||||
def test_stale_slot_is_misfire(self):
|
||||
t0 = 1000.0
|
||||
job = _interval_job(every=60, last_run_at=t0, grace=90)
|
||||
# Slot at t0+60, now is t0+300 → age 240 > 90 grace.
|
||||
assert is_misfire(job, t0 + 300) is True
|
||||
|
||||
def test_fresh_slot_within_grace_is_not_misfire(self):
|
||||
t0 = 1000.0
|
||||
job = _interval_job(every=60, last_run_at=t0, grace=90)
|
||||
# Slot at t0+60, now is t0+70 → age 10 < 90 grace.
|
||||
assert is_misfire(job, t0 + 70) is False
|
||||
|
||||
def test_first_run_never_misfire(self):
|
||||
job = _interval_job(every=60, grace=1) # never run
|
||||
assert is_misfire(job, time.time()) is False
|
||||
|
||||
def test_string_grace_does_not_raise(self):
|
||||
# A hand-edited quoted YAML value (``'90'``) reaching is_misfire as a
|
||||
# string must not raise TypeError and abort the claim pass — it is
|
||||
# coerced, so a genuinely stale slot is still detected.
|
||||
t0 = 1000.0
|
||||
job = _interval_job(every=60, last_run_at=t0, grace=90)
|
||||
job.misfire_grace_seconds = "90" # simulate un-coerced legacy value
|
||||
assert is_misfire(job, t0 + 300) is True
|
||||
|
||||
def test_non_numeric_grace_is_not_misfire(self):
|
||||
# A non-numeric junk value must fail safe to "not a misfire" rather
|
||||
# than raising and starving other due jobs in the same pass.
|
||||
t0 = 1000.0
|
||||
job = _interval_job(every=60, last_run_at=t0, grace=90)
|
||||
job.misfire_grace_seconds = "notanumber"
|
||||
assert is_misfire(job, t0 + 300) is False
|
||||
|
||||
|
||||
class TestGraceCoercion:
|
||||
def test_quoted_grace_coerced_to_float(self):
|
||||
# from_dict must coerce a quoted YAML numeric to float so the later
|
||||
# age comparison never raises.
|
||||
d = _interval_job(every=60, grace=None).to_dict()
|
||||
d["misfire_grace_seconds"] = "3600"
|
||||
restored = ScheduleJob.from_dict(d)
|
||||
assert restored.misfire_grace_seconds == 3600.0
|
||||
assert isinstance(restored.misfire_grace_seconds, float)
|
||||
|
||||
def test_junk_grace_coerced_to_none(self):
|
||||
d = _interval_job(every=60, grace=None).to_dict()
|
||||
d["misfire_grace_seconds"] = "not-a-number"
|
||||
restored = ScheduleJob.from_dict(d)
|
||||
assert restored.misfire_grace_seconds is None
|
||||
|
||||
|
||||
class TestClaimMisfire:
|
||||
def test_stale_occurrence_recorded_missed_not_claimed(self):
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
store = FileScheduleStore(store_dir=d)
|
||||
t0 = time.time() - 10000
|
||||
job = _interval_job(every=60, grace=90)
|
||||
job.last_run_at = t0
|
||||
store.add(job)
|
||||
claimed = store.claim_due(time.time(), owner_id="A")
|
||||
# Not fired.
|
||||
assert claimed == []
|
||||
# Recorded as observable missed, not silently dropped.
|
||||
history = store.get_history(job_id=job.id)
|
||||
assert len(history) == 1
|
||||
assert history[0].status == "missed"
|
||||
# Schedule advanced so it is not re-seen as due immediately.
|
||||
reloaded = store.get(job.id)
|
||||
assert reloaded.last_run_at is not None
|
||||
assert reloaded.last_run_at > t0
|
||||
|
||||
def test_within_grace_still_fires_once(self):
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
store = FileScheduleStore(store_dir=d)
|
||||
now = time.time()
|
||||
job = _interval_job(every=60, grace=300)
|
||||
job.last_run_at = now - 65 # one slot ago, age ~5s < grace
|
||||
store.add(job)
|
||||
claimed = store.claim_due(now, owner_id="A")
|
||||
assert len(claimed) == 1
|
||||
|
||||
def test_first_run_fires_despite_grace(self):
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
store = FileScheduleStore(store_dir=d)
|
||||
job = _interval_job(every=60, grace=1) # never run
|
||||
store.add(job)
|
||||
claimed = store.claim_due(time.time(), owner_id="A")
|
||||
assert len(claimed) == 1
|
||||
|
||||
def test_no_grace_backward_compatible_fires(self):
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
store = FileScheduleStore(store_dir=d)
|
||||
job = _interval_job(every=60, grace=None)
|
||||
job.last_run_at = time.time() - 100000 # long downtime
|
||||
store.add(job)
|
||||
claimed = store.claim_due(time.time(), owner_id="A")
|
||||
# Unchanged behaviour: still fires once on recovery.
|
||||
assert len(claimed) == 1
|
||||
assert store.get_history(job_id=job.id) == []
|
||||
|
||||
@pytest.mark.skipif(
|
||||
__import__("importlib").util.find_spec("croniter") is None,
|
||||
reason="croniter not installed",
|
||||
)
|
||||
def test_cron_stale_occurrence_recorded_missed(self):
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
store = FileScheduleStore(store_dir=d)
|
||||
# Daily 07:00 UTC; last run long ago, now well past a 07:00 slot.
|
||||
job = ScheduleJob(
|
||||
name="brief",
|
||||
schedule=Schedule(kind="cron", cron_expr="0 7 * * *", tz="UTC"),
|
||||
message="brief",
|
||||
misfire_grace_seconds=600, # 10 min grace
|
||||
)
|
||||
job.last_run_at = time.time() - 3 * 86400
|
||||
store.add(job)
|
||||
claimed = store.claim_due(time.time(), owner_id="A")
|
||||
assert claimed == []
|
||||
history = store.get_history(job_id=job.id)
|
||||
assert len(history) == 1
|
||||
assert history[0].status == "missed"
|
||||
|
||||
@pytest.mark.skipif(
|
||||
__import__("importlib").util.find_spec("croniter") is None,
|
||||
reason="croniter not installed",
|
||||
)
|
||||
def test_cron_first_run_fires_despite_stale_slot(self):
|
||||
# A never-run cron job first polled after its slot has aged past grace
|
||||
# must still fire its initial run — the first fire is not a recovered
|
||||
# recurrence to suppress.
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
store = FileScheduleStore(store_dir=d)
|
||||
job = ScheduleJob(
|
||||
name="brief",
|
||||
schedule=Schedule(kind="cron", cron_expr="0 7 * * *", tz="UTC"),
|
||||
message="brief",
|
||||
misfire_grace_seconds=600,
|
||||
# created far in the past so the first 07:00 slot is already
|
||||
# due; last_run_at left None → never run.
|
||||
created_at=time.time() - 3 * 86400,
|
||||
)
|
||||
store.add(job)
|
||||
claimed = store.claim_due(time.time(), owner_id="A")
|
||||
assert len(claimed) == 1
|
||||
# No missed record for a first run.
|
||||
assert store.get_history(job_id=job.id) == []
|
||||
|
||||
|
||||
class TestSerialisation:
|
||||
def test_grace_round_trips(self):
|
||||
job = _interval_job(grace=120)
|
||||
restored = ScheduleJob.from_dict(job.to_dict())
|
||||
assert restored.misfire_grace_seconds == 120
|
||||
|
||||
def test_grace_omitted_when_unset(self):
|
||||
job = _interval_job(grace=None)
|
||||
assert "misfire_grace_seconds" not in job.to_dict()
|
||||
|
||||
|
||||
class TestConfigYamlMisfire:
|
||||
def test_stale_occurrence_recorded_missed(self):
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
store = ConfigYamlScheduleStore(
|
||||
config_path=os.path.join(d, "config.yaml")
|
||||
)
|
||||
job = _interval_job(every=60, grace=90)
|
||||
job.last_run_at = time.time() - 10000
|
||||
store.add(job)
|
||||
claimed = store.claim_due(time.time(), owner_id="A")
|
||||
assert claimed == []
|
||||
history = store.get_history(job_id=job.id)
|
||||
assert len(history) == 1
|
||||
assert history[0].status == "missed"
|
||||
|
||||
def test_yaml_grace_round_trips_through_store(self):
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
path = os.path.join(d, "config.yaml")
|
||||
store = ConfigYamlScheduleStore(config_path=path)
|
||||
job = _interval_job(every=60, grace=120)
|
||||
store.add(job)
|
||||
# Reopen from disk and confirm the field survives serialisation.
|
||||
reopened = ConfigYamlScheduleStore(config_path=path)
|
||||
reloaded = reopened.get(job.id)
|
||||
assert reloaded is not None
|
||||
assert reloaded.misfire_grace_seconds == 120
|
||||
@@ -0,0 +1,136 @@
|
||||
"""Core tests for the provider-quota-aware schedule hold (Issue #5386).
|
||||
|
||||
A recurring job whose model provider is rate-limited must stop re-firing and
|
||||
re-failing every tick: the first 429 parks ``next`` due instant past the
|
||||
provider's own reset window (``Retry-After``), intervening fires coalesce, and
|
||||
the job becomes due again once the window elapses. Covers the pure decision
|
||||
helper ``quota_hold_from_failure`` and the ``is_due`` hold guard, plus the
|
||||
serialisation round-trip of the new ``hold_until`` field.
|
||||
"""
|
||||
|
||||
from praisonaiagents.scheduler import quota_hold_from_failure
|
||||
from praisonaiagents.scheduler.due import is_due
|
||||
from praisonaiagents.scheduler.models import Schedule, ScheduleJob
|
||||
|
||||
|
||||
class _RateLimited(Exception):
|
||||
"""A 429-shaped provider error carrying a Retry-After header."""
|
||||
|
||||
def __init__(self, retry_after):
|
||||
super().__init__("Rate limit exceeded")
|
||||
self.status_code = 429
|
||||
|
||||
class _Resp:
|
||||
headers = {"retry-after": str(retry_after)}
|
||||
|
||||
self.response = _Resp()
|
||||
|
||||
|
||||
# ── quota_hold_from_failure (pure decision helper) ────────────────────
|
||||
|
||||
|
||||
def test_rate_limit_with_retry_after_parks_past_window():
|
||||
now = 1000.0
|
||||
hold = quota_hold_from_failure(_RateLimited(3600), now, slack_seconds=60.0)
|
||||
assert hold == now + 3600 + 60
|
||||
|
||||
|
||||
def test_retry_after_parsed_from_message_string():
|
||||
now = 500.0
|
||||
hold = quota_hold_from_failure(
|
||||
"429 Too Many Requests, retry after 30 seconds", now, slack_seconds=10.0
|
||||
)
|
||||
assert hold == now + 30 + 10
|
||||
|
||||
|
||||
def test_retry_after_http_date_parks_past_window():
|
||||
# RFC 7231 allows Retry-After as an absolute HTTP-date, not just
|
||||
# delta-seconds. A provider that sends the reset instant as a date must
|
||||
# still park the job (greptile #2) — else a long quota window is ignored.
|
||||
import time
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from email.utils import format_datetime
|
||||
|
||||
now = time.time()
|
||||
reset = format_datetime(
|
||||
datetime.now(timezone.utc) + timedelta(seconds=1800)
|
||||
)
|
||||
hold = quota_hold_from_failure(_RateLimited(reset), now, slack_seconds=60.0)
|
||||
assert hold is not None
|
||||
# ~1800s window + 60s slack, allowing a little scheduling drift.
|
||||
assert now + 1800 + 60 - 5 <= hold <= now + 1800 + 60 + 5
|
||||
|
||||
|
||||
def test_retry_after_past_http_date_does_not_hold():
|
||||
# An HTTP-date already in the past yields no usable window → no park.
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from email.utils import format_datetime
|
||||
|
||||
past = format_datetime(datetime.now(timezone.utc) - timedelta(seconds=120))
|
||||
assert quota_hold_from_failure(_RateLimited(past), 0.0) is None
|
||||
|
||||
|
||||
def test_non_quota_failure_never_holds():
|
||||
assert quota_hold_from_failure(Exception("boom: invalid request"), 0.0) is None
|
||||
assert quota_hold_from_failure(Exception("connection reset"), 0.0) is None
|
||||
|
||||
|
||||
def test_rate_limit_without_reset_hint_does_not_hold():
|
||||
# A 429 with no Retry-After / reset window carries no usable park instant.
|
||||
class _Bare(Exception):
|
||||
status_code = 429
|
||||
|
||||
assert quota_hold_from_failure(_Bare("rate limited"), 0.0) is None
|
||||
|
||||
|
||||
def test_slack_is_clamped_non_negative():
|
||||
now = 0.0
|
||||
assert quota_hold_from_failure(_RateLimited(100), now, slack_seconds=-5.0) == 100.0
|
||||
|
||||
|
||||
# ── is_due hold guard ─────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _every_job(**kw):
|
||||
return ScheduleJob(
|
||||
name="j", schedule=Schedule(kind="every", every_seconds=1), message="hi", **kw
|
||||
)
|
||||
|
||||
|
||||
def test_held_job_not_due_within_window():
|
||||
job = _every_job(hold_until=2000.0)
|
||||
# Would otherwise be due (every-1s, never run) but is parked.
|
||||
assert is_due(job, now=1999.0) is False
|
||||
|
||||
|
||||
def test_held_job_due_again_after_window():
|
||||
job = _every_job(hold_until=2000.0)
|
||||
assert is_due(job, now=2000.0) is True
|
||||
assert is_due(job, now=2500.0) is True
|
||||
|
||||
|
||||
def test_unparked_job_unchanged():
|
||||
# No hold → today's behaviour exactly.
|
||||
assert is_due(_every_job(), now=123.0) is True
|
||||
|
||||
|
||||
def test_intervening_fires_coalesce():
|
||||
# A parked every-1s job stays not-due across every intervening tick and
|
||||
# fires exactly once at the first legal instant after the window.
|
||||
job = _every_job(hold_until=100.0)
|
||||
assert [is_due(job, now=t) for t in (10, 50, 99)] == [False, False, False]
|
||||
assert is_due(job, now=100.0) is True
|
||||
|
||||
|
||||
# ── serialisation round-trip ──────────────────────────────────────────
|
||||
|
||||
|
||||
def test_hold_until_round_trips():
|
||||
job = _every_job(hold_until=4242.0)
|
||||
restored = ScheduleJob.from_dict(job.to_dict())
|
||||
assert restored.hold_until == 4242.0
|
||||
|
||||
|
||||
def test_unparked_job_omits_hold_until_key():
|
||||
# A job that never hit a quota wall stays byte-for-byte unchanged.
|
||||
assert "hold_until" not in _every_job().to_dict()
|
||||
@@ -189,6 +189,12 @@ class StreamingConfigSchema(BaseModel):
|
||||
flood_backoff_factor: float = 2.0 # Multiply interval on each flood/429
|
||||
max_interval: float = 30.0 # Cap for the adaptively-widened interval
|
||||
strip_reasoning_tags: bool = True # Strip <think>/<reasoning> from output
|
||||
# Opt-in privacy-safe activity surface (PROGRESS mode). Retained here so the
|
||||
# documented ``activity_status: {enabled, phrases}`` block (and the inline
|
||||
# ``activity_phrases`` catalogue) reach ``StreamingConfig.from_dict`` instead
|
||||
# of being silently dropped by the schema. Off by default.
|
||||
activity_status: Optional[Dict[str, Any]] = None
|
||||
activity_phrases: Optional[Dict[str, str]] = None
|
||||
|
||||
@field_validator("mode")
|
||||
@classmethod
|
||||
|
||||
@@ -23,11 +23,18 @@ from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import os
|
||||
import secrets
|
||||
import time
|
||||
import uuid
|
||||
from typing import Any, Dict, Optional
|
||||
from typing import Any, Dict, Iterable, Optional
|
||||
|
||||
from ._approval_base import DEFAULT_APPROVAL_TIMEOUT, DurableApprovalMixin
|
||||
from ._approval_base import (
|
||||
DEFAULT_APPROVAL_TIMEOUT,
|
||||
DurableApprovalMixin,
|
||||
is_authorized_actor,
|
||||
normalize_approvers,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -45,6 +52,14 @@ class HTTPApproval(DurableApprovalMixin):
|
||||
host: Bind address (default ``127.0.0.1``).
|
||||
port: Port to listen on (default ``8899``).
|
||||
timeout: Max seconds to wait for a response (default 300).
|
||||
allowed_approvers: Optional allowlist of approver IDs permitted to
|
||||
resolve an approval. When provided, each pending approval mints a
|
||||
single-use token that is carried only in the per-approval URL; the
|
||||
resolver must present that token (and, when the request supplies an
|
||||
``approver``, be in the allowlist) or the decision is rejected. When
|
||||
``None`` (default) any visitor may decide (legacy behaviour,
|
||||
backward compatible). Falls back to a comma-separated
|
||||
``HTTP_APPROVERS`` env var when not passed.
|
||||
|
||||
Example::
|
||||
|
||||
@@ -57,11 +72,17 @@ class HTTPApproval(DurableApprovalMixin):
|
||||
host: str = "127.0.0.1",
|
||||
port: int = 8899,
|
||||
timeout: float = DEFAULT_APPROVAL_TIMEOUT,
|
||||
allowed_approvers: Optional[Iterable[str]] = None,
|
||||
store: Optional[Any] = None,
|
||||
):
|
||||
self._host = host
|
||||
self._port = port
|
||||
self._timeout = timeout
|
||||
if allowed_approvers is None:
|
||||
_env = os.environ.get("HTTP_APPROVERS", "").strip()
|
||||
if _env:
|
||||
allowed_approvers = [a.strip() for a in _env.split(",") if a.strip()]
|
||||
self._allowed_approvers = normalize_approvers(allowed_approvers)
|
||||
self._pending: Dict[str, Optional[Dict[str, Any]]] = {}
|
||||
self._server_started = False
|
||||
self._runner: Optional[Any] = None
|
||||
@@ -91,6 +112,20 @@ class HTTPApproval(DurableApprovalMixin):
|
||||
self._server_started = True
|
||||
logger.info(f"HTTPApproval server started on http://{self._host}:{self._port}")
|
||||
|
||||
def _token_ok(self, pending: Dict[str, Any], request) -> bool:
|
||||
"""Return whether *request* carries the per-approval token.
|
||||
|
||||
The token is minted per pending approval and embedded only in the URL
|
||||
handed to the authorised approver. A constant-time comparison is used so
|
||||
the endpoint does not leak the token via timing. When no allowlist is
|
||||
configured there is no token, so this is a no-op (legacy behaviour).
|
||||
"""
|
||||
expected = pending.get("token")
|
||||
if not expected:
|
||||
return True
|
||||
supplied = request.query.get("token") or request.headers.get("X-Approval-Token", "")
|
||||
return secrets.compare_digest(str(supplied), str(expected))
|
||||
|
||||
async def _handle_page(self, request) -> Any:
|
||||
"""Serve the approval page HTML."""
|
||||
from aiohttp import web
|
||||
@@ -101,11 +136,18 @@ class HTTPApproval(DurableApprovalMixin):
|
||||
if pending is None:
|
||||
return web.Response(text="Approval request not found or already decided.", status=404)
|
||||
|
||||
if not self._token_ok(pending, request):
|
||||
logger.warning(
|
||||
"Unauthorised HTTP approval page access for %s", request_id,
|
||||
)
|
||||
return web.Response(text="Not authorized to view this approval.", status=403)
|
||||
|
||||
if pending.get("decided"):
|
||||
return web.Response(text="This approval has already been decided.", status=200)
|
||||
|
||||
info = pending.get("info", {})
|
||||
html = self._build_html(request_id, info)
|
||||
token = pending.get("token")
|
||||
html = self._build_html(request_id, info, token=token)
|
||||
return web.Response(text=html, content_type="text/html")
|
||||
|
||||
async def _handle_decide(self, request) -> Any:
|
||||
@@ -117,6 +159,13 @@ class HTTPApproval(DurableApprovalMixin):
|
||||
|
||||
if pending is None:
|
||||
return web.Response(text="Not found", status=404)
|
||||
|
||||
if not self._token_ok(pending, request):
|
||||
logger.warning(
|
||||
"Unauthorised HTTP approval decision attempt for %s", request_id,
|
||||
)
|
||||
return web.json_response({"ok": False, "error": "unauthorized"}, status=403)
|
||||
|
||||
if pending.get("decided"):
|
||||
return web.Response(text="Already decided", status=200)
|
||||
|
||||
@@ -125,18 +174,32 @@ class HTTPApproval(DurableApprovalMixin):
|
||||
except Exception:
|
||||
body = {}
|
||||
|
||||
approver = body.get("approver", "http_user")
|
||||
|
||||
# Authorisation boundary: when an allowlist is configured a free-text
|
||||
# ``approver`` must be one of the authorised approvers. The per-approval
|
||||
# token already proves the resolver received the private URL; the
|
||||
# allowlist additionally pins the recorded approver identity.
|
||||
if not is_authorized_actor(approver, self._allowed_approvers):
|
||||
logger.warning(
|
||||
"Unauthorised HTTP approver %r for %s", approver, request_id,
|
||||
)
|
||||
return web.json_response({"ok": False, "error": "unauthorized"}, status=403)
|
||||
|
||||
decision = body.get("decision", "deny")
|
||||
pending["decided"] = True
|
||||
pending["approved"] = decision == "approve"
|
||||
pending["reason"] = body.get("reason", f"{'Approved' if pending['approved'] else 'Denied'} via HTTP dashboard")
|
||||
pending["approver"] = body.get("approver", "http_user")
|
||||
pending["approver"] = approver
|
||||
|
||||
return web.json_response({"ok": True, "decision": decision})
|
||||
|
||||
def _build_html(self, request_id: str, info: Dict[str, Any]) -> str:
|
||||
def _build_html(self, request_id: str, info: Dict[str, Any], token: Optional[str] = None) -> str:
|
||||
"""Build a minimal approval page."""
|
||||
import html as html_module
|
||||
|
||||
token_qs = f"?token={html_module.escape(str(token))}" if token else ""
|
||||
|
||||
tool_name = html_module.escape(str(info.get("tool_name", "unknown")))
|
||||
risk_level = html_module.escape(str(info.get("risk_level", "unknown")))
|
||||
agent_name = html_module.escape(str(info.get("agent_name", "")))
|
||||
@@ -181,7 +244,7 @@ td {{ padding: 8px; border-bottom: 1px solid #333; }} code {{ background: #2a2a4
|
||||
<div class="done" id="result" style="display:none"></div>
|
||||
<script>
|
||||
async function decide(d) {{
|
||||
const res = await fetch('/approve/{request_id}/decide', {{
|
||||
const res = await fetch('/approve/{request_id}/decide{token_qs}', {{
|
||||
method: 'POST', headers: {{'Content-Type': 'application/json'}},
|
||||
body: JSON.stringify({{decision: d}})
|
||||
}});
|
||||
@@ -202,9 +265,13 @@ async function decide(d) {{
|
||||
await self._ensure_server()
|
||||
|
||||
request_id = str(uuid.uuid4())
|
||||
# Mint a single-use token bound to this approval when an allowlist is
|
||||
# configured, so only the holder of the private URL can resolve it.
|
||||
token = secrets.token_urlsafe(32) if self._allowed_approvers is not None else None
|
||||
self._pending[request_id] = {
|
||||
"decided": False,
|
||||
"approved": False,
|
||||
"token": token,
|
||||
"info": {
|
||||
"tool_name": request.tool_name,
|
||||
"arguments": request.arguments,
|
||||
@@ -214,6 +281,8 @@ async function decide(d) {{
|
||||
}
|
||||
|
||||
url = f"http://{self._host}:{self._port}/approve/{request_id}"
|
||||
if token:
|
||||
url = f"{url}?token={token}"
|
||||
logger.info(f"HTTPApproval: Waiting for decision at {url}")
|
||||
print(f"\n🔗 Open this URL to approve/deny:\n {url}\n")
|
||||
|
||||
|
||||
@@ -172,7 +172,16 @@ class OutboundQueue:
|
||||
ordering: Literal["strict", "best_effort"] = "best_effort",
|
||||
dead_letter_min_age: float = _DEFAULT_DEAD_LETTER_MIN_AGE_SECONDS,
|
||||
dead_letter_policy: Optional[Any] = None,
|
||||
read_only: bool = False,
|
||||
) -> None:
|
||||
# Operator inspection (``list``/``stats``) must NEVER run crash recovery
|
||||
# on a live database: flipping in-flight ``sending`` rows to
|
||||
# ``recovered`` while the owning gateway is mid-send lets a later drain
|
||||
# re-claim and duplicate a message. ``read_only=True`` opens the store
|
||||
# for reads without the ``sending``→``recovered`` migration; targeted
|
||||
# ops (``retry``/``purge_entry``) still work but leave in-flight rows
|
||||
# untouched.
|
||||
self.read_only = bool(read_only)
|
||||
self.path = Path(path).expanduser()
|
||||
self.max_size = int(max_size)
|
||||
self.ttl_seconds = int(ttl_seconds)
|
||||
@@ -255,11 +264,17 @@ class OutboundQueue:
|
||||
# reconciled before re-dispatch to avoid duplicate delivery. Drain
|
||||
# treats 'recovered' like a retryable entry but offers it to a
|
||||
# reconciler first.
|
||||
conn.execute("""
|
||||
UPDATE outbound_queue
|
||||
SET status = 'recovered'
|
||||
WHERE status = 'sending'
|
||||
""")
|
||||
#
|
||||
# NEVER run this under read_only inspection: the gateway may be live
|
||||
# and a 'sending' row genuinely in flight. Flipping it to 'recovered'
|
||||
# here would let the running drain re-claim and duplicate it, so
|
||||
# operator ``list``/``stats`` open the store without touching claims.
|
||||
if not self.read_only:
|
||||
conn.execute("""
|
||||
UPDATE outbound_queue
|
||||
SET status = 'recovered'
|
||||
WHERE status = 'sending'
|
||||
""")
|
||||
conn.commit()
|
||||
|
||||
# ── Core API ────────────────────────────────────────────────────
|
||||
@@ -743,6 +758,31 @@ class OutboundQueue:
|
||||
return int(key.split(":")[-1])
|
||||
except (ValueError, IndexError):
|
||||
raise ValueError(f"Invalid entry key format: {key}")
|
||||
|
||||
def _row_status_if_key_matches(
|
||||
self, conn: sqlite3.Connection, key: str
|
||||
) -> Optional[str]:
|
||||
"""Return the row's status only if ``key`` fully matches a stored entry.
|
||||
|
||||
:meth:`enqueue` builds the key as ``f"{target}:{idempotency_key}:{id}"``.
|
||||
Targeted ops (:meth:`retry`, :meth:`purge_entry`) previously acted on the
|
||||
trailing id alone, so a stale key from another outbox — or a mistyped
|
||||
target/idempotency that kept an existing id — would silently mutate the
|
||||
WRONG message. This verifies the whole reconstructed key against the row
|
||||
before any state change. Returns ``None`` when no row matches the full
|
||||
key (unknown id, or id matches but target/idempotency differ).
|
||||
"""
|
||||
entry_id = self._extract_id_from_key(key)
|
||||
row = conn.execute(
|
||||
"SELECT target, idempotency_key, status FROM outbound_queue WHERE id = ?",
|
||||
(entry_id,),
|
||||
).fetchone()
|
||||
if row is None:
|
||||
return None
|
||||
target, idempotency, status = row
|
||||
if f"{target}:{idempotency}:{entry_id}" != key:
|
||||
return None
|
||||
return str(status)
|
||||
|
||||
# ── Maintenance ─────────────────────────────────────────────────
|
||||
def _evict_expired_locked(self, conn: sqlite3.Connection) -> int:
|
||||
@@ -883,6 +923,118 @@ class OutboundQueue:
|
||||
conn.execute("DELETE FROM outbound_queue")
|
||||
conn.commit()
|
||||
return int(n)
|
||||
|
||||
# ── Operator introspection & targeted ops (parity with InboundDLQ) ──
|
||||
def list(
|
||||
self,
|
||||
*,
|
||||
status: Optional[str] = None,
|
||||
target: Optional[str] = None,
|
||||
limit: int = 100,
|
||||
) -> List[OutboundEntry]:
|
||||
"""Enumerate outbound entries for operator inspection (newest first).
|
||||
|
||||
Optionally filter by ``status`` (e.g. ``"failed"``,
|
||||
``"permanent_failure"``, ``"pending"``) and/or ``target`` (exact match,
|
||||
e.g. ``"telegram:12345"``). Purely read-only; it never mutates state.
|
||||
"""
|
||||
clauses: List[str] = []
|
||||
params: List[Any] = []
|
||||
if status is not None:
|
||||
clauses.append("status = ?")
|
||||
params.append(status)
|
||||
if target is not None:
|
||||
clauses.append("target = ?")
|
||||
params.append(target)
|
||||
where = (" WHERE " + " AND ".join(clauses)) if clauses else ""
|
||||
params.append(int(limit))
|
||||
with self._lock, closing(self._connect()) as conn:
|
||||
rows = conn.execute(
|
||||
"""
|
||||
SELECT id, ts, idempotency_key, target, payload, metadata,
|
||||
status, attempts, last_attempt, error, sent_at
|
||||
FROM outbound_queue
|
||||
"""
|
||||
+ where
|
||||
+ " ORDER BY ts DESC LIMIT ?",
|
||||
params,
|
||||
).fetchall()
|
||||
return [OutboundEntry(*row) for row in rows]
|
||||
|
||||
def stats(self) -> Dict[str, int]:
|
||||
"""Return a per-status count map (e.g. ``{"pending": 3, "failed": 1}``).
|
||||
|
||||
Only statuses with at least one entry are present. Gives operators an
|
||||
at-a-glance breakdown of the outbox instead of a single bulk counter.
|
||||
"""
|
||||
with self._lock, closing(self._connect()) as conn:
|
||||
rows = conn.execute(
|
||||
"SELECT status, COUNT(*) FROM outbound_queue GROUP BY status"
|
||||
).fetchall()
|
||||
return {str(status): int(count) for status, count in rows}
|
||||
|
||||
async def retry(self, key: str) -> bool:
|
||||
"""Requeue a single entry for the next drain.
|
||||
|
||||
Resets a ``failed``/``permanent_failure`` entry back to ``pending`` and
|
||||
clears its ``attempts`` so the next :meth:`drain` re-dispatches it —
|
||||
e.g. after a channel outage recovers. Returns True if an entry was
|
||||
requeued, False if the key is unknown, already terminal (``sent``),
|
||||
in-flight (``sending``), or crash-``recovered``.
|
||||
|
||||
A ``recovered`` entry is DELIBERATELY not manually retriable: it was
|
||||
in-flight when the process last crashed, so its delivery outcome is
|
||||
unknown and the next :meth:`drain` already handles it safely (reconcile
|
||||
first, else re-send with the "possible duplicate" annotator). Flipping it
|
||||
to ``pending`` here would strip that safeguard and risk an unlabelled
|
||||
duplicate. ``key`` is the tracking key form ``target:idempotency:id`` and
|
||||
is matched in full against the stored row.
|
||||
"""
|
||||
loop = asyncio.get_event_loop()
|
||||
return await loop.run_in_executor(None, self._sync_retry, key)
|
||||
|
||||
def _sync_retry(self, key: str) -> bool:
|
||||
"""Synchronous version of retry for thread pool execution."""
|
||||
with self._lock, closing(self._connect()) as conn:
|
||||
if self._row_status_if_key_matches(conn, key) is None:
|
||||
return False # unknown id or full key mismatch — never guess
|
||||
entry_id = self._extract_id_from_key(key)
|
||||
cur = conn.execute(
|
||||
"""
|
||||
UPDATE outbound_queue
|
||||
SET status = 'pending', attempts = 0, error = NULL,
|
||||
last_attempt = NULL
|
||||
WHERE id = ? AND status IN ('failed', 'permanent_failure')
|
||||
""",
|
||||
(entry_id,),
|
||||
)
|
||||
conn.commit()
|
||||
self._active_claims.pop(key, None)
|
||||
return cur.rowcount > 0
|
||||
|
||||
def purge_entry(self, key: str) -> bool:
|
||||
"""Delete a single entry by its full tracking key. Returns True if removed.
|
||||
|
||||
Refuses to delete an in-flight ``sending`` entry: the gateway may have
|
||||
already handed it to the channel API and be awaiting the terminal
|
||||
``mark_sent``/``mark_failed`` write. Deleting the row out from under that
|
||||
send makes ``mark_sent`` a no-op and the caller report a spurious
|
||||
failure despite delivery. The key is matched in full against the stored
|
||||
row so a stale/mistyped key cannot delete the wrong message.
|
||||
"""
|
||||
with self._lock, closing(self._connect()) as conn:
|
||||
status = self._row_status_if_key_matches(conn, key)
|
||||
if status is None:
|
||||
return False # unknown id or full key mismatch
|
||||
if status == "sending":
|
||||
return False # in-flight — never delete under an active send
|
||||
entry_id = self._extract_id_from_key(key)
|
||||
cur = conn.execute(
|
||||
"DELETE FROM outbound_queue WHERE id = ?", (entry_id,)
|
||||
)
|
||||
conn.commit()
|
||||
self._active_claims.pop(key, None)
|
||||
return cur.rowcount > 0
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return (
|
||||
|
||||
@@ -210,6 +210,7 @@ class BotSessionManager:
|
||||
admission_gate: Optional[Any] = None,
|
||||
turn_lock_map: Optional["LockMap"] = None,
|
||||
surface_completion_reason: bool = False,
|
||||
defer_journal_completion: bool = False,
|
||||
) -> None:
|
||||
self._histories: Dict[str, List[Dict[str, Any]]] = {}
|
||||
# Issue #3232: the per-turn lock map is keyed on the *resolved* storage
|
||||
@@ -270,6 +271,21 @@ class BotSessionManager:
|
||||
# behaviour (the tool returns its "no gateway available" message).
|
||||
self._delivery_router = delivery_router
|
||||
self._last_journal_key = None # Store key for delayed completion
|
||||
# Issue #5152: set of inbound journal keys deferred for delayed
|
||||
# completion. Unlike the single ``_last_journal_key`` slot, this holds
|
||||
# *every* in-flight deferred key so two concurrent turns (different
|
||||
# users run under different per-storage-key locks, so ``chat()`` is not
|
||||
# globally serialised) can never clobber one another — settling one
|
||||
# turn must never settle another turn's still-undelivered inbound.
|
||||
self._deferred_journal_keys: set[str] = set()
|
||||
# Issue #5152: when True, chat() does NOT mark the inbound journal
|
||||
# complete on clean exit; instead it stashes the journal key so the
|
||||
# adapter can call complete_last_journal_entry() *after* the reply is
|
||||
# durably delivered. This closes the window where a crash between
|
||||
# "agent finished" and "reply sent" would settle the inbound (no
|
||||
# replay) yet lose the user's reply. Default False keeps the legacy
|
||||
# settle-on-agent-completion behaviour byte-for-byte unchanged.
|
||||
self._defer_journal_completion = defer_journal_completion
|
||||
# Run control for in-flight message handling
|
||||
self._run_control = run_control
|
||||
# Issue #3296: opt-in surfacing of *why* a turn ended. When True, a turn
|
||||
@@ -1641,13 +1657,26 @@ class BotSessionManager:
|
||||
await claim_ctx.__aexit__(type(e), e, e.__traceback__)
|
||||
raise
|
||||
else:
|
||||
# Clean exit - mark journal complete before releasing claim
|
||||
# Clean exit - settle the inbound journal.
|
||||
if journal_key is not None and self._ingress_journal is not None:
|
||||
try:
|
||||
self._ingress_journal.complete(journal_key)
|
||||
self._last_journal_key = None
|
||||
except Exception as e:
|
||||
logger.warning("Failed to complete journal entry: %s", e)
|
||||
if self._defer_journal_completion:
|
||||
# Issue #5152: defer completion until the adapter has
|
||||
# durably delivered the reply. Record the key so the
|
||||
# adapter calls complete_last_journal_entry(journal_key)
|
||||
# *after* send; on a crash before delivery the entry
|
||||
# stays pending and is redeliverable rather than lost.
|
||||
# The key is added to the per-turn set (race-safe under
|
||||
# concurrent turns) and also mirrored into the single
|
||||
# slot for the convenience no-arg completion path.
|
||||
self._deferred_journal_keys.add(journal_key)
|
||||
self._last_journal_key = journal_key
|
||||
else:
|
||||
# Legacy behaviour: mark complete on agent completion.
|
||||
try:
|
||||
self._ingress_journal.complete(journal_key)
|
||||
self._last_journal_key = None
|
||||
except Exception as e:
|
||||
logger.warning("Failed to complete journal entry: %s", e)
|
||||
if claim_ctx is not None:
|
||||
await claim_ctx.__aexit__(None, None, None)
|
||||
return result or ""
|
||||
@@ -2006,22 +2035,44 @@ class BotSessionManager:
|
||||
)
|
||||
return evicted
|
||||
|
||||
def complete_last_journal_entry(self) -> bool:
|
||||
"""Complete the last journal entry if one exists.
|
||||
|
||||
Call this after successfully delivering a message to the platform
|
||||
to ensure the journal entry is marked as completed.
|
||||
|
||||
Returns True if an entry was completed, False if no entry was pending.
|
||||
def complete_last_journal_entry(
|
||||
self, journal_key: Optional[str] = None
|
||||
) -> bool:
|
||||
"""Complete a deferred inbound journal entry after delivery.
|
||||
|
||||
Call this *after* the reply has been durably delivered to the platform
|
||||
so the inbound entry is settled only once the user has actually been
|
||||
answered (Issue #5152).
|
||||
|
||||
Args:
|
||||
journal_key: The exact deferred key to settle. **Prefer passing
|
||||
this** — under concurrent turns (different users run under
|
||||
different per-storage-key locks) the manager-wide "last" slot
|
||||
is ambiguous, so settling a specific key guarantees turn A's
|
||||
acknowledgement never settles turn B's still-undelivered
|
||||
inbound. When ``None`` (single-turn convenience), the most
|
||||
recently deferred key is used.
|
||||
|
||||
Returns True if an entry was completed, False if no matching entry was
|
||||
pending.
|
||||
"""
|
||||
if self._last_journal_key is not None and self._ingress_journal is not None:
|
||||
try:
|
||||
self._ingress_journal.complete(self._last_journal_key)
|
||||
self._last_journal_key = None
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.warning("Failed to complete journal entry: %s", e)
|
||||
return False
|
||||
if self._ingress_journal is None:
|
||||
return False
|
||||
key = journal_key if journal_key is not None else self._last_journal_key
|
||||
if key is None:
|
||||
return False
|
||||
try:
|
||||
self._ingress_journal.complete(key)
|
||||
except Exception as e:
|
||||
logger.warning("Failed to complete journal entry: %s", e)
|
||||
return False
|
||||
# Settle bookkeeping: drop the key from the deferred set and clear the
|
||||
# convenience slot only when it referred to this key (so a concurrent
|
||||
# turn's slot is never silently cleared out from under it).
|
||||
self._deferred_journal_keys.discard(key)
|
||||
if self._last_journal_key == key:
|
||||
self._last_journal_key = None
|
||||
return True
|
||||
|
||||
def _should_reset_session(self, storage_key: str) -> bool:
|
||||
"""Check if a session should be reset based on the policy.
|
||||
|
||||
@@ -68,6 +68,43 @@ def redact_outbound_text(text: str, adapter: Any = None) -> str:
|
||||
return text
|
||||
|
||||
|
||||
# Privacy-safe activity surface: stable mapping from tool-name substrings to a
|
||||
# coarse activity *category*. Categories key the operator-configured phrase
|
||||
# catalogue so a curated line ("Searching the web…") can be shown during an
|
||||
# output-silent tool run WITHOUT ever interpolating tool args, command strings,
|
||||
# URLs, file paths or model reasoning. Matching is substring-based on the
|
||||
# lowercased tool name only; unknown tools fall back to the ``default`` phrase.
|
||||
_ACTIVITY_CATEGORY_HINTS: tuple = (
|
||||
("web", ("web", "browser", "fetch", "url", "http", "crawl", "scrape")),
|
||||
("search", ("search", "google", "duckduckgo", "tavily", "serper", "wiki")),
|
||||
("shell", ("shell", "bash", "command", "terminal", "exec", "subprocess")),
|
||||
("code", ("code", "python", "interpreter", "repl", "compile", "script")),
|
||||
("file", ("file", "read", "write", "open", "save", "load", "csv", "pdf", "doc")),
|
||||
("mcp", ("mcp",)),
|
||||
)
|
||||
|
||||
# Neutral line used when a curated catalogue is enabled but has no phrase for the
|
||||
# current category and no ``default``. Guarantees opting in can never fall back to
|
||||
# surfacing the raw tool name.
|
||||
_SAFE_ACTIVITY_FALLBACK = "Working on it…"
|
||||
|
||||
|
||||
def resolve_activity_category(tool_name: Optional[str]) -> str:
|
||||
"""Map a tool name to a stable activity *category* for the phrase catalogue.
|
||||
|
||||
Uses substring hints on the lowercased tool name only — never the tool's
|
||||
arguments — so the resulting category (and the phrase it selects) can never
|
||||
leak sensitive input. Returns ``"default"`` when nothing matches.
|
||||
"""
|
||||
if not tool_name:
|
||||
return "default"
|
||||
lowered = str(tool_name).lower()
|
||||
for category, hints in _ACTIVITY_CATEGORY_HINTS:
|
||||
if any(hint in lowered for hint in hints):
|
||||
return category
|
||||
return "default"
|
||||
|
||||
|
||||
def strip_reasoning_tags(text: str) -> str:
|
||||
"""Remove ``<think>``/``<reasoning>`` spans from streamed content.
|
||||
|
||||
@@ -116,6 +153,12 @@ class StreamingConfig:
|
||||
flood_backoff_factor: float = 2.0 # Multiply interval on each flood/429
|
||||
max_interval: float = 30.0 # Cap for the adaptively-widened interval
|
||||
strip_reasoning_tags: bool = True # Strip <think>/<reasoning> from output
|
||||
# Opt-in privacy-safe activity surface (PROGRESS mode). When set, a curated
|
||||
# phrase keyed by tool *category* (see ``resolve_activity_category``) is
|
||||
# shown during an output-silent tool run instead of the raw tool name.
|
||||
# Empty/None => today's behaviour (renders "Running <tool>..."). Phrases are
|
||||
# the ONLY text surfaced; tool args/commands/URLs are never interpolated.
|
||||
activity_phrases: Optional[Dict[str, str]] = None
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: Dict[str, Any]) -> "StreamingConfig":
|
||||
@@ -127,6 +170,20 @@ class StreamingConfig:
|
||||
logger.warning("Invalid streaming mode '%s', using 'off'", mode_str)
|
||||
mode = StreamingMode.OFF
|
||||
|
||||
# Accept the curated phrase catalogue either inline as ``activity_phrases``
|
||||
# or under an ``activity_status`` block ({enabled, phrases}) matching the
|
||||
# documented bot-config shape. ``enabled: false`` disables the surface.
|
||||
activity_phrases: Optional[Dict[str, str]] = None
|
||||
raw_phrases = data.get("activity_phrases")
|
||||
activity_block = data.get("activity_status")
|
||||
if isinstance(activity_block, dict):
|
||||
if activity_block.get("enabled", True):
|
||||
raw_phrases = activity_block.get("phrases", raw_phrases)
|
||||
else:
|
||||
raw_phrases = None
|
||||
if isinstance(raw_phrases, dict) and raw_phrases:
|
||||
activity_phrases = {str(k): str(v) for k, v in raw_phrases.items()}
|
||||
|
||||
return cls(
|
||||
mode=mode,
|
||||
min_interval=data.get("min_interval", 1.5),
|
||||
@@ -140,6 +197,7 @@ class StreamingConfig:
|
||||
flood_backoff_factor=data.get("flood_backoff_factor", 2.0),
|
||||
max_interval=data.get("max_interval", 30.0),
|
||||
strip_reasoning_tags=data.get("strip_reasoning_tags", True),
|
||||
activity_phrases=activity_phrases,
|
||||
)
|
||||
|
||||
|
||||
@@ -412,6 +470,26 @@ class DraftStreamer:
|
||||
# Schedule new update task
|
||||
self._update_task = asyncio.create_task(_delayed_update())
|
||||
|
||||
def _activity_line(self, tool_name: str) -> str:
|
||||
"""Render the progress line for a running tool.
|
||||
|
||||
When a curated ``activity_phrases`` catalogue is configured, the line is
|
||||
drawn EXCLUSIVELY from that catalogue keyed by the tool's activity
|
||||
*category* (falling back to the ``default`` phrase). The raw tool name is
|
||||
never surfaced in that mode, closing the privacy footgun where tool
|
||||
identity/args could leak to chat. With no catalogue configured, behaviour
|
||||
is unchanged ("Running <tool>...").
|
||||
"""
|
||||
phrases = self._config.activity_phrases
|
||||
if phrases:
|
||||
category = resolve_activity_category(tool_name)
|
||||
phrase = phrases.get(category) or phrases.get("default")
|
||||
# Opting in must NEVER silently restore the raw tool name: when a
|
||||
# partial catalogue lacks both the matched category and a ``default``
|
||||
# phrase, fall back to a safe generic line rather than the tool name.
|
||||
return f"{self._config.progress_prefix}{phrase or _SAFE_ACTIVITY_FALLBACK}"
|
||||
return f"{self._config.progress_prefix}Running {tool_name}..."
|
||||
|
||||
def _render_content(self) -> Optional[str]:
|
||||
"""Render the buffer for the current mode, applying reasoning stripping."""
|
||||
buffer = self._content_buffer
|
||||
@@ -422,7 +500,11 @@ class DraftStreamer:
|
||||
if self._config.mode == StreamingMode.DRAFT:
|
||||
content = buffer or self._config.placeholder_text
|
||||
elif self._config.mode == StreamingMode.PROGRESS:
|
||||
if self._feed_style and self._progress_lines:
|
||||
# When a curated catalogue is opted in, the raw feed (which folds tool
|
||||
# names/summaries/output into its lines) must not be rendered — that
|
||||
# would bypass the privacy-safe surface. Fall through to the curated
|
||||
# single-line ``_activity_line`` instead so only phrases are shown.
|
||||
if self._feed_style and self._progress_lines and not self._config.activity_phrases:
|
||||
from praisonaiagents.streaming.progress import render_progress
|
||||
|
||||
content = render_progress(
|
||||
@@ -433,7 +515,7 @@ class DraftStreamer:
|
||||
if not content:
|
||||
content = self._config.placeholder_text
|
||||
elif self._current_tool:
|
||||
content = f"{self._config.progress_prefix}Running {self._current_tool}..."
|
||||
content = self._activity_line(self._current_tool)
|
||||
elif buffer:
|
||||
content = buffer
|
||||
else:
|
||||
|
||||
@@ -27,9 +27,14 @@ import logging
|
||||
import os
|
||||
import time
|
||||
import uuid
|
||||
from typing import Any, Dict, Optional
|
||||
from typing import Any, Dict, Iterable, Optional
|
||||
|
||||
from ._approval_base import DEFAULT_APPROVAL_TIMEOUT, DurableApprovalMixin
|
||||
from ._approval_base import (
|
||||
DEFAULT_APPROVAL_TIMEOUT,
|
||||
DurableApprovalMixin,
|
||||
is_authorized_actor,
|
||||
normalize_approvers,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -51,6 +56,13 @@ class WebhookApproval(DurableApprovalMixin):
|
||||
headers: Extra HTTP headers (e.g. auth tokens).
|
||||
timeout: Max seconds to wait for a response (default 300).
|
||||
poll_interval: Seconds between status polls (default 5.0).
|
||||
allowed_approvers: Optional allowlist of approver IDs permitted to
|
||||
resolve an approval. When provided, a decision whose ``approver``
|
||||
is not in the allowlist is dropped (polling continues) so an
|
||||
unauthorised responder to the status endpoint cannot resolve a
|
||||
gated tool. When ``None`` (default) any responder may decide
|
||||
(legacy behaviour, backward compatible). Falls back to a
|
||||
comma-separated ``WEBHOOK_APPROVERS`` env var when not passed.
|
||||
|
||||
Example::
|
||||
|
||||
@@ -71,6 +83,7 @@ class WebhookApproval(DurableApprovalMixin):
|
||||
headers: Optional[Dict[str, str]] = None,
|
||||
timeout: float = DEFAULT_APPROVAL_TIMEOUT,
|
||||
poll_interval: float = 5.0,
|
||||
allowed_approvers: Optional[Iterable[str]] = None,
|
||||
store: Optional[Any] = None,
|
||||
):
|
||||
self._webhook_url = webhook_url or os.environ.get("APPROVAL_WEBHOOK_URL", "")
|
||||
@@ -82,11 +95,29 @@ class WebhookApproval(DurableApprovalMixin):
|
||||
self._headers = headers or {}
|
||||
self._timeout = timeout
|
||||
self._poll_interval = poll_interval
|
||||
if allowed_approvers is None:
|
||||
_env = os.environ.get("WEBHOOK_APPROVERS", "").strip()
|
||||
if _env:
|
||||
allowed_approvers = [a.strip() for a in _env.split(",") if a.strip()]
|
||||
self._allowed_approvers = normalize_approvers(allowed_approvers)
|
||||
self._init_store(store)
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return f"WebhookApproval(webhook_url={self._webhook_url!r})"
|
||||
|
||||
def _approver_authorized(self, approver: Any) -> bool:
|
||||
"""Return whether *approver* may resolve an approval.
|
||||
|
||||
No-op (always ``True``) when no allowlist is configured (legacy
|
||||
behaviour). Otherwise the responder's ``approver`` must be in the
|
||||
allowlist, so an unauthorised party that can reach the status endpoint
|
||||
cannot resolve a gated tool.
|
||||
"""
|
||||
return is_authorized_actor(
|
||||
None if approver is None else str(approver),
|
||||
self._allowed_approvers,
|
||||
)
|
||||
|
||||
# ── Internal HTTP helper ───────────────────────────────────────────
|
||||
|
||||
async def _http_request(
|
||||
@@ -158,7 +189,9 @@ class WebhookApproval(DurableApprovalMixin):
|
||||
|
||||
# Check for immediate decision
|
||||
if isinstance(post_data, dict):
|
||||
if "approved" in post_data:
|
||||
if "approved" in post_data and self._approver_authorized(
|
||||
post_data.get("approver")
|
||||
):
|
||||
decision = ApprovalDecision(
|
||||
approved=bool(post_data["approved"]),
|
||||
reason=post_data.get("reason", "Webhook immediate response"),
|
||||
@@ -167,6 +200,11 @@ class WebhookApproval(DurableApprovalMixin):
|
||||
)
|
||||
await self._resolve_pending(request, decision)
|
||||
return decision
|
||||
if "approved" in post_data:
|
||||
logger.warning(
|
||||
"Dropping webhook immediate decision from unauthorised "
|
||||
"approver %r", post_data.get("approver"),
|
||||
)
|
||||
|
||||
# 2. Poll for decision
|
||||
status_url = self._status_url or f"{self._webhook_url}/{request_id}"
|
||||
@@ -227,6 +265,17 @@ class WebhookApproval(DurableApprovalMixin):
|
||||
if status == "pending":
|
||||
continue
|
||||
|
||||
has_decision = (
|
||||
"approved" in data
|
||||
or status in ("approved", "approve", "yes", "denied", "deny", "rejected", "no")
|
||||
)
|
||||
if has_decision and not self._approver_authorized(data.get("approver")):
|
||||
logger.warning(
|
||||
"Dropping webhook decision from unauthorised approver %r "
|
||||
"for %s", data.get("approver"), request_id,
|
||||
)
|
||||
continue
|
||||
|
||||
if "approved" in data:
|
||||
return ApprovalDecision(
|
||||
approved=bool(data["approved"]),
|
||||
|
||||
@@ -706,12 +706,14 @@ class BotOS:
|
||||
_status = "succeeded"
|
||||
_result = None
|
||||
_error = None
|
||||
_exc: Optional[Exception] = None
|
||||
_delivered = False
|
||||
try:
|
||||
_result, _delivered = await self._execute_schedule_job(job)
|
||||
except Exception as e:
|
||||
_status = "failed"
|
||||
_error = str(e)
|
||||
_exc = e
|
||||
logger.warning(f"BotOS: schedule job {job.name} failed: {e}")
|
||||
finally:
|
||||
# Release the lease so it does not linger until expiry.
|
||||
@@ -731,6 +733,15 @@ class BotOS:
|
||||
f"{job.name}: {e}"
|
||||
)
|
||||
_duration = _time.time() - _start
|
||||
# Provider-quota hold: park a job past a 429/quota reset
|
||||
# window so this loop stops re-firing and re-failing every
|
||||
# tick against a benched provider — the same core decision
|
||||
# the gateway executor applies (issue #5386). A success
|
||||
# clears any prior park (the window reopened); the hold is
|
||||
# written by the mark_run below.
|
||||
self._apply_schedule_quota_hold(
|
||||
job, succeeded=_status == "succeeded", error=_exc,
|
||||
)
|
||||
runner.mark_run(
|
||||
job,
|
||||
status=_status,
|
||||
@@ -826,6 +837,52 @@ class BotOS:
|
||||
logger.info(f"BotOS: executed schedule job '{job.name}'")
|
||||
return (result_str, delivered)
|
||||
|
||||
@staticmethod
|
||||
def _apply_schedule_quota_hold(
|
||||
job: Any, *, succeeded: bool, error: Optional[Exception],
|
||||
) -> None:
|
||||
"""Park a due-loop job past a provider rate-limit window (issue #5386).
|
||||
|
||||
The BotOS schedule loop runs ``agent.chat`` directly rather than through
|
||||
:class:`ScheduledAgentExecutor`, so it needs the same provider-quota
|
||||
hold or a benched provider is hammered every tick. Uses the *core* pure
|
||||
decision ``quota_hold_from_failure`` (no new surface): a 429/quota
|
||||
failure carrying a reset hint sets ``job.hold_until`` (never shortening
|
||||
an existing later park), which the subsequent ``mark_run`` persists so
|
||||
``is_due`` coalesces intervening fires. A success clears any prior park
|
||||
(proof the window reopened). A no-op when core is unavailable or the
|
||||
failure is not a quota signal, so today's behaviour is unchanged.
|
||||
"""
|
||||
if succeeded:
|
||||
if getattr(job, "hold_until", None) is not None:
|
||||
try:
|
||||
job.hold_until = None
|
||||
except Exception: # pragma: no cover - defensive
|
||||
pass
|
||||
return
|
||||
if error is None:
|
||||
return
|
||||
try:
|
||||
from praisonaiagents.scheduler import quota_hold_from_failure
|
||||
except Exception: # pragma: no cover - core primitive always present
|
||||
return
|
||||
import time as _time
|
||||
hold_until = quota_hold_from_failure(error, _time.time())
|
||||
if hold_until is None:
|
||||
return
|
||||
prior = getattr(job, "hold_until", None)
|
||||
if isinstance(prior, (int, float)) and prior > hold_until:
|
||||
hold_until = prior
|
||||
try:
|
||||
job.hold_until = hold_until
|
||||
except Exception as e: # pragma: no cover - defensive
|
||||
logger.warning(f"BotOS: could not set quota hold for {job.name}: {e}")
|
||||
return
|
||||
logger.info(
|
||||
"BotOS: job '%s' held until %.0f after provider rate-limit",
|
||||
job.name, hold_until,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _enforce_pinned_model(job: Any, agent: Any) -> Optional[Any]:
|
||||
"""Fail closed when a pinned job has drifted from its model snapshot.
|
||||
|
||||
@@ -5,7 +5,7 @@ Provides commands for managing the WebSocket gateway with multi-bot support.
|
||||
"""
|
||||
|
||||
import sys
|
||||
from typing import Optional
|
||||
from typing import List, Optional
|
||||
|
||||
import typer
|
||||
|
||||
@@ -221,6 +221,13 @@ def gateway_start(
|
||||
help="Seconds the event loop may stall before the watchdog trips a "
|
||||
"restart (default ~15s = 5s x 3 strikes; #3410)",
|
||||
),
|
||||
trusted_proxy: Optional[List[str]] = typer.Option(
|
||||
None, "--trusted-proxy",
|
||||
help="CIDR/IP of an upstream proxy or tunnel to trust for real-client-IP "
|
||||
"resolution (repeatable, e.g. --trusted-proxy 10.0.0.0/8). Behind a "
|
||||
"reverse proxy this keeps per-IP rate-limits and the operator id keyed "
|
||||
"on the real client; unattributable proxies fail closed (#5312)",
|
||||
),
|
||||
):
|
||||
"""Start the gateway server.
|
||||
|
||||
@@ -409,6 +416,11 @@ def gateway_start(
|
||||
drain_marker=drain_marker,
|
||||
watchdog=True if watchdog else None,
|
||||
watchdog_timeout=watchdog_timeout,
|
||||
trusted_proxies=(
|
||||
list(trusted_proxy)
|
||||
if isinstance(trusted_proxy, (list, tuple))
|
||||
else None
|
||||
),
|
||||
)
|
||||
raise typer.Exit(code if isinstance(code, int) else 0)
|
||||
|
||||
@@ -627,6 +639,10 @@ def gateway_status(
|
||||
output.print_info(
|
||||
f"DLQ ({name}): praisonai bot dlq list --path {dlq_path}"
|
||||
)
|
||||
output.print_info(
|
||||
"Outbox: praisonai gateway outbox list "
|
||||
"[--status failed] [--target <chan>]"
|
||||
)
|
||||
if probe and config and os.path.exists(config):
|
||||
import asyncio
|
||||
|
||||
@@ -1866,6 +1882,154 @@ diagnostics_app = typer.Typer(
|
||||
app.add_typer(diagnostics_app, name="diagnostics")
|
||||
|
||||
|
||||
# ── Outbound queue operator surface (parity with `praisonai bot dlq`) ──
|
||||
outbox_app = typer.Typer(
|
||||
help="Inspect / retry / purge the durable outbound delivery queue.",
|
||||
no_args_is_help=True,
|
||||
)
|
||||
app.add_typer(outbox_app, name="outbox")
|
||||
|
||||
|
||||
def _resolve_outbox_path(path: Optional[str]) -> str:
|
||||
"""Resolve the durable outbox sqlite path.
|
||||
|
||||
Precedence: explicit ``--path`` → ``PRAISONAI_OUTBOX_PATH`` env →
|
||||
the canonical gateway outbox (``~/.praisonai/state/gateway_outbox.sqlite``)
|
||||
that ``GatewayServer.scheduled_outbox`` writes.
|
||||
"""
|
||||
import os
|
||||
|
||||
if path:
|
||||
return os.path.expanduser(path)
|
||||
return os.path.expanduser(
|
||||
os.environ.get(
|
||||
"PRAISONAI_OUTBOX_PATH",
|
||||
"~/.praisonai/state/gateway_outbox.sqlite",
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@outbox_app.command("list")
|
||||
def outbox_list(
|
||||
path: Optional[str] = typer.Option(
|
||||
None, "--path", "-p", help="Path to outbox sqlite file."
|
||||
),
|
||||
status: Optional[str] = typer.Option(
|
||||
None, "--status", "-s",
|
||||
help="Filter by status (pending/failed/permanent_failure/recovered/sent).",
|
||||
),
|
||||
target: Optional[str] = typer.Option(
|
||||
None, "--target", "-t", help="Filter by exact target (e.g. telegram:12345)."
|
||||
),
|
||||
limit: int = typer.Option(20, "--limit", "-n", help="Max entries to show."),
|
||||
):
|
||||
"""List outbound entries (newest first) with target/status/attempts/error."""
|
||||
import time
|
||||
|
||||
from praisonai_bot.bots import OutboundQueue
|
||||
|
||||
outbox = OutboundQueue(path=_resolve_outbox_path(path), read_only=True)
|
||||
entries = outbox.list(status=status, target=target, limit=limit)
|
||||
if not entries:
|
||||
typer.echo(f"Outbox empty (path={outbox.path})")
|
||||
raise typer.Exit(0)
|
||||
|
||||
typer.echo(
|
||||
f"Outbox {outbox.path} — {outbox.size()} entries "
|
||||
f"(showing {len(entries)}):"
|
||||
)
|
||||
# Print the FULL tracking key: operators copy it verbatim into `outbox
|
||||
# retry`/`outbox purge`, so truncating it (with an ellipsis over the id)
|
||||
# would make listed entries unactionable. Metadata follows on the next line.
|
||||
now = time.time()
|
||||
for e in entries:
|
||||
key = f"{e.target}:{e.idempotency_key}:{e.id}"
|
||||
err = (e.error or "")[:80]
|
||||
age = f"{int(now - e.ts)}s"
|
||||
typer.echo(f" key={key}")
|
||||
typer.echo(
|
||||
f" target={e.target} status={e.status} "
|
||||
f"attempts={e.attempts} age={age}"
|
||||
+ (f" error={err}" if err else "")
|
||||
)
|
||||
|
||||
|
||||
@outbox_app.command("stats")
|
||||
def outbox_stats(
|
||||
path: Optional[str] = typer.Option(None, "--path", "-p"),
|
||||
):
|
||||
"""Show a per-status breakdown of the outbound queue."""
|
||||
from praisonai_bot.bots import OutboundQueue
|
||||
|
||||
outbox = OutboundQueue(path=_resolve_outbox_path(path), read_only=True)
|
||||
counts = outbox.stats()
|
||||
if not counts:
|
||||
typer.echo(f"Outbox empty (path={outbox.path})")
|
||||
raise typer.Exit(0)
|
||||
|
||||
typer.echo(f"Outbox {outbox.path}:")
|
||||
for status in sorted(counts):
|
||||
typer.echo(f" {status:<18} {counts[status]}")
|
||||
|
||||
|
||||
@outbox_app.command("retry")
|
||||
def outbox_retry(
|
||||
key: str = typer.Argument(..., help="Entry key (target:idempotency:id)."),
|
||||
path: Optional[str] = typer.Option(None, "--path", "-p"),
|
||||
):
|
||||
"""Requeue a specific entry for the next drain (e.g. after channel recovery)."""
|
||||
import asyncio
|
||||
|
||||
from praisonai_bot.bots import OutboundQueue
|
||||
|
||||
outbox = OutboundQueue(path=_resolve_outbox_path(path))
|
||||
ok = asyncio.run(outbox.retry(key))
|
||||
if ok:
|
||||
typer.echo("requeued 1 entry")
|
||||
else:
|
||||
typer.echo("no matching entry to requeue (unknown key or already sent)")
|
||||
raise typer.Exit(1)
|
||||
|
||||
|
||||
@outbox_app.command("purge")
|
||||
def outbox_purge(
|
||||
key: Optional[str] = typer.Argument(
|
||||
None, help="Entry key to delete. Omit with --all to clear everything."
|
||||
),
|
||||
path: Optional[str] = typer.Option(None, "--path", "-p"),
|
||||
all_entries: bool = typer.Option(
|
||||
False, "--all", help="Delete ALL entries instead of a single key."
|
||||
),
|
||||
yes: bool = typer.Option(False, "--yes", "-y", help="Skip confirmation."),
|
||||
):
|
||||
"""Delete a single outbound entry by key, or all entries with --all."""
|
||||
from praisonai_bot.bots import OutboundQueue
|
||||
|
||||
outbox = OutboundQueue(path=_resolve_outbox_path(path))
|
||||
|
||||
if all_entries:
|
||||
n = outbox.size()
|
||||
if n == 0:
|
||||
typer.echo("Outbox already empty.")
|
||||
raise typer.Exit(0)
|
||||
if not yes and not typer.confirm(
|
||||
f"Delete all {n} entries from {outbox.path}?"
|
||||
):
|
||||
raise typer.Exit(0)
|
||||
typer.echo(f"purged {outbox.purge()} entries")
|
||||
return
|
||||
|
||||
if not key:
|
||||
typer.echo("Provide an entry key or pass --all to clear the outbox.")
|
||||
raise typer.Exit(1)
|
||||
|
||||
if outbox.purge_entry(key):
|
||||
typer.echo("purged 1 entry")
|
||||
else:
|
||||
typer.echo("no matching entry to purge (unknown key)")
|
||||
raise typer.Exit(1)
|
||||
|
||||
|
||||
@diagnostics_app.command("export")
|
||||
def gateway_diagnostics_export(
|
||||
config: str = typer.Option(
|
||||
|
||||
@@ -109,7 +109,7 @@ _START_FLAG_KEYS = (
|
||||
"agent_file", "config_file", "drain_timeout", "max_concurrent_runs",
|
||||
"queue_depth", "overflow_policy", "reliability", "openai_api", "mcp",
|
||||
"identity_store", "scale_to_zero", "idle_minutes", "drain_marker",
|
||||
"watchdog", "watchdog_timeout",
|
||||
"watchdog", "watchdog_timeout", "trusted_proxies",
|
||||
)
|
||||
|
||||
|
||||
@@ -188,6 +188,7 @@ class GatewayHandler:
|
||||
drain_marker: Optional[str] = None,
|
||||
watchdog: Optional[bool] = None,
|
||||
watchdog_timeout: Optional[float] = None,
|
||||
trusted_proxies: Optional[list] = None,
|
||||
) -> int:
|
||||
"""Start the gateway server.
|
||||
|
||||
@@ -269,6 +270,7 @@ class GatewayHandler:
|
||||
"drain_marker": drain_marker,
|
||||
"watchdog": watchdog,
|
||||
"watchdog_timeout": watchdog_timeout,
|
||||
"trusted_proxies": trusted_proxies,
|
||||
}
|
||||
|
||||
def _commit_start_flags() -> None:
|
||||
@@ -303,6 +305,9 @@ class GatewayHandler:
|
||||
self._gateway._queue_depth_override = queue_depth
|
||||
if overflow_policy is not None:
|
||||
self._gateway._overflow_policy_override = overflow_policy
|
||||
# CLI --trusted-proxy overrides gateway.trusted_proxies in YAML (#5312)
|
||||
if trusted_proxies:
|
||||
self._gateway._trusted_proxies_override = list(trusted_proxies)
|
||||
# CLI --reliability preset overrides gateway.reliability in YAML (#2531)
|
||||
if reliability is not None:
|
||||
self._gateway._reliability_override = reliability
|
||||
@@ -378,6 +383,12 @@ class GatewayHandler:
|
||||
|
||||
# Standard WebSocket-only mode
|
||||
config = GatewayConfig(host=host, port=port)
|
||||
# CLI --trusted-proxy applies in no-config mode too (#5312): stamp it on
|
||||
# the config directly since start_with_config's YAML wiring is skipped.
|
||||
if trusted_proxies:
|
||||
config.trusted_proxies = [
|
||||
str(p).strip() for p in trusted_proxies if str(p).strip()
|
||||
]
|
||||
self._gateway = WebSocketGateway(
|
||||
config=config, openai_api=openai_api, mcp=mcp
|
||||
)
|
||||
|
||||
@@ -49,6 +49,15 @@ from praisonaiagents.gateway.protocols import (
|
||||
evaluate_pressure,
|
||||
)
|
||||
|
||||
try: # Trusted-proxy ingress attribution (Issue #5312); optional at import time.
|
||||
from praisonaiagents.gateway.protocols import (
|
||||
resolve_ingress_attribution,
|
||||
parse_forwarded_for,
|
||||
)
|
||||
except ImportError: # pragma: no cover - core predates ingress attribution
|
||||
resolve_ingress_attribution = None # type: ignore[assignment]
|
||||
parse_forwarded_for = None # type: ignore[assignment]
|
||||
|
||||
try: # Server→client interactive request/reply over the transport (Issue #5351).
|
||||
from praisonaiagents.gateway.protocols import (
|
||||
GatewayServerRequest,
|
||||
@@ -1331,6 +1340,15 @@ class WebSocketGateway:
|
||||
self._ledger_heartbeat_task: Optional[asyncio.Task] = None
|
||||
self._ledger_heartbeat_interval: float = 15.0
|
||||
|
||||
# Issue #5153: shed the coldest *already-persisted* live sessions before
|
||||
# the cgroup memory ceiling OOM-kills the process. Reuses the core
|
||||
# planner (``plan_pressure_evictions``) + the concrete stdlib cgroup
|
||||
# reader that the bot warm-agent cache already uses; folded into the
|
||||
# existing liveness-heartbeat tick, so there is no new task and it is a
|
||||
# no-op on hosts without a cgroup limit or without a session store.
|
||||
self._memory_pressure: Optional[Any] = None
|
||||
self._pressure_headroom: float = 0.9
|
||||
|
||||
@property
|
||||
def is_running(self) -> bool:
|
||||
return self._is_running
|
||||
@@ -1589,6 +1607,56 @@ class WebSocketGateway:
|
||||
status_code=403,
|
||||
)
|
||||
|
||||
def _client_subject(peer_ip, headers) -> str:
|
||||
"""Resolve the per-IP policy subject for an inbound request.
|
||||
|
||||
Consults the pure core ``resolve_ingress_attribution`` contract
|
||||
(Issue #5312) so rate-limiting, the pre-auth budget, and the
|
||||
operator id all key on the *real* client behind a declared trusted
|
||||
proxy — and fail closed (one bucket keyed on the socket peer, never
|
||||
a spoofable header) for anything proxy-shaped but unattributable.
|
||||
|
||||
When core predates the contract, or no ``trusted_proxies`` are
|
||||
configured, this returns the raw socket peer — today's behaviour.
|
||||
"""
|
||||
peer = peer_ip or "unknown"
|
||||
# Precedence (highest first) so a live config reload is honoured
|
||||
# and a removed proxy stops being trusted without a restart (#5312):
|
||||
# 1. CLI ``--trusted-proxy`` override — operator-pinned, wins always.
|
||||
# 2. Live YAML ``gateway.trusted_proxies`` from ``_loaded_config`` —
|
||||
# re-read every request so reload adds/removals take effect.
|
||||
# 3. ``self.config.trusted_proxies`` — the Python/no-config value.
|
||||
override = getattr(self, "_trusted_proxies_override", None)
|
||||
trusted: list = []
|
||||
if override:
|
||||
trusted = [str(p).strip() for p in override if str(p).strip()]
|
||||
else:
|
||||
loaded = getattr(self, "_loaded_config", None) or {}
|
||||
gw = loaded.get("gateway", {}) if isinstance(loaded, dict) else {}
|
||||
if isinstance(gw, dict) and "trusted_proxies" in gw:
|
||||
raw = gw.get("trusted_proxies")
|
||||
if isinstance(raw, (list, tuple)):
|
||||
trusted = [str(p).strip() for p in raw if str(p).strip()]
|
||||
else:
|
||||
trusted = [
|
||||
str(p).strip()
|
||||
for p in (getattr(self.config, "trusted_proxies", None) or [])
|
||||
if str(p).strip()
|
||||
]
|
||||
if resolve_ingress_attribution is None or not trusted:
|
||||
return peer
|
||||
attr = resolve_ingress_attribution(
|
||||
peer_ip=peer,
|
||||
forwarded_for=parse_forwarded_for(
|
||||
headers.get("x-forwarded-for")
|
||||
),
|
||||
real_ip=headers.get("x-real-ip"),
|
||||
trusted_proxies=trusted,
|
||||
)
|
||||
if attr.fail_closed:
|
||||
return f"proxy:{peer}"
|
||||
return attr.client_ip
|
||||
|
||||
def _resolve_operator_identity(request) -> str:
|
||||
"""Derive a stable, non-secret operator identity for the audit trail.
|
||||
|
||||
@@ -1616,7 +1684,10 @@ class WebSocketGateway:
|
||||
digest = hashlib.sha256(token.encode("utf-8")).hexdigest()[:12]
|
||||
return f"operator:{digest}"
|
||||
client_ip = request.client.host if request.client else None
|
||||
return f"operator:{client_ip}" if client_ip else "gateway"
|
||||
if not client_ip:
|
||||
return "gateway"
|
||||
subject = _client_subject(client_ip, request.headers)
|
||||
return f"operator:{subject}"
|
||||
|
||||
async def info(request):
|
||||
auth_err = _check_auth(request)
|
||||
@@ -1660,11 +1731,16 @@ class WebSocketGateway:
|
||||
)
|
||||
|
||||
async def websocket_endpoint(websocket: WebSocket):
|
||||
# Get client IP for rate limiting
|
||||
client_ip = websocket.client.host if websocket.client else "unknown"
|
||||
# Raw socket peer: used for the loopback exemption decision (a
|
||||
# trusted proxy hop is not loopback) and as the fail-closed bucket.
|
||||
peer_ip = websocket.client.host if websocket.client else "unknown"
|
||||
# Resolve the real client behind a declared trusted proxy so per-IP
|
||||
# rate-limit/budget buckets are not collapsed onto the proxy peer
|
||||
# (Issue #5312). Falls back to the peer when no proxy is trusted.
|
||||
client_ip = _client_subject(peer_ip, websocket.headers)
|
||||
|
||||
# Rate limiting for WebSocket upgrades (exempt loopback per acceptance criteria)
|
||||
if not is_loopback(client_ip) and not is_loopback(self._host):
|
||||
if not is_loopback(peer_ip) and not is_loopback(self._host):
|
||||
if not _ws_upgrade_rate.allow("ws_upgrade", client_ip):
|
||||
retry = _ws_upgrade_rate.time_until_allowed("ws_upgrade", client_ip)
|
||||
await _reject_connection(
|
||||
@@ -1685,7 +1761,7 @@ class WebSocketGateway:
|
||||
# once so a hostile client cannot park many half-open connections up
|
||||
# to max_connections. Loopback is exempt so local CLIs are never
|
||||
# locked out. The slot is released on auth-success or on close.
|
||||
_ip_is_loopback = is_loopback(client_ip) or is_loopback(self._host)
|
||||
_ip_is_loopback = is_loopback(peer_ip) or is_loopback(self._host)
|
||||
_released = {"done": False}
|
||||
|
||||
def _release_budget():
|
||||
@@ -1896,7 +1972,8 @@ class WebSocketGateway:
|
||||
if auth_err:
|
||||
return auth_err
|
||||
|
||||
client_ip = request.client.host if request.client else "unknown"
|
||||
peer_ip = request.client.host if request.client else "unknown"
|
||||
client_ip = _client_subject(peer_ip, request.headers)
|
||||
if not _approval_rate.allow("approval_pending", client_ip):
|
||||
retry = _approval_rate.time_until_allowed("approval_pending", client_ip)
|
||||
return JSONResponse(
|
||||
@@ -1922,7 +1999,8 @@ class WebSocketGateway:
|
||||
if scope_err:
|
||||
return scope_err
|
||||
|
||||
client_ip = request.client.host if request.client else "unknown"
|
||||
peer_ip = request.client.host if request.client else "unknown"
|
||||
client_ip = _client_subject(peer_ip, request.headers)
|
||||
if not _approval_rate.allow("approval_resolve", client_ip):
|
||||
retry = _approval_rate.time_until_allowed("approval_resolve", client_ip)
|
||||
return JSONResponse(
|
||||
@@ -1999,7 +2077,8 @@ class WebSocketGateway:
|
||||
if scope_err:
|
||||
return scope_err
|
||||
|
||||
client_ip = request.client.host if request.client else "unknown"
|
||||
peer_ip = request.client.host if request.client else "unknown"
|
||||
client_ip = _client_subject(peer_ip, request.headers)
|
||||
if not _approval_rate.allow("approval_allowlist", client_ip):
|
||||
retry = _approval_rate.time_until_allowed("approval_allowlist", client_ip)
|
||||
return JSONResponse(
|
||||
@@ -2988,7 +3067,10 @@ class WebSocketGateway:
|
||||
return merged
|
||||
|
||||
def _merge_watchdog_overrides(
|
||||
self, watchdog_cfg: Optional[Dict[str, Any]]
|
||||
self,
|
||||
watchdog_cfg: Optional[Dict[str, Any]],
|
||||
*,
|
||||
yaml_present: bool = False,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
"""Fold CLI watchdog overrides into the YAML ``watchdog`` block (#3410).
|
||||
|
||||
@@ -3001,6 +3083,21 @@ class WebSocketGateway:
|
||||
"""
|
||||
enable = getattr(self, "_watchdog_override", None)
|
||||
timeout = getattr(self, "_watchdog_timeout_override", None)
|
||||
# Issue #5362: fall back to the public ``GatewayConfig`` fields so a
|
||||
# Python embedder building ``GatewayConfig(watchdog=True, …)`` enables
|
||||
# the backstop without a YAML block or CLI flag. Precedence: CLI flag >
|
||||
# explicit YAML (block *or* scalar, incl. ``watchdog: false``) > Python
|
||||
# config field. ``yaml_present`` distinguishes an explicit scalar/block
|
||||
# from an absent key so an explicit ``watchdog: false`` is honoured and
|
||||
# not silently re-enabled by ``GatewayConfig(watchdog=True)``.
|
||||
if enable is None and not yaml_present:
|
||||
cfg_enable = getattr(self.config, "watchdog", None)
|
||||
if cfg_enable:
|
||||
enable = True
|
||||
if timeout is None:
|
||||
cfg_timeout = getattr(self.config, "watchdog_timeout", None)
|
||||
if cfg_timeout is not None:
|
||||
timeout = cfg_timeout
|
||||
if enable is None and timeout is None:
|
||||
return watchdog_cfg
|
||||
|
||||
@@ -5057,39 +5154,211 @@ class WebSocketGateway:
|
||||
``LIVENESS_TIMEOUT`` and its state released. A connection with no bound
|
||||
session yet (stalled pre-``hello``/``join`` handshake) is evaluated
|
||||
against its per-connection ``_client_last_seen`` clock so it too ages
|
||||
out. No-op when the policy is disabled (``interval_ms == 0``).
|
||||
out. Heartbeat/reap is a no-op when the policy is disabled
|
||||
(``interval_ms == 0``), but the loop still runs so the memory-pressure
|
||||
sweep (issue #5153) keeps shedding cold sessions — that OOM guard must
|
||||
not depend on liveness being enabled.
|
||||
"""
|
||||
policy = self._liveness_policy()
|
||||
if not policy.enabled:
|
||||
return
|
||||
# When liveness is disabled we still need a housekeeping cadence for the
|
||||
# pressure sweep; fall back to the config's interval (or the policy
|
||||
# default) rather than exiting, so shedding is never silently disabled.
|
||||
interval = policy.interval_seconds
|
||||
if not policy.enabled:
|
||||
from praisonaiagents.gateway.protocols import LivenessPolicy
|
||||
|
||||
interval = LivenessPolicy().interval_seconds
|
||||
while self._is_running:
|
||||
try:
|
||||
await asyncio.sleep(interval)
|
||||
if self._draining:
|
||||
continue
|
||||
now = time.time()
|
||||
for client_id in list(self._clients.keys()):
|
||||
session_id = self._client_sessions.get(client_id)
|
||||
session = self._sessions.get(session_id) if session_id else None
|
||||
if session is not None:
|
||||
last_activity = session.last_activity
|
||||
else:
|
||||
# No bound session yet (pre-``hello``/``join``): fall
|
||||
# back to the per-connection last-seen clock so a
|
||||
# stalled handshake ages out instead of living forever.
|
||||
last_activity = self._client_last_seen.get(client_id, now)
|
||||
if policy.evaluate(last_activity, now) is LivenessDecision.REAP:
|
||||
await self._reap_session(client_id)
|
||||
else:
|
||||
await self._send_to_client(
|
||||
client_id, {"type": EventType.PING.value}
|
||||
if policy.enabled:
|
||||
for client_id in list(self._clients.keys()):
|
||||
session_id = self._client_sessions.get(client_id)
|
||||
session = (
|
||||
self._sessions.get(session_id) if session_id else None
|
||||
)
|
||||
if session is not None:
|
||||
last_activity = session.last_activity
|
||||
else:
|
||||
# No bound session yet (pre-``hello``/``join``): fall
|
||||
# back to the per-connection last-seen clock so a
|
||||
# stalled handshake ages out instead of living forever.
|
||||
last_activity = self._client_last_seen.get(client_id, now)
|
||||
if policy.evaluate(last_activity, now) is LivenessDecision.REAP:
|
||||
await self._reap_session(client_id)
|
||||
else:
|
||||
await self._send_to_client(
|
||||
client_id, {"type": EventType.PING.value}
|
||||
)
|
||||
# Issue #5153: on the same housekeeping tick, shed the coldest
|
||||
# already-persisted sessions if the resident set is approaching
|
||||
# the cgroup budget — before the kernel OOM-kills the process.
|
||||
# Runs regardless of whether liveness ping/reap is enabled.
|
||||
self._sweep_sessions_under_pressure()
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception:
|
||||
logger.exception("Liveness heartbeat/reaper iteration failed")
|
||||
|
||||
def _sweep_sessions_under_pressure(self) -> int:
|
||||
"""Soft-evict the coldest already-persisted live sessions under RSS pressure.
|
||||
|
||||
Issue #5153: the gateway's own runtime enactor for the core planner,
|
||||
mirroring ``BotSessionManager.sweep_under_pressure`` (the warm-agent
|
||||
cache). It reads the real container budget/anonymous RSS from a
|
||||
:class:`praisonaiagents.gateway.MemoryPressureProtocol` (the stdlib
|
||||
``CgroupMemoryPressure`` reader), asks
|
||||
:func:`praisonaiagents.gateway.plan_pressure_evictions` which sessions to
|
||||
shed, and evicts them LRU-first via ``close_session(persist=True)`` — the
|
||||
transcript is left durable so the next ``join`` transparently rehydrates
|
||||
it. It re-samples RSS as it goes and stops as soon as RSS is back within
|
||||
``headroom`` of the budget, so it sheds the minimum.
|
||||
|
||||
Persistence-gated: a session is only evictable when a session store is
|
||||
configured (its transcript is durable and rebuildable), it is not
|
||||
currently executing a turn (never abort live work), and **no live
|
||||
WebSocket is bound to it** — a connected client's next ordinary
|
||||
``message`` looks the session up in ``_sessions`` directly and would be
|
||||
silently dropped (only ``hello``/``join`` rehydrate), so evicting a
|
||||
bound session would strand that client. A no-op (returns ``0``) when no
|
||||
store is configured or the platform reports no cgroup budget, preserving
|
||||
today's behaviour exactly.
|
||||
"""
|
||||
# No durable store ⇒ nothing is rebuildable ⇒ evicting would lose data.
|
||||
if self._session_store is None or not self._sessions:
|
||||
return 0
|
||||
try:
|
||||
from praisonaiagents.gateway import (
|
||||
WarmSession,
|
||||
plan_pressure_evictions,
|
||||
)
|
||||
except Exception: # pragma: no cover — old/absent core
|
||||
return 0
|
||||
if self._memory_pressure is None:
|
||||
try:
|
||||
from .memory_pressure import CgroupMemoryPressure
|
||||
|
||||
self._memory_pressure = CgroupMemoryPressure()
|
||||
except Exception: # pragma: no cover — defensive
|
||||
return 0
|
||||
mp = self._memory_pressure
|
||||
try:
|
||||
budget = mp.cgroup_limit_mb()
|
||||
rss = mp.anon_rss_mb()
|
||||
except Exception: # pragma: no cover — defensive
|
||||
return 0
|
||||
headroom = self._pressure_headroom
|
||||
# Sessions with a live WebSocket bound to them are excluded from the
|
||||
# eviction candidate set: evicting them would strand the connection
|
||||
# (Finding: ordinary ``message`` frames do not rehydrate). Only cold,
|
||||
# disconnected-but-resumable sessions are candidates.
|
||||
bound_session_ids = set(self._client_sessions.values())
|
||||
warm = [
|
||||
WarmSession(
|
||||
session_id=sid,
|
||||
last_activity=session.last_activity,
|
||||
in_flight=bool(getattr(session, "_is_executing", False)),
|
||||
flushed=True,
|
||||
)
|
||||
for sid, session in self._sessions.items()
|
||||
if sid not in bound_session_ids
|
||||
]
|
||||
if not warm:
|
||||
return 0
|
||||
plan = plan_pressure_evictions(budget, rss, warm, headroom_ratio=headroom)
|
||||
if not plan:
|
||||
return 0
|
||||
try:
|
||||
target = float(budget) * float(headroom) # type: ignore[arg-type]
|
||||
except (TypeError, ValueError):
|
||||
target = 0.0
|
||||
evicted = 0
|
||||
for sid in plan:
|
||||
session = self._sessions.get(sid)
|
||||
# Re-check at enact time: a turn may have started, the session may
|
||||
# already be gone, or a client may have (re)bound to it since the
|
||||
# plan was computed. Never abort live work (an executing turn) and
|
||||
# never strand a now-bound connection.
|
||||
if session is None or getattr(session, "_is_executing", False):
|
||||
continue
|
||||
if sid in set(self._client_sessions.values()):
|
||||
continue
|
||||
if self._evict_persisted_session(sid):
|
||||
evicted += 1
|
||||
try:
|
||||
if mp.anon_rss_mb() <= target:
|
||||
break
|
||||
except Exception: # pragma: no cover — defensive
|
||||
break
|
||||
if evicted:
|
||||
logger.info(
|
||||
"WebSocketGateway: soft-evicted %d cold session(s) under "
|
||||
"memory pressure (durable, rehydrate on next join)",
|
||||
evicted,
|
||||
)
|
||||
return evicted
|
||||
|
||||
def _evict_persisted_session(self, session_id: str) -> bool:
|
||||
"""Remove ``session_id`` from memory **only after** its transcript is durable.
|
||||
|
||||
Unlike ``close_session(persist=True)`` — which best-effort persists but
|
||||
still drops the in-memory session even when the store write fails — this
|
||||
enactor refuses to evict a session whose final snapshot could not be
|
||||
persisted. Under memory pressure the live session may hold the only copy
|
||||
of recent messages/events, so a failed persist must leave it in memory
|
||||
(no data loss) rather than count it as safely shed. ``add_message``
|
||||
signals failure two ways — a raised exception *or* a falsy return
|
||||
(``SessionStoreProtocol.add_message`` returns ``bool`` and both built-in
|
||||
stores return ``False`` on a failed write); both must retain the session.
|
||||
"""
|
||||
session = self._sessions.get(session_id)
|
||||
if session is None:
|
||||
return False
|
||||
store = self._session_store
|
||||
if store is None: # pragma: no cover — guarded by caller
|
||||
return False
|
||||
try:
|
||||
persisted = store.add_message(
|
||||
session_id=session_id,
|
||||
role="system",
|
||||
content="Session closed",
|
||||
metadata={"session_data": session.to_dict()},
|
||||
)
|
||||
except Exception as exc: # persist raised ⇒ keep session in memory
|
||||
logger.warning(
|
||||
"WebSocketGateway: skipped pressure eviction of %s — "
|
||||
"persistence failed (%s); session retained in memory",
|
||||
session_id,
|
||||
exc,
|
||||
)
|
||||
return False
|
||||
# A store may report a failed write via a falsy return instead of
|
||||
# raising; treat that identically so we never drop the only live copy.
|
||||
# ``None`` (legacy stores with no return) is treated as success to
|
||||
# preserve today's behaviour for stores that persist without reporting.
|
||||
if persisted is False:
|
||||
logger.warning(
|
||||
"WebSocketGateway: skipped pressure eviction of %s — "
|
||||
"store reported a failed write; session retained in memory",
|
||||
session_id,
|
||||
)
|
||||
return False
|
||||
# Durable now: mark resumable, close, and drop from the live set.
|
||||
try:
|
||||
session.close()
|
||||
except Exception: # pragma: no cover — defensive
|
||||
pass
|
||||
resume_window = self.config.session_config.resume_window
|
||||
self._session_ttls[session_id] = time.time() + resume_window
|
||||
self._sessions.pop(session_id, None)
|
||||
logger.info(
|
||||
"Session %s persisted, resumable for %ss", session_id, resume_window
|
||||
)
|
||||
return True
|
||||
|
||||
async def _send_to_client(self, client_id: str, data: Dict[str, Any]) -> None:
|
||||
"""Send data to a specific client through its bounded outbound queue."""
|
||||
ws = self._clients.get(client_id)
|
||||
@@ -10791,6 +11060,29 @@ class WebSocketGateway:
|
||||
self._host = gw_cfg["host"]
|
||||
if gw_cfg.get("port"):
|
||||
self._port = int(gw_cfg["port"])
|
||||
# Issue #5312: resolve the operator-declared trusted-proxy set. CLI
|
||||
# ``--trusted-proxy`` (``_trusted_proxies_override``) wins over the YAML
|
||||
# ``gateway.trusted_proxies``. Stamp the resolved set onto ``self.config``
|
||||
# so the ingress-attribution seam reads it uniformly. Only overwrite when
|
||||
# a source actually supplies one — YAML that *omits* ``trusted_proxies``
|
||||
# must not wipe a value a Python caller set on ``GatewayConfig`` before
|
||||
# ``start_with_config`` (Greptile P1).
|
||||
_tp_override = getattr(self, "_trusted_proxies_override", None)
|
||||
_trusted = None
|
||||
if _tp_override:
|
||||
_trusted = [str(p).strip() for p in _tp_override if str(p).strip()]
|
||||
elif "trusted_proxies" in gw_cfg:
|
||||
_raw_tp = gw_cfg.get("trusted_proxies")
|
||||
_trusted = (
|
||||
[str(p).strip() for p in _raw_tp if str(p).strip()]
|
||||
if isinstance(_raw_tp, (list, tuple))
|
||||
else []
|
||||
)
|
||||
if _trusted is not None:
|
||||
try:
|
||||
self.config.trusted_proxies = _trusted
|
||||
except Exception: # pragma: no cover - config is always a GatewayConfig
|
||||
pass
|
||||
# Issue #3593: the multi-bot CLI builds this gateway with a *default*
|
||||
# session config, so ``__init__`` already picked a store before this
|
||||
# YAML loaded. Re-read the ``session:`` block here and re-select the
|
||||
@@ -10879,6 +11171,13 @@ class WebSocketGateway:
|
||||
drain_timeout_cfg = getattr(self, "_drain_timeout_override", None)
|
||||
if drain_timeout_cfg is None:
|
||||
drain_timeout_cfg = gw_cfg.get("drain_timeout")
|
||||
# Issue #5362: a Python embedder building ``GatewayConfig(drain_timeout=…)``
|
||||
# directly (no YAML key, no CLI flag) must still take effect, closing the
|
||||
# CLI/YAML/Python parity gap for the drain window. Only fall back when
|
||||
# the YAML key is *absent* — an explicit ``drain_timeout: null`` is an
|
||||
# operator choice to disable the drain and must win over the field.
|
||||
if drain_timeout_cfg is None and "drain_timeout" not in gw_cfg:
|
||||
drain_timeout_cfg = getattr(self.config, "drain_timeout", None)
|
||||
# YAML/env-substituted values may arrive as strings (e.g. "30");
|
||||
# coerce once so later ``> 0`` comparisons never raise TypeError.
|
||||
if drain_timeout_cfg is not None:
|
||||
@@ -11043,8 +11342,26 @@ class WebSocketGateway:
|
||||
# ``watchdog:`` block nested under ``gateway:``; a CLI ``--watchdog``
|
||||
# override (stamped on the instance) wins over / synthesises the YAML.
|
||||
# No-op unless enabled, so always-on gateways are unchanged.
|
||||
# Issue #5362: distinguish an explicit YAML key (block *or* scalar,
|
||||
# including ``watchdog: false``) from an absent one so an explicit
|
||||
# scalar disable is honoured over the Python ``GatewayConfig`` field.
|
||||
watchdog_yaml_present = "watchdog" in gw_cfg
|
||||
watchdog_cfg = gw_cfg.get("watchdog")
|
||||
watchdog_cfg = self._merge_watchdog_overrides(watchdog_cfg)
|
||||
# Normalise a scalar (``watchdog: true`` / ``false``) into the block
|
||||
# form ``_configure_watchdog`` expects, so an explicit scalar carries
|
||||
# its enabled decision rather than being ignored as a non-dict.
|
||||
if watchdog_yaml_present and not isinstance(watchdog_cfg, dict):
|
||||
_wd_scalar = watchdog_cfg
|
||||
if isinstance(_wd_scalar, str):
|
||||
_wd_enabled = _wd_scalar.strip().lower() in (
|
||||
"1", "true", "yes", "on"
|
||||
)
|
||||
else:
|
||||
_wd_enabled = bool(_wd_scalar)
|
||||
watchdog_cfg = {"enabled": _wd_enabled}
|
||||
watchdog_cfg = self._merge_watchdog_overrides(
|
||||
watchdog_cfg, yaml_present=watchdog_yaml_present
|
||||
)
|
||||
self._configure_watchdog(watchdog_cfg)
|
||||
|
||||
# Start channels + WebSocket server concurrently
|
||||
|
||||
@@ -567,6 +567,11 @@ class ScheduledAgentExecutor:
|
||||
logger.warning("Job '%s' execution failed: %s", job.id, e)
|
||||
err = str(e)
|
||||
duration = time.time() - started
|
||||
# Provider-quota hold: a 429 / quota failure parks the job past the
|
||||
# provider's own reset window so it stops re-firing and re-failing
|
||||
# every tick against a benched provider. ``e`` (not ``err``) is
|
||||
# passed so the classifier can read the response headers.
|
||||
held_until = self._maybe_hold_on_rate_limit(job, e)
|
||||
self._runner.mark_run(job, status="failed", error=err, duration=duration)
|
||||
if self._on_failure:
|
||||
self._on_failure(job, err)
|
||||
@@ -574,12 +579,20 @@ class ScheduledAgentExecutor:
|
||||
job=job, status="failed", error=err, duration=duration,
|
||||
)
|
||||
await asyncio.to_thread(self._audit_output, job, failed)
|
||||
await self._maybe_deliver_failure(job, failed)
|
||||
if held_until is not None:
|
||||
await self._maybe_deliver_hold(job, failed, held_until)
|
||||
else:
|
||||
await self._maybe_deliver_failure(job, failed)
|
||||
return failed
|
||||
finally:
|
||||
if _restore_tools is not None:
|
||||
_restore_tools()
|
||||
|
||||
# The turn reached the model and returned — clear any prior quota hold
|
||||
# (proof the provider's window reopened), fingerprinted so a stale error
|
||||
# can't keep the job parked.
|
||||
self._clear_quota_hold(job)
|
||||
|
||||
# Success - calculate duration before delivery
|
||||
duration = time.time() - started
|
||||
|
||||
@@ -871,6 +884,15 @@ class ScheduledAgentExecutor:
|
||||
if succeeded and pending_state_updates:
|
||||
self._persist_job_state(job, prior_state, pending_state_updates)
|
||||
|
||||
# Provider-quota hold: a backend turn that reached the model reopens the
|
||||
# window (clear any prior hold); a 429/quota error parks the job past
|
||||
# the provider's reset window like the native-agent path.
|
||||
held_until: Optional[float] = None
|
||||
if succeeded:
|
||||
self._clear_quota_hold(job)
|
||||
else:
|
||||
held_until = self._maybe_hold_on_rate_limit(job, Exception(str(error)))
|
||||
|
||||
status = "succeeded" if succeeded else "failed"
|
||||
self._runner.mark_run(
|
||||
job,
|
||||
@@ -896,7 +918,10 @@ class ScheduledAgentExecutor:
|
||||
)
|
||||
await asyncio.to_thread(self._audit_output, job, result)
|
||||
if not succeeded:
|
||||
await self._maybe_deliver_failure(job, result)
|
||||
if held_until is not None:
|
||||
await self._maybe_deliver_hold(job, result, held_until)
|
||||
else:
|
||||
await self._maybe_deliver_failure(job, result)
|
||||
return result
|
||||
|
||||
async def _execute_command(
|
||||
@@ -1365,6 +1390,118 @@ class ScheduledAgentExecutor:
|
||||
|
||||
return None, _restore
|
||||
|
||||
# ── provider-quota hold helpers ──────────────────────────────────
|
||||
|
||||
def _maybe_hold_on_rate_limit(
|
||||
self, job: "ScheduleJob", error: Exception,
|
||||
) -> Optional[float]:
|
||||
"""Park ``job`` past a provider's rate-limit window on a 429/quota fail.
|
||||
|
||||
Returns the epoch the job is parked until when a hold was applied, else
|
||||
``None`` (non-quota failure, no usable reset window, or policy disabled).
|
||||
The decision is the core pure helper ``quota_hold_from_failure`` — the
|
||||
reschedule contract lives next to ``is_due``; here we only apply it to
|
||||
the in-memory job so the subsequent ``mark_run`` persists it. The parked
|
||||
instant coalesces intervening fires: ``is_due`` returns ``False`` until
|
||||
it elapses, so the job stops re-firing and re-failing every tick.
|
||||
"""
|
||||
policy = self._run_policy
|
||||
if policy is None or not getattr(policy, "hold_on_rate_limit", False):
|
||||
return None
|
||||
try:
|
||||
from praisonaiagents.scheduler import quota_hold_from_failure
|
||||
except Exception: # pragma: no cover - core primitive always present
|
||||
return None
|
||||
slack = getattr(policy, "hold_slack_seconds", 60.0)
|
||||
hold_until = quota_hold_from_failure(error, time.time(), slack)
|
||||
if hold_until is None:
|
||||
return None
|
||||
# Coalesce: never shorten an existing (later) hold. A fingerprint of the
|
||||
# schedule ties the hold to the job's current cadence so a stale error
|
||||
# cannot keep re-parking a job whose schedule changed underneath it.
|
||||
prior = getattr(job, "hold_until", None)
|
||||
if isinstance(prior, (int, float)) and prior > hold_until:
|
||||
hold_until = prior
|
||||
try:
|
||||
job.hold_until = hold_until
|
||||
except Exception as e: # pragma: no cover - defensive
|
||||
logger.warning("Could not set quota hold for job '%s': %s", job.id, e)
|
||||
return None
|
||||
logger.info(
|
||||
"Job '%s' held until %.0f after provider rate-limit", job.id, hold_until,
|
||||
)
|
||||
return hold_until
|
||||
|
||||
@staticmethod
|
||||
def _clear_quota_hold(job: "ScheduleJob") -> None:
|
||||
"""Clear a prior quota hold once a run reaches the model.
|
||||
|
||||
A completed model turn is proof the provider's window reopened, so the
|
||||
park is lifted; the subsequent ``mark_run`` persists the cleared field.
|
||||
No-op when the job was never parked.
|
||||
"""
|
||||
if getattr(job, "hold_until", None) is not None:
|
||||
try:
|
||||
job.hold_until = None
|
||||
except Exception: # pragma: no cover - defensive
|
||||
pass
|
||||
|
||||
async def _maybe_deliver_hold(
|
||||
self, job: "ScheduleJob", result: JobResult, held_until: float,
|
||||
) -> None:
|
||||
"""Emit one "held until N" notice for a parked job via the incident path.
|
||||
|
||||
Reuses the same de-duplicating incident/alert plumbing as
|
||||
:meth:`_maybe_deliver_failure` so a benched provider produces a single
|
||||
notice, not one alert per skipped tick. Falls back to nothing extra when
|
||||
failure delivery is disabled — the hold itself still applies.
|
||||
"""
|
||||
if self._run_policy is None or not self._run_policy.deliver_on_failure:
|
||||
return
|
||||
delivery = getattr(job, "delivery", None)
|
||||
if not delivery or not self._can_deliver(delivery):
|
||||
return
|
||||
# A hold is a distinct operational state (the job is now silenced for a
|
||||
# whole provider-reset window) and is emitted at most once per window:
|
||||
# ``is_due`` coalesces intervening fires, so this method only runs on the
|
||||
# tick that *newly* parks the job. It must therefore NOT be gated on the
|
||||
# failure ``alert_after_failures`` threshold — otherwise a job configured
|
||||
# to alert only after N failures would park silently and the operator
|
||||
# would never learn it was held (greptile #3). We still fold the failure
|
||||
# through the incident tracker so failure/recovery accounting stays
|
||||
# consistent, but deliver the hold notice regardless of its alert state.
|
||||
pending: Optional[Dict[str, Any]] = None
|
||||
observed = self._observe_incident(job, result)
|
||||
if observed is not None:
|
||||
_incident, pending = observed
|
||||
try:
|
||||
when = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(held_until))
|
||||
except Exception: # pragma: no cover - defensive
|
||||
when = str(held_until)
|
||||
summary = (
|
||||
f"⏸️ Scheduled job '{getattr(job, 'name', job.id)}' held until "
|
||||
f"{when} (provider rate-limited): {result.error or 'quota exceeded'}"
|
||||
)
|
||||
try:
|
||||
if await self._dispatch_delivery(delivery, summary):
|
||||
result.delivered = True
|
||||
self._commit_incident(job, pending)
|
||||
logger.info("Delivered hold notice for job '%s'", job.id)
|
||||
else:
|
||||
result.delivery_error = (
|
||||
f"hold-notice delivery to {delivery.channel}:"
|
||||
f"{delivery.channel_id} did not complete"
|
||||
)
|
||||
logger.warning(
|
||||
"Hold-notice delivery did not complete for job '%s'; "
|
||||
"leaving incident un-alerted to retry", job.id,
|
||||
)
|
||||
except Exception as e: # pragma: no cover - defensive
|
||||
result.delivery_error = str(e)
|
||||
logger.warning(
|
||||
"Hold-notice delivery failed for job '%s': %s", job.id, e,
|
||||
)
|
||||
|
||||
# ── run-policy helpers ───────────────────────────────────────────
|
||||
|
||||
def _assemble_scan_target(self, agent: Any, message: str) -> str:
|
||||
|
||||
@@ -0,0 +1,211 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Tests for the operator inspect/retry/purge surface on the durable outbound
|
||||
queue (Issue #5366), giving parity with the inbound DLQ CLI.
|
||||
|
||||
Covers ``OutboundQueue.list`` (with status/target filters), ``stats``,
|
||||
``retry`` (requeue a failed/permanent entry for the next drain), and
|
||||
``purge_entry`` (targeted single-entry delete).
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
|
||||
from praisonai_bot.bots._outbox import OutboundQueue
|
||||
|
||||
|
||||
def test_list_returns_enqueued_entries(tmp_path):
|
||||
async def run():
|
||||
q = OutboundQueue(path=str(tmp_path / "o.sqlite"))
|
||||
await q.enqueue("m1", "telegram:1", {"text": "a"})
|
||||
await q.enqueue("m2", "slack:C0", {"text": "b"})
|
||||
entries = q.list()
|
||||
assert {e.idempotency_key for e in entries} == {"m1", "m2"}
|
||||
assert all(e.status == "pending" for e in entries)
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
def test_list_filters_by_status_and_target(tmp_path):
|
||||
async def run():
|
||||
q = OutboundQueue(path=str(tmp_path / "o.sqlite"))
|
||||
k1 = await q.enqueue("m1", "telegram:1", {"text": "a"})
|
||||
await q.enqueue("m2", "slack:C0", {"text": "b"})
|
||||
await q.mark_failed(k1, "boom", permanent=True)
|
||||
|
||||
failed = q.list(status="permanent_failure")
|
||||
assert [e.idempotency_key for e in failed] == ["m1"]
|
||||
|
||||
by_target = q.list(target="slack:C0")
|
||||
assert [e.idempotency_key for e in by_target] == ["m2"]
|
||||
|
||||
assert q.list(status="pending", target="slack:C0")[0].idempotency_key == "m2"
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
def test_stats_counts_per_status(tmp_path):
|
||||
async def run():
|
||||
q = OutboundQueue(path=str(tmp_path / "o.sqlite"))
|
||||
k1 = await q.enqueue("m1", "telegram:1", {"text": "a"})
|
||||
await q.enqueue("m2", "telegram:2", {"text": "b"})
|
||||
await q.mark_failed(k1, "boom", permanent=True)
|
||||
|
||||
stats = q.stats()
|
||||
assert stats.get("pending") == 1
|
||||
assert stats.get("permanent_failure") == 1
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
def test_retry_requeues_permanent_failure(tmp_path):
|
||||
async def run():
|
||||
q = OutboundQueue(path=str(tmp_path / "o.sqlite"))
|
||||
key = await q.enqueue("m1", "telegram:1", {"text": "a"})
|
||||
await q.mark_failed(key, "boom", permanent=True)
|
||||
assert q.stats().get("permanent_failure") == 1
|
||||
|
||||
assert await q.retry(key) is True
|
||||
entry = q.list()[0]
|
||||
assert entry.status == "pending"
|
||||
assert entry.attempts == 0
|
||||
assert entry.error is None
|
||||
|
||||
# Second retry is a no-op (already pending, not terminal-failed).
|
||||
assert await q.retry(key) is False
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
def test_retry_unknown_key_returns_false(tmp_path):
|
||||
async def run():
|
||||
q = OutboundQueue(path=str(tmp_path / "o.sqlite"))
|
||||
assert await q.retry("telegram:1:999") is False
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
def test_purge_entry_removes_single_entry(tmp_path):
|
||||
async def run():
|
||||
q = OutboundQueue(path=str(tmp_path / "o.sqlite"))
|
||||
key = await q.enqueue("m1", "telegram:1", {"text": "a"})
|
||||
await q.enqueue("m2", "telegram:2", {"text": "b"})
|
||||
|
||||
assert q.purge_entry(key) is True
|
||||
remaining = q.list()
|
||||
assert [e.idempotency_key for e in remaining] == ["m2"]
|
||||
|
||||
assert q.purge_entry(key) is False
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
def test_retried_entry_drains_successfully(tmp_path):
|
||||
async def run():
|
||||
q = OutboundQueue(path=str(tmp_path / "o.sqlite"))
|
||||
key = await q.enqueue("m1", "telegram:1", {"text": "a"})
|
||||
await q.mark_failed(key, "chat unavailable", permanent=True)
|
||||
|
||||
await q.retry(key)
|
||||
|
||||
sent = []
|
||||
|
||||
async def sender(target, payload):
|
||||
sent.append((target, payload))
|
||||
return True
|
||||
|
||||
succeeded, failed = await q.drain(sender)
|
||||
assert succeeded == 1 and failed == 0
|
||||
assert sent == [("telegram:1", {"text": "a"})]
|
||||
assert q.list(status="sent")[0].idempotency_key == "m1"
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
def test_retry_does_not_touch_recovered_entry(tmp_path):
|
||||
"""A crash-recovered entry must keep its 'recovered' safeguard; manual retry
|
||||
is a no-op so the next drain still reconciles/annotates it (no unlabelled
|
||||
duplicate)."""
|
||||
|
||||
async def run():
|
||||
path = tmp_path / "o.sqlite"
|
||||
q1 = OutboundQueue(path=str(path))
|
||||
key = await q1.enqueue("m1", "telegram:1", {"text": "a"})
|
||||
# Simulate an in-flight send interrupted by a crash: leave it 'sending'.
|
||||
with q1._lock, __import__("contextlib").closing(q1._connect()) as conn:
|
||||
conn.execute(
|
||||
"UPDATE outbound_queue SET status = 'sending' WHERE id = ?",
|
||||
(q1._extract_id_from_key(key),),
|
||||
)
|
||||
conn.commit()
|
||||
# Reopen: crash recovery flips 'sending' -> 'recovered'.
|
||||
q2 = OutboundQueue(path=str(path))
|
||||
assert q2.list()[0].status == "recovered"
|
||||
|
||||
assert await q2.retry(key) is False
|
||||
assert q2.list()[0].status == "recovered"
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
def test_purge_refuses_in_flight_sending_entry(tmp_path):
|
||||
"""purge_entry must not delete a row an active send is awaiting a terminal
|
||||
write on."""
|
||||
|
||||
async def run():
|
||||
path = tmp_path / "o.sqlite"
|
||||
q = OutboundQueue(path=str(path))
|
||||
key = await q.enqueue("m1", "telegram:1", {"text": "a"})
|
||||
with q._lock, __import__("contextlib").closing(q._connect()) as conn:
|
||||
conn.execute(
|
||||
"UPDATE outbound_queue SET status = 'sending' WHERE id = ?",
|
||||
(q._extract_id_from_key(key),),
|
||||
)
|
||||
conn.commit()
|
||||
|
||||
assert q.purge_entry(key) is False
|
||||
assert q.list()[0].status == "sending"
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
def test_targeted_ops_reject_mismatched_key(tmp_path):
|
||||
"""A key whose id exists but whose target/idempotency differ must not mutate
|
||||
the real entry (stale/mistyped key protection)."""
|
||||
|
||||
async def run():
|
||||
q = OutboundQueue(path=str(tmp_path / "o.sqlite"))
|
||||
key = await q.enqueue("m1", "telegram:1", {"text": "a"})
|
||||
await q.mark_failed(key, "boom", permanent=True)
|
||||
entry_id = q._extract_id_from_key(key)
|
||||
|
||||
wrong_key = f"telegram:999:wrong:{entry_id}"
|
||||
assert await q.retry(wrong_key) is False
|
||||
assert q.purge_entry(wrong_key) is False
|
||||
# Real entry untouched and still actionable via its true key.
|
||||
assert q.list()[0].status == "permanent_failure"
|
||||
assert await q.retry(key) is True
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
def test_read_only_does_not_recover_sending_rows(tmp_path):
|
||||
"""Opening the outbox read_only (operator inspection) must NOT flip live
|
||||
'sending' rows to 'recovered' — doing so would let a running drain re-claim
|
||||
and duplicate an in-flight message."""
|
||||
|
||||
async def run():
|
||||
path = tmp_path / "o.sqlite"
|
||||
q = OutboundQueue(path=str(path))
|
||||
key = await q.enqueue("m1", "telegram:1", {"text": "a"})
|
||||
with q._lock, __import__("contextlib").closing(q._connect()) as conn:
|
||||
conn.execute(
|
||||
"UPDATE outbound_queue SET status = 'sending' WHERE id = ?",
|
||||
(q._extract_id_from_key(key),),
|
||||
)
|
||||
conn.commit()
|
||||
|
||||
ro = OutboundQueue(path=str(path), read_only=True)
|
||||
assert ro.list()[0].status == "sending"
|
||||
assert ro.stats().get("sending") == 1
|
||||
|
||||
asyncio.run(run())
|
||||
@@ -0,0 +1,118 @@
|
||||
"""Issue #5152 — Defer inbound journal completion until the reply is delivered.
|
||||
|
||||
By default ``BotSessionManager.chat()`` marks the durable inbound journal entry
|
||||
complete on the agent's clean exit — *before* the adapter delivers the reply —
|
||||
so a crash in the send window loses the reply with no replay. When
|
||||
``defer_journal_completion=True`` the entry is left pending and the key is
|
||||
stashed so the adapter can call ``complete_last_journal_entry()`` only *after*
|
||||
the reply is durably delivered. Off by default → behaviour unchanged.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from praisonai_bot.bots import InboundJournal
|
||||
from praisonai_bot.bots._session import BotSessionManager
|
||||
|
||||
|
||||
class _Agent:
|
||||
def __init__(self, reply: str = "answer"):
|
||||
self.chat_history = []
|
||||
self._reply = reply
|
||||
|
||||
def chat(self, prompt):
|
||||
self.chat_history.append({"role": "user", "content": prompt})
|
||||
self.chat_history.append({"role": "assistant", "content": self._reply})
|
||||
return self._reply
|
||||
|
||||
|
||||
class TestDeferredJournalCompletion:
|
||||
@pytest.mark.asyncio
|
||||
async def test_default_completes_on_agent_exit(self, tmp_path):
|
||||
journal = InboundJournal(path=tmp_path / "ingress.sqlite")
|
||||
mgr = BotSessionManager(platform="telegram", ingress_journal=journal)
|
||||
|
||||
out = await mgr.chat(
|
||||
_Agent(), "u1", "hi", message_id="m1", account="acct"
|
||||
)
|
||||
|
||||
assert out == "answer"
|
||||
# Legacy path: settled on agent completion, so a redelivery is deduped.
|
||||
assert journal.receive("telegram", "acct", "u1", "m1", {}) is None
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_deferred_leaves_entry_pending_until_delivered(self, tmp_path):
|
||||
journal = InboundJournal(path=tmp_path / "ingress.sqlite")
|
||||
mgr = BotSessionManager(
|
||||
platform="telegram",
|
||||
ingress_journal=journal,
|
||||
defer_journal_completion=True,
|
||||
)
|
||||
|
||||
out = await mgr.chat(
|
||||
_Agent(), "u1", "hi", message_id="m1", account="acct"
|
||||
)
|
||||
|
||||
assert out == "answer"
|
||||
# Not yet delivered → entry stays pending (a crash here would replay).
|
||||
assert journal.pending_count() == 1
|
||||
|
||||
# Adapter delivers the reply, then settles the inbound.
|
||||
assert mgr.complete_last_journal_entry() is True
|
||||
assert journal.pending_count() == 0
|
||||
# Now a redelivery is deduped.
|
||||
assert journal.receive("telegram", "acct", "u1", "m1", {}) is None
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_deferred_entry_replayable_before_delivery(self, tmp_path):
|
||||
journal = InboundJournal(path=tmp_path / "ingress.sqlite")
|
||||
mgr = BotSessionManager(
|
||||
platform="telegram",
|
||||
ingress_journal=journal,
|
||||
defer_journal_completion=True,
|
||||
)
|
||||
|
||||
await mgr.chat(_Agent(), "u1", "hi", message_id="m1", account="acct")
|
||||
|
||||
# Simulate a crash before delivery: the entry is left pending (not
|
||||
# completed), so a redelivery of the same inbound is not deduped away
|
||||
# and returns the pending key for reprocessing rather than silently
|
||||
# losing the reply.
|
||||
assert journal.pending_count() == 1
|
||||
assert journal.receive("telegram", "acct", "u1", "m1", {}) is not None
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_explicit_key_settles_only_that_turn(self, tmp_path):
|
||||
"""Regression (Issue #5152): concurrent turns run under different
|
||||
per-storage-key locks, so one turn's delivery acknowledgement must
|
||||
settle *only* that turn's inbound. Passing the explicit journal key to
|
||||
``complete_last_journal_entry`` guarantees it never settles another
|
||||
still-undelivered turn's entry.
|
||||
"""
|
||||
journal = InboundJournal(path=tmp_path / "ingress.sqlite")
|
||||
mgr = BotSessionManager(
|
||||
platform="telegram",
|
||||
ingress_journal=journal,
|
||||
defer_journal_completion=True,
|
||||
)
|
||||
|
||||
# Two different users → two deferred entries, both pending.
|
||||
await mgr.chat(_Agent(), "uA", "hi", message_id="mA", account="acct")
|
||||
key_a = mgr._last_journal_key
|
||||
await mgr.chat(_Agent(), "uB", "hi", message_id="mB", account="acct")
|
||||
key_b = mgr._last_journal_key
|
||||
|
||||
assert key_a is not None and key_b is not None and key_a != key_b
|
||||
assert journal.pending_count() == 2
|
||||
|
||||
# Turn A's reply is delivered first (out of order): settle ONLY A.
|
||||
assert mgr.complete_last_journal_entry(key_a) is True
|
||||
assert journal.pending_count() == 1
|
||||
# B is still pending/redeliverable — not clobbered by A's ack.
|
||||
assert mgr._last_journal_key == key_b
|
||||
assert journal.receive("telegram", "acct", "uB", "mB", {}) is not None
|
||||
|
||||
# Now settle B.
|
||||
assert mgr.complete_last_journal_entry(key_b) is True
|
||||
assert journal.pending_count() == 0
|
||||
@@ -0,0 +1,169 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Tests for the opt-in, privacy-safe activity surface (issue #5314).
|
||||
|
||||
During an output-silent tool run in PROGRESS mode, an operator-configured
|
||||
phrase keyed by tool *category* may be shown instead of the raw tool name.
|
||||
Tool names/args/URLs are never surfaced when a catalogue is configured, and
|
||||
behaviour is unchanged when it is not.
|
||||
"""
|
||||
|
||||
from praisonai_bot.bots._streaming import (
|
||||
_SAFE_ACTIVITY_FALLBACK,
|
||||
DraftStreamer,
|
||||
StreamingConfig,
|
||||
StreamingMode,
|
||||
resolve_activity_category,
|
||||
)
|
||||
|
||||
|
||||
def _streamer(config: StreamingConfig) -> DraftStreamer:
|
||||
class _Adapter:
|
||||
capabilities = {"live_edit": True}
|
||||
|
||||
s = DraftStreamer(_Adapter(), "chan", config, platform="telegram")
|
||||
s._message_id = "m1"
|
||||
return s
|
||||
|
||||
|
||||
def test_resolve_activity_category_mappings():
|
||||
assert resolve_activity_category("web_search") == "web"
|
||||
assert resolve_activity_category("duckduckgo_search") == "search"
|
||||
assert resolve_activity_category("run_shell_command") == "shell"
|
||||
assert resolve_activity_category("python_interpreter") == "code"
|
||||
assert resolve_activity_category("read_file") == "file"
|
||||
assert resolve_activity_category("mcp_tool") == "mcp"
|
||||
|
||||
|
||||
def test_resolve_activity_category_falls_back_to_default():
|
||||
assert resolve_activity_category("some_unmapped_tool") == "default"
|
||||
assert resolve_activity_category("") == "default"
|
||||
assert resolve_activity_category(None) == "default"
|
||||
|
||||
|
||||
def test_activity_line_uses_curated_phrase_not_tool_name():
|
||||
cfg = StreamingConfig(
|
||||
mode=StreamingMode.PROGRESS,
|
||||
progress_prefix="",
|
||||
activity_phrases={
|
||||
"web": "Searching the web…",
|
||||
"shell": "Running a command…",
|
||||
"default": "Working on it…",
|
||||
},
|
||||
)
|
||||
s = _streamer(cfg)
|
||||
s._current_tool = "web_fetch_secret_url"
|
||||
content = s._render_content()
|
||||
assert content == "Searching the web…"
|
||||
# The raw tool name must never leak.
|
||||
assert "web_fetch_secret_url" not in content
|
||||
|
||||
|
||||
def test_activity_line_default_phrase_for_unmapped_category():
|
||||
cfg = StreamingConfig(
|
||||
mode=StreamingMode.PROGRESS,
|
||||
progress_prefix="",
|
||||
activity_phrases={"default": "Working on it…"},
|
||||
)
|
||||
s = _streamer(cfg)
|
||||
s._current_tool = "totally_custom_tool"
|
||||
assert s._render_content() == "Working on it…"
|
||||
|
||||
|
||||
def test_no_catalogue_preserves_legacy_behaviour():
|
||||
cfg = StreamingConfig(mode=StreamingMode.PROGRESS, progress_prefix="")
|
||||
s = _streamer(cfg)
|
||||
s._current_tool = "web_search"
|
||||
assert s._render_content() == "Running web_search..."
|
||||
|
||||
|
||||
def test_from_dict_inline_activity_phrases():
|
||||
cfg = StreamingConfig.from_dict(
|
||||
{"mode": "progress", "activity_phrases": {"web": "Searching…"}}
|
||||
)
|
||||
assert cfg.activity_phrases == {"web": "Searching…"}
|
||||
|
||||
|
||||
def test_from_dict_activity_status_block_enabled():
|
||||
cfg = StreamingConfig.from_dict(
|
||||
{
|
||||
"mode": "progress",
|
||||
"activity_status": {
|
||||
"enabled": True,
|
||||
"phrases": {"web": "Searching the web…", "default": "Working…"},
|
||||
},
|
||||
}
|
||||
)
|
||||
assert cfg.activity_phrases == {
|
||||
"web": "Searching the web…",
|
||||
"default": "Working…",
|
||||
}
|
||||
|
||||
|
||||
def test_from_dict_activity_status_disabled_yields_none():
|
||||
cfg = StreamingConfig.from_dict(
|
||||
{
|
||||
"mode": "progress",
|
||||
"activity_status": {"enabled": False, "phrases": {"web": "x"}},
|
||||
}
|
||||
)
|
||||
assert cfg.activity_phrases is None
|
||||
|
||||
|
||||
def test_from_dict_defaults_to_none():
|
||||
cfg = StreamingConfig.from_dict({"mode": "progress"})
|
||||
assert cfg.activity_phrases is None
|
||||
|
||||
|
||||
def test_partial_catalogue_never_leaks_tool_name():
|
||||
# Catalogue enabled but missing both the matched category AND a default:
|
||||
# opting in must fall back to a safe generic line, NOT the raw tool name.
|
||||
cfg = StreamingConfig(
|
||||
mode=StreamingMode.PROGRESS,
|
||||
progress_prefix="",
|
||||
activity_phrases={"web": "Searching the web…"},
|
||||
)
|
||||
s = _streamer(cfg)
|
||||
s._current_tool = "run_shell_command"
|
||||
content = s._render_content()
|
||||
assert content == _SAFE_ACTIVITY_FALLBACK
|
||||
assert "run_shell_command" not in content
|
||||
|
||||
|
||||
def test_schema_retains_activity_status_and_forwards_to_config():
|
||||
# The bot/gateway streaming schema must not silently drop the documented
|
||||
# activity_status block; it must survive validation and reach from_dict.
|
||||
from praisonai_bot.bots._config_schema import StreamingConfigSchema
|
||||
|
||||
schema = StreamingConfigSchema(
|
||||
mode="progress",
|
||||
activity_status={
|
||||
"enabled": True,
|
||||
"phrases": {"web": "Searching the web…", "default": "Working…"},
|
||||
},
|
||||
)
|
||||
assert schema.activity_status is not None
|
||||
cfg = StreamingConfig.from_dict(schema.model_dump())
|
||||
assert cfg.activity_phrases == {
|
||||
"web": "Searching the web…",
|
||||
"default": "Working…",
|
||||
}
|
||||
|
||||
|
||||
def test_feed_style_does_not_bypass_curated_phrases():
|
||||
# In feed style with a catalogue enabled, the raw feed (which folds tool
|
||||
# names/summaries) must NOT be rendered; the curated line is shown instead.
|
||||
cfg = StreamingConfig(
|
||||
mode=StreamingMode.PROGRESS,
|
||||
progress_prefix="",
|
||||
progress_style="feed",
|
||||
activity_phrases={"web": "Searching the web…", "default": "Working…"},
|
||||
)
|
||||
s = _streamer(cfg)
|
||||
s._current_tool = "web_fetch"
|
||||
# Simulate a populated feed carrying a raw tool name.
|
||||
s._progress_lines = ["run_shell_command: rm -rf /secret"]
|
||||
content = s._render_content()
|
||||
assert content == "Searching the web…"
|
||||
assert "run_shell_command" not in content
|
||||
assert "secret" not in content
|
||||
@@ -108,6 +108,26 @@ def test_merge_watchdog_overrides_cli_wins_over_yaml():
|
||||
assert merged["enabled"] is True
|
||||
|
||||
|
||||
def test_merge_watchdog_config_field_enables_when_yaml_absent():
|
||||
# Issue #5362: GatewayConfig(watchdog=True) enables the backstop when no
|
||||
# CLI flag and no YAML key are set (yaml_present=False).
|
||||
gw = _make_gateway()
|
||||
gw.config.watchdog = True
|
||||
merged = gw._merge_watchdog_overrides(None, yaml_present=False)
|
||||
assert merged["enabled"] is True
|
||||
|
||||
|
||||
def test_explicit_yaml_scalar_false_beats_config_field():
|
||||
# Issue #5362: an explicit ``watchdog: false`` (yaml_present=True) must not
|
||||
# be re-enabled by GatewayConfig(watchdog=True).
|
||||
gw = _make_gateway()
|
||||
gw.config.watchdog = True
|
||||
merged = gw._merge_watchdog_overrides(
|
||||
{"enabled": False}, yaml_present=True
|
||||
)
|
||||
assert merged["enabled"] is False
|
||||
|
||||
|
||||
def test_arm_and_disarm_are_safe_without_watchdog():
|
||||
gw = _make_gateway()
|
||||
# No watchdog configured: both are no-ops and must not raise.
|
||||
|
||||
@@ -0,0 +1,83 @@
|
||||
"""BotOS schedule-loop provider-quota hold (Issue #5386, greptile #1).
|
||||
|
||||
The BotOS ``_run_schedule_loop`` runs ``agent.chat`` directly rather than
|
||||
through :class:`ScheduledAgentExecutor`, so it needs the same provider-quota
|
||||
hold or a 429/quota failure re-fires every tick against a benched provider.
|
||||
These tests cover the pure ``_apply_schedule_quota_hold`` decision the loop
|
||||
applies before ``mark_run``: a 429 with a reset hint parks the job, a
|
||||
non-quota failure never parks, a later hold is never shortened, and a success
|
||||
clears a prior park.
|
||||
"""
|
||||
|
||||
import time
|
||||
|
||||
import pytest
|
||||
|
||||
try:
|
||||
from praisonai_bot.bots.botos import BotOS
|
||||
except ImportError: # pragma: no cover - optional deps not installed
|
||||
BotOS = None
|
||||
|
||||
pytestmark = pytest.mark.skipif(BotOS is None, reason="praisonai bots not importable")
|
||||
|
||||
|
||||
class _Job:
|
||||
def __init__(self):
|
||||
self.name = "j"
|
||||
self.hold_until = None
|
||||
|
||||
|
||||
class _RateLimited(Exception):
|
||||
"""A 429-shaped provider error carrying a Retry-After header."""
|
||||
|
||||
def __init__(self, retry_after):
|
||||
super().__init__("Rate limit exceeded")
|
||||
self.status_code = 429
|
||||
|
||||
class _Resp:
|
||||
headers = {"retry-after": str(retry_after)}
|
||||
|
||||
self.response = _Resp()
|
||||
|
||||
|
||||
def test_rate_limit_failure_parks_botos_job():
|
||||
job = _Job()
|
||||
now = time.time()
|
||||
BotOS._apply_schedule_quota_hold(
|
||||
job, succeeded=False, error=_RateLimited(3600),
|
||||
)
|
||||
assert job.hold_until is not None
|
||||
# Parked past the ~3600s window (+ default 60s slack), allowing drift.
|
||||
assert now + 3600 <= job.hold_until <= now + 3600 + 120
|
||||
|
||||
|
||||
def test_non_quota_failure_does_not_park():
|
||||
job = _Job()
|
||||
BotOS._apply_schedule_quota_hold(
|
||||
job, succeeded=False, error=Exception("boom: invalid request"),
|
||||
)
|
||||
assert job.hold_until is None
|
||||
|
||||
|
||||
def test_never_shortens_an_existing_later_hold():
|
||||
job = _Job()
|
||||
job.hold_until = time.time() + 10_000
|
||||
prior = job.hold_until
|
||||
BotOS._apply_schedule_quota_hold(
|
||||
job, succeeded=False, error=_RateLimited(60),
|
||||
)
|
||||
# A shorter new window must not shorten a job already parked further out.
|
||||
assert job.hold_until == prior
|
||||
|
||||
|
||||
def test_success_clears_prior_hold():
|
||||
job = _Job()
|
||||
job.hold_until = time.time() + 3600
|
||||
BotOS._apply_schedule_quota_hold(job, succeeded=True, error=None)
|
||||
assert job.hold_until is None
|
||||
|
||||
|
||||
def test_failure_without_exception_is_inert():
|
||||
job = _Job()
|
||||
BotOS._apply_schedule_quota_hold(job, succeeded=False, error=None)
|
||||
assert job.hold_until is None
|
||||
@@ -11,6 +11,7 @@ import pytest
|
||||
|
||||
from praisonai_bot.bots._session import BotSessionManager
|
||||
from praisonai_bot.gateway.memory_pressure import CgroupMemoryPressure
|
||||
from praisonai_bot.gateway.server import GatewaySession, WebSocketGateway
|
||||
|
||||
|
||||
class _Store:
|
||||
@@ -230,3 +231,183 @@ def test_sweep_skips_cache_when_last_store_write_failed():
|
||||
mp = _Pressure(100.0, [1000.0])
|
||||
assert mgr.sweep_under_pressure(mp) == 0
|
||||
assert "a" in mgr._histories
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# WebSocketGateway live-session pressure shedding (Issue #5153)
|
||||
#
|
||||
# The gateway server is the primary layer that owns ``WebSocketGateway._sessions``
|
||||
# and must call the core planner on its housekeeping tick. These tests exercise
|
||||
# ``_sweep_sessions_under_pressure`` directly on a bare gateway wired with only
|
||||
# the attributes the method (and ``close_session``) touch.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class _SessionStore:
|
||||
"""Minimal SessionStore: records that a session's state was persisted."""
|
||||
|
||||
def __init__(self):
|
||||
self.persisted = {}
|
||||
|
||||
def add_message(self, session_id, role, content, metadata=None):
|
||||
self.persisted[session_id] = metadata
|
||||
|
||||
|
||||
def _bare_gateway(store, budget, rss_samples, headroom=0.9):
|
||||
"""A ``WebSocketGateway`` with just enough state to run the pressure sweep."""
|
||||
gw = WebSocketGateway.__new__(WebSocketGateway)
|
||||
gw._session_store = store
|
||||
gw._sessions = {}
|
||||
gw._clients = {}
|
||||
gw._client_sessions = {}
|
||||
gw._session_ttls = {}
|
||||
gw._memory_pressure = _Pressure(budget, rss_samples)
|
||||
gw._pressure_headroom = headroom
|
||||
|
||||
class _SC:
|
||||
resume_window = 3600
|
||||
class _Cfg:
|
||||
session_config = _SC()
|
||||
gw.config = _Cfg()
|
||||
return gw
|
||||
|
||||
|
||||
def _add_session(gw, sid, last_activity, executing=False):
|
||||
session = GatewaySession(_session_id=sid, _agent_id="agent")
|
||||
session._last_activity = last_activity
|
||||
session._is_executing = executing
|
||||
gw._sessions[sid] = session
|
||||
return session
|
||||
|
||||
|
||||
def test_gateway_sheds_coldest_session_first_until_within_budget():
|
||||
store = _SessionStore()
|
||||
# Budget 1000 MiB, target 900; start over at 1000, drop to 850 after the
|
||||
# first (coldest) eviction so exactly one session is shed.
|
||||
gw = _bare_gateway(store, 1000.0, [1000.0, 850.0])
|
||||
_add_session(gw, "cold", 1.0)
|
||||
_add_session(gw, "warm", 100.0)
|
||||
evicted = gw._sweep_sessions_under_pressure()
|
||||
assert evicted == 1
|
||||
assert "cold" not in gw._sessions # coldest dropped from memory
|
||||
assert "warm" in gw._sessions # warmest kept live
|
||||
assert "cold" in store.persisted # transcript left durable
|
||||
|
||||
|
||||
def test_gateway_never_evicts_executing_session():
|
||||
store = _SessionStore()
|
||||
gw = _bare_gateway(store, 100.0, [1000.0])
|
||||
_add_session(gw, "busy", 1.0, executing=True)
|
||||
assert gw._sweep_sessions_under_pressure() == 0
|
||||
assert "busy" in gw._sessions
|
||||
|
||||
|
||||
def test_gateway_noop_without_session_store():
|
||||
# No durable store ⇒ nothing rebuildable ⇒ never shed (would lose data).
|
||||
gw = _bare_gateway(None, 100.0, [1000.0])
|
||||
_add_session(gw, "a", 1.0)
|
||||
assert gw._sweep_sessions_under_pressure() == 0
|
||||
assert "a" in gw._sessions
|
||||
|
||||
|
||||
def test_gateway_noop_when_no_cgroup_budget():
|
||||
store = _SessionStore()
|
||||
gw = _bare_gateway(store, None, [1000.0]) # host without a cgroup limit
|
||||
_add_session(gw, "a", 1.0)
|
||||
assert gw._sweep_sessions_under_pressure() == 0
|
||||
assert "a" in gw._sessions
|
||||
|
||||
|
||||
def test_gateway_noop_when_within_budget():
|
||||
store = _SessionStore()
|
||||
gw = _bare_gateway(store, 1000.0, [500.0]) # RSS well under the 900 target
|
||||
_add_session(gw, "a", 1.0)
|
||||
assert gw._sweep_sessions_under_pressure() == 0
|
||||
assert "a" in gw._sessions
|
||||
|
||||
|
||||
def test_gateway_noop_when_no_sessions():
|
||||
store = _SessionStore()
|
||||
gw = _bare_gateway(store, 1000.0, [1000.0])
|
||||
assert gw._sweep_sessions_under_pressure() == 0
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Review fixes (PR #5237): never strand a bound client, never lose a session on
|
||||
# a failed persist, and shed even when liveness ping/reap is disabled.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_gateway_never_evicts_session_with_live_client():
|
||||
"""A session a live WebSocket is bound to is never shed (would strand it).
|
||||
|
||||
Ordinary ``message`` frames look the session up in ``_sessions`` directly and
|
||||
are silently dropped if it is gone (only ``hello``/``join`` rehydrate), so a
|
||||
connected client must keep its session in memory.
|
||||
"""
|
||||
store = _SessionStore()
|
||||
gw = _bare_gateway(store, 100.0, [1000.0])
|
||||
_add_session(gw, "bound", 1.0)
|
||||
gw._client_sessions["client-1"] = "bound" # a live connection holds it
|
||||
assert gw._sweep_sessions_under_pressure() == 0
|
||||
assert "bound" in gw._sessions
|
||||
assert "bound" not in store.persisted
|
||||
|
||||
|
||||
def test_gateway_sheds_disconnected_but_keeps_bound_session():
|
||||
"""Under pressure, only the disconnected cold session is shed."""
|
||||
store = _SessionStore()
|
||||
gw = _bare_gateway(store, 1000.0, [1000.0, 850.0])
|
||||
_add_session(gw, "cold", 1.0) # coldest, no client → evictable
|
||||
_add_session(gw, "bound", 2.0) # colder than warm but has a client
|
||||
gw._client_sessions["client-1"] = "bound"
|
||||
evicted = gw._sweep_sessions_under_pressure()
|
||||
assert evicted == 1
|
||||
assert "cold" not in gw._sessions
|
||||
assert "bound" in gw._sessions # protected despite being cold
|
||||
|
||||
|
||||
def test_gateway_keeps_session_when_persist_fails():
|
||||
"""A failed final-persist must leave the session in memory (no data loss)."""
|
||||
|
||||
class _FlakyStore(_SessionStore):
|
||||
def add_message(self, session_id, role, content, metadata=None):
|
||||
raise RuntimeError("store backend down")
|
||||
|
||||
store = _FlakyStore()
|
||||
gw = _bare_gateway(store, 1000.0, [1000.0, 1000.0])
|
||||
_add_session(gw, "cold", 1.0)
|
||||
assert gw._sweep_sessions_under_pressure() == 0
|
||||
assert "cold" in gw._sessions # retained — its only copy is in memory
|
||||
assert "cold" not in store.persisted
|
||||
|
||||
|
||||
def test_gateway_keeps_session_when_store_returns_false():
|
||||
"""A falsy ``add_message`` result (failed write, no exception) retains it.
|
||||
|
||||
``SessionStoreProtocol.add_message`` returns ``bool`` and both built-in
|
||||
stores return ``False`` when a write fails; treating that as success would
|
||||
drop the session's only current copy under pressure (Greptile P1).
|
||||
"""
|
||||
|
||||
class _FalsyStore(_SessionStore):
|
||||
def add_message(self, session_id, role, content, metadata=None):
|
||||
return False # write failed but did not raise
|
||||
|
||||
store = _FalsyStore()
|
||||
gw = _bare_gateway(store, 1000.0, [1000.0, 1000.0])
|
||||
_add_session(gw, "cold", 1.0)
|
||||
assert gw._sweep_sessions_under_pressure() == 0
|
||||
assert "cold" in gw._sessions # retained — the write did not persist
|
||||
assert "cold" not in gw._session_ttls
|
||||
assert gw._evict_persisted_session("cold") is False
|
||||
|
||||
|
||||
def test_gateway_evict_persisted_session_sets_resume_ttl():
|
||||
store = _SessionStore()
|
||||
gw = _bare_gateway(store, 1000.0, [1000.0])
|
||||
_add_session(gw, "cold", 1.0)
|
||||
assert gw._evict_persisted_session("cold") is True
|
||||
assert "cold" not in gw._sessions
|
||||
assert "cold" in store.persisted
|
||||
assert "cold" in gw._session_ttls # resumable window stamped
|
||||
Loaded 100 of 156 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user