Compare commits

...
6 Commits
Author SHA1 Message Date
Mervin Praison a795248cf7 Release v4.4.2 2026-02-03 09:15:50 +00:00
Mervin Praison 92542b7c3c Release v4.4.1 2026-02-02 20:26:43 +00:00
Mervin Praison 041c3d5897 Release v4.4.0 2026-02-02 17:57:56 +00:00
github-actions[bot] e890aade27 docs: auto-update api.md [skip ci] 2026-02-02 15:07:54 +00:00
Mervin Praison 4d262fc832 Release v4.3.1 2026-02-02 15:07:34 +00:00
github-actions[bot] c31c5b9fc7 docs: auto-update api.md [skip ci] 2026-02-02 10:05:53 +00:00
91 changed files with 6602 additions and 211 deletions

No files matched your search

+8
View File
@@ -81,6 +81,8 @@ Methods:
* <code title="class Agent">Agent.<a href="./src/praisonai-agents/praisonaiagents/agent/agent.py">run</a>(prompt: str, **kwargs)</code>
* <code title="class Agent">Agent.<a href="./src/praisonai-agents/praisonaiagents/agent/agent.py">run_autonomous</a>(prompt: str, max_iterations: Optional[int] = None, timeout_seconds: Optional[float] = None, completion_promise: Optional[str] = None, clear_context: bool = False)</code>
* <code title="class Agent">Agent.<a href="./src/praisonai-agents/praisonaiagents/agent/agent.py">run_autonomous_async</a>(prompt: str, max_iterations: Optional[int] = None, timeout_seconds: Optional[float] = None, completion_promise: Optional[str] = None, clear_context: bool = False)</code>
* <code title="class Agent">Agent.<a href="./src/praisonai-agents/praisonaiagents/agent/agent.py">run_until</a>(prompt: str, criteria: str, threshold: float = 8.0, max_iterations: int = 5, mode: str = 'optimize', on_iteration: Optional[Callable[[Any], None]] = None, verbose: bool = False) -> 'EvaluationLoopResult'</code>
* <code title="class Agent">Agent.<a href="./src/praisonai-agents/praisonaiagents/agent/agent.py">run_until_async</a>(prompt: str, criteria: str, threshold: float = 8.0, max_iterations: int = 5, mode: str = 'optimize', on_iteration: Optional[Callable[[Any], None]] = None, verbose: bool = False) -> 'EvaluationLoopResult'</code>
* <code title="class Agent">Agent.<a href="./src/praisonai-agents/praisonaiagents/agent/agent.py">session_id</a>() -> Optional[str]</code>
* <code title="class Agent">Agent.<a href="./src/praisonai-agents/praisonaiagents/agent/agent.py">skill_manager</a>()</code>
* <code title="class Agent">Agent.<a href="./src/praisonai-agents/praisonaiagents/agent/agent.py">start</a>(prompt: str = None, **kwargs)</code>
@@ -208,10 +210,15 @@ from praisonaiagents import Loop, Parallel, Pipeline, Repeat, Route, Task, Workf
Methods:
* <code title="class Task">Task.<a href="./src/praisonai-agents/praisonaiagents/task/task.py">depends_on</a>()</code>
* <code title="class Task">Task.<a href="./src/praisonai-agents/praisonaiagents/task/task.py">depends_on</a>(value)</code>
* <code title="class Task">Task.<a href="./src/praisonai-agents/praisonaiagents/task/task.py">evaluate_when</a>(context: Dict[str, Any]) -> bool</code>
* <code title="class Task">Task.<a href="./src/praisonai-agents/praisonaiagents/task/task.py">execute_callback</a>(task_output: TaskOutput) -> None</code>
* <code title="class Task">Task.<a href="./src/praisonai-agents/praisonaiagents/task/task.py">execute_callback_sync</a>(task_output: TaskOutput) -> None</code>
* <code title="class Task">Task.<a href="./src/praisonai-agents/praisonaiagents/task/task.py">get_next_task</a>(context: Dict[str, Any]) -> Optional[str]</code>
* <code title="class Task">Task.<a href="./src/praisonai-agents/praisonaiagents/task/task.py">initialize_memory</a>()</code>
* <code title="class Task">Task.<a href="./src/praisonai-agents/praisonaiagents/task/task.py">store_in_memory</a>(content: str, agent_name: str = None, task_id: str = None)</code>
* <code title="class Task">Task.<a href="./src/praisonai-agents/praisonaiagents/task/task.py">to_dict</a>() -> Dict[str, Any]</code>
* <code title="function">praisonaiagents.<a href="./src/praisonai-agents/praisonaiagents/workflows/workflows.py">loop</a>(step: Any = None, steps: Optional[List[Any]] = None, over: Optional[str] = None, from_csv: Optional[str] = None, from_file: Optional[str] = None, var_name: str = 'item', parallel: bool = False, max_workers: Optional[int] = None, output_variable: Optional[str] = None) -> Loop</code>
* <code title="function">praisonaiagents.<a href="./src/praisonai-agents/praisonaiagents/workflows/workflows.py">parallel</a>(steps: List) -> Parallel</code>
* <code title="function">praisonaiagents.<a href="./src/praisonai-agents/praisonaiagents/workflows/workflows.py">repeat</a>(step: Any, until: Optional[Callable[[WorkflowContext], bool]] = None, max_iterations: int = 10) -> Repeat</code>
@@ -1001,6 +1008,7 @@ Methods:
* <code title="cli">praisonai traces list <a href="./src/praisonai/praisonai/cli/commands/traces.py">--help</a></code>
* <code title="cli">praisonai traces status <a href="./src/praisonai/praisonai/cli/commands/traces.py">--help</a></code>
* <code title="cli">praisonai train agents <a href="./src/praisonai/praisonai/cli/commands/train.py">--help</a></code>
* <code title="cli">praisonai train apply <a href="./src/praisonai/praisonai/cli/commands/train.py">--help</a></code>
* <code title="cli">praisonai train list <a href="./src/praisonai/praisonai/cli/commands/train.py">--help</a></code>
* <code title="cli">praisonai train llm <a href="./src/praisonai/praisonai/cli/commands/train.py">--help</a></code>
* <code title="cli">praisonai train show <a href="./src/praisonai/praisonai/cli/commands/train.py">--help</a></code>
+1 -1
View File
@@ -16,7 +16,7 @@ RUN mkdir -p /root/.praison
# Install Python packages (using latest versions)
RUN pip install --no-cache-dir \
praisonai_tools \
"praisonai>=4.3.0" \
"praisonai>=4.4.2" \
"praisonai[chat]" \
"embedchain[github,youtube]"
+1 -1
View File
@@ -20,7 +20,7 @@ RUN mkdir -p /root/.praison
# Install Python packages (using latest versions)
RUN pip install --no-cache-dir \
praisonai_tools \
"praisonai>=4.3.0" \
"praisonai>=4.4.2" \
"praisonai[ui]" \
"praisonai[chat]" \
"praisonai[realtime]" \
+1 -1
View File
@@ -16,7 +16,7 @@ RUN mkdir -p /root/.praison
# Install Python packages (using latest versions)
RUN pip install --no-cache-dir \
praisonai_tools \
"praisonai>=4.3.0" \
"praisonai>=4.4.2" \
"praisonai[ui]" \
"praisonai[crewai]"
+43
View File
@@ -0,0 +1,43 @@
# EvaluationLoop Examples
Examples demonstrating iterative evaluation and improvement patterns.
## Quick Start
```bash
pip install praisonaiagents
export OPENAI_API_KEY=your-key
```
## Examples
| File | Description |
|------|-------------|
| `basic_evaluation_loop.py` | Simple loop: Run → Judge → Improve |
| `multi_step_task_loop.py` | Sequential tasks with context passing |
| `recipe_optimization_loop.py` | CLI-based recipe optimization |
## Pattern
```
┌─────────┐ ┌─────────┐ ┌──────────┐
│ Agent │ ──► │ Judge │ ──► │ Feedback │
│ runs │ │ scores │ │ improves │
└─────────┘ └─────────┘ └──────────┘
▲ │
└───────────────────────────────┘
(repeat until 8.0+)
```
## Run Examples
```bash
# Basic loop
python basic_evaluation_loop.py
# Multi-step with context
python multi_step_task_loop.py
# CLI recipe loop
python recipe_optimization_loop.py "research AI trends"
```
@@ -0,0 +1,73 @@
"""
Basic EvaluationLoop Example
Demonstrates the iterative evaluation loop pattern:
Run agent → Judge output → Improve → Repeat until threshold met.
Usage:
pip install praisonaiagents
export OPENAI_API_KEY=your-key
python basic_evaluation_loop.py
"""
from praisonaiagents import Agent
from praisonaiagents.eval import EvaluationLoop
def on_iteration(result):
"""Callback showing progress after each iteration."""
status = "✅" if result.score >= 8.0 else "⚠️" if result.score >= 6.0 else "❌"
print(f"{status} Iteration {result.iteration}: {result.score:.1f}/10")
if result.findings:
print(f" → {result.findings[0][:60]}...")
def main():
# Create an agent
agent = Agent(
name="writer",
instructions="You are a technical content writer. Write clear, accurate explanations."
)
# Run evaluation loop
loop = EvaluationLoop(
agent=agent,
criteria="""
Evaluate the response on:
1. CLARITY: Is the explanation easy to understand?
2. ACCURACY: Is the technical content correct?
3. COMPLETENESS: Does it cover the key aspects?
4. EXAMPLES: Does it include practical examples?
Score 1-10 based on all criteria.
""",
threshold=8.0,
max_iterations=5,
on_iteration=on_iteration,
verbose=False,
)
print("╔════════════════════════════════════════════════════════════╗")
print("║ Basic EvaluationLoop Example ║")
print("║ Pattern: Run → Judge → Improve → Repeat ║")
print("║ Target: 8.0/10 score ║")
print("╚════════════════════════════════════════════════════════════╝\n")
result = loop.run("Explain how API rate limiting works and why it matters")
print("\n" + "─" * 60)
print("RESULTS:")
print("─" * 60)
print(f"Success: {result.success}")
print(f"Final Score: {result.final_score:.1f}/10")
print(f"Iterations: {result.num_iterations}")
print(f"Score History: {[f'{s:.1f}' for s in result.score_history]}")
print(f"Duration: {result.total_duration_seconds:.2f}s")
print("\n" + "─" * 60)
print("FINAL OUTPUT (first 500 chars):")
print("─" * 60)
print(result.final_output[:500] + "..." if len(result.final_output) > 500 else result.final_output)
if __name__ == "__main__":
main()
@@ -0,0 +1,144 @@
"""
Multi-Step Task Loop Example
Demonstrates evaluating sequential task steps where each step can
reference output from previous steps.
Pattern:
1. Research → Evaluate → Improve
2. Analyze (using research) → Evaluate → Improve
3. Report (using analysis) → Evaluate → Improve
Usage:
pip install praisonaiagents
export OPENAI_API_KEY=your-key
python multi_step_task_loop.py
"""
from praisonaiagents import Agent
from praisonaiagents.eval import EvaluationLoop
def on_iteration(result):
"""Show iteration progress."""
status = "✅" if result.score >= 8.0 else "⚠️" if result.score >= 6.0 else "❌"
print(f" {status} Iteration {result.iteration}: {result.score:.1f}/10")
def run_step(step_name: str, agent: Agent, task: str, criteria: str, context: str = "") -> dict:
"""Run a single step through an evaluation loop."""
print(f"\n{'─'*60}")
print(f"STEP: {step_name}")
print(f"{'─'*60}")
# Include context from previous step if provided
full_task = task
if context:
full_task = f"{task}\n\nContext from previous step:\n{context[:1000]}"
loop = EvaluationLoop(
agent=agent,
criteria=criteria,
threshold=8.0,
max_iterations=3,
on_iteration=on_iteration,
verbose=False,
)
result = loop.run(full_task)
print(f" → Final: {result.final_score:.1f}/10 ({result.num_iterations} iterations)")
return {
"name": step_name,
"output": result.final_output,
"score": result.final_score,
"success": result.success,
"iterations": result.num_iterations,
}
def main():
print("╔════════════════════════════════════════════════════════════╗")
print("║ Multi-Step Task Loop Example ║")
print("║ Pattern: Step1 → Step2 → Step3 (with context passing) ║")
print("║ Each step: Run → Judge → Improve until 8.0+ ║")
print("╚════════════════════════════════════════════════════════════╝")
# Create specialized agents
researcher = Agent(
name="researcher",
instructions="You are a research specialist. Find and summarize key information."
)
analyst = Agent(
name="analyst",
instructions="You are an analyst. Identify patterns, insights, and implications."
)
writer = Agent(
name="writer",
instructions="You are a report writer. Create clear, actionable summaries."
)
# Define task steps
steps = [
{
"name": "Research",
"agent": researcher,
"task": "Research the top 3 trends in AI for 2024-2025. Provide brief descriptions.",
"criteria": "Covers 3 distinct trends with accurate descriptions.",
},
{
"name": "Analysis",
"agent": analyst,
"task": "Analyze the implications of these AI trends for software development.",
"criteria": "Identifies actionable insights and potential challenges.",
},
{
"name": "Report",
"agent": writer,
"task": "Create a brief executive summary (3-4 bullet points) for a CTO.",
"criteria": "Concise, actionable, appropriate for executive audience.",
},
]
# Run all steps with context passing
results = []
context = ""
for step in steps:
result = run_step(
step_name=step["name"],
agent=step["agent"],
task=step["task"],
criteria=step["criteria"],
context=context,
)
results.append(result)
context = result["output"] # Pass output as context to next step
# Summary
print("\n" + "═"*60)
print("SUMMARY")
print("═"*60)
total_iterations = sum(r["iterations"] for r in results)
avg_score = sum(r["score"] for r in results) / len(results)
all_success = all(r["success"] for r in results)
for r in results:
status = "✅" if r["success"] else "❌"
print(f" {status} {r['name']}: {r['score']:.1f}/10 ({r['iterations']} iterations)")
print(f"\n Total Iterations: {total_iterations}")
print(f" Average Score: {avg_score:.1f}/10")
print(f" All Steps Passed: {'Yes' if all_success else 'No'}")
print("\n" + "─"*60)
print("FINAL REPORT OUTPUT:")
print("─"*60)
print(results[-1]["output"][:600])
if __name__ == "__main__":
main()
@@ -0,0 +1,183 @@
"""
Recipe Optimization Loop (CLI-based)
Demonstrates the 5-cycle recipe improvement pattern using CLI commands:
Create (once) → Run → Judge → Optimize (loop 5x)
Usage:
pip install praisonai
export OPENAI_API_KEY=your-key
export TAVILY_API_KEY=your-key
python recipe_optimization_loop.py "research AI trends in healthcare"
"""
import subprocess
import sys
import os
import json
from pathlib import Path
def run_cli(cmd: str, cwd: str = None) -> tuple[int, str, str]:
"""Run a CLI command and return (exit_code, stdout, stderr)."""
result = subprocess.run(
cmd,
shell=True,
capture_output=True,
text=True,
cwd=cwd,
)
return result.returncode, result.stdout, result.stderr
def create_recipe(goal: str, work_dir: str) -> str:
"""Create a recipe once. Returns recipe name."""
print(f"\n{'═'*60}")
print("STEP 1: CREATE RECIPE (once)")
print(f"{'═'*60}")
print(f"Goal: {goal}")
cmd = f'praisonai recipe create "{goal}" --no-optimize'
code, stdout, stderr = run_cli(cmd, cwd=work_dir)
if code != 0:
print(f"❌ Failed to create recipe: {stderr}")
return None
# Extract recipe name from output
for line in stdout.split('\n'):
if 'Created recipe:' in line:
recipe_name = line.split('Created recipe:')[1].strip()
print(f"✅ Created: {recipe_name}")
return recipe_name
print(f"✅ Recipe created")
return None
def run_recipe(recipe_name: str, trace_name: str, work_dir: str) -> tuple[bool, str]:
"""Run the recipe and save trace. Returns (success, output)."""
cmd = f'praisonai recipe run {recipe_name} --save --name {trace_name}'
code, stdout, stderr = run_cli(cmd, cwd=work_dir)
success = code == 0 and 'status: completed' in stdout.lower()
return success, stdout
def judge_recipe(trace_name: str, work_dir: str) -> tuple[float, str]:
"""Judge the trace. Returns (score, feedback)."""
cmd = f'praisonai recipe judge {trace_name}'
code, stdout, stderr = run_cli(cmd, cwd=work_dir)
# Extract score from output
score = 0.0
for line in stdout.split('\n'):
if 'Overall Score:' in line:
try:
score = float(line.split(':')[1].strip().split('/')[0])
except:
pass
return score, stdout
def optimize_recipe(recipe_name: str, iterations: int, work_dir: str) -> tuple[float, str]:
"""Optimize the recipe. Returns (final_score, output)."""
cmd = f'praisonai recipe optimize {recipe_name} --iterations {iterations}'
code, stdout, stderr = run_cli(cmd, cwd=work_dir)
# Extract final score
score = 0.0
for line in stdout.split('\n'):
if 'Final score:' in line or 'score:' in line.lower():
try:
score = float(line.split(':')[1].strip().split('/')[0])
except:
pass
return score, stdout
def run_optimization_loop(goal: str, num_cycles: int = 5, threshold: float = 8.0):
"""Run the full Create → Run → Judge → Optimize loop."""
print("╔════════════════════════════════════════════════════════════╗")
print("║ Recipe Optimization Loop (CLI-based) ║")
print("║ Pattern: Create (once) → Run → Judge → Optimize (loop) ║")
print(f"║ Cycles: {num_cycles} | Threshold: {threshold}/10 ║")
print("╚════════════════════════════════════════════════════════════╝")
# Create work directory
work_dir = Path("/tmp/recipe-optimization-loop")
work_dir.mkdir(parents=True, exist_ok=True)
os.chdir(work_dir)
# Step 1: Create recipe (once)
recipe_name = create_recipe(goal, str(work_dir))
if not recipe_name:
print("Failed to create recipe. Exiting.")
return
# Step 2-N: Run → Judge → Optimize loop
results = []
for cycle in range(1, num_cycles + 1):
print(f"\n{'─'*60}")
print(f"CYCLE {cycle}/{num_cycles}")
print(f"{'─'*60}")
trace_name = f"cycle-{cycle}"
# Run
print(f" 📝 Running recipe...")
success, run_output = run_recipe(recipe_name, trace_name, str(work_dir))
run_status = "✅" if success else "⚠️"
print(f" {run_status} Run complete")
# Judge
print(f" ⚖️ Judging output...")
score, judge_output = judge_recipe(trace_name, str(work_dir))
score_status = "✅" if score >= threshold else "⚠️" if score >= 6.0 else "❌"
print(f" {score_status} Score: {score}/10")
# Check if threshold met
if score >= threshold:
print(f" 🎉 Threshold {threshold}/10 reached!")
results.append({"cycle": cycle, "score": score, "status": "passed"})
break
# Optimize
print(f" 🔧 Optimizing recipe...")
opt_score, opt_output = optimize_recipe(recipe_name, 1, str(work_dir))
print(f" → Optimized score: {opt_score}/10")
results.append({
"cycle": cycle,
"initial_score": score,
"optimized_score": opt_score,
"status": "optimized" if opt_score > score else "no_change"
})
# Summary
print(f"\n{'═'*60}")
print("SUMMARY")
print(f"{'═'*60}")
for r in results:
if "initial_score" in r:
print(f" Cycle {r['cycle']}: {r['initial_score']} → {r['optimized_score']}/10")
else:
print(f" Cycle {r['cycle']}: {r['score']}/10 ✅ PASSED")
final_score = results[-1].get("optimized_score", results[-1].get("score", 0))
success = final_score >= threshold
print(f"\n Final Score: {final_score}/10")
print(f" Success: {'Yes ✅' if success else 'No ❌'}")
print(f" Recipe: {work_dir}/{recipe_name}")
return results
if __name__ == "__main__":
goal = sys.argv[1] if len(sys.argv) > 1 else "research latest AI trends and create a summary report"
run_optimization_loop(goal, num_cycles=5, threshold=8.0)
+3 -1
View File
@@ -89,4 +89,6 @@ graph.py
chroma_db/
.qodo
*_COMPARISON_ANALYSIS.md
*_REPORT.md
*_REPORT.md
.praisonai
@@ -0,0 +1,9 @@
{
"short_term_limit": 100,
"long_term_limit": 1000,
"episodic_retention_days": 30,
"enable_summarization": false,
"summarization_threshold": 50,
"importance_threshold": 0.7,
"auto_promote": true
}
@@ -0,0 +1,21 @@
{
"root_path": "/var/folders/0b/t9c4hwvx6lg6jk58gw8w_dp80000gn/T/tmp6_tn0h1t",
"total_files": 0,
"total_tokens": 11,
"levels": 3,
"built_at": "2026-02-02T17:47:19.241146",
"nodes": {
"/var/folders/0b/t9c4hwvx6lg6jk58gw8w_dp80000gn/T/tmp6_tn0h1t": {
"path": "/var/folders/0b/t9c4hwvx6lg6jk58gw8w_dp80000gn/T/tmp6_tn0h1t",
"level": 3,
"summary": "Project with 0 folders and 0 root files: ",
"children": [],
"token_count": 11,
"created_at": "2026-02-02T17:47:19.241224",
"metadata": {
"folder_count": 0,
"root_file_count": 0
}
}
}
}
@@ -0,0 +1,35 @@
{
"root_path": "/var/folders/0b/t9c4hwvx6lg6jk58gw8w_dp80000gn/T/tmp7wxr57wm",
"total_files": 1,
"total_tokens": 16,
"levels": 2,
"built_at": "2026-02-02T17:47:19.243732",
"nodes": {
"/var/folders/0b/t9c4hwvx6lg6jk58gw8w_dp80000gn/T/tmp7wxr57wm/src/app.py": {
"path": "/var/folders/0b/t9c4hwvx6lg6jk58gw8w_dp80000gn/T/tmp7wxr57wm/src/app.py",
"level": 1,
"summary": "# Application code",
"children": [],
"token_count": 5,
"created_at": "2026-02-02T17:47:19.243833",
"metadata": {
"filename": "app.py",
"relative_path": "src/app.py"
}
},
"/var/folders/0b/t9c4hwvx6lg6jk58gw8w_dp80000gn/T/tmp7wxr57wm/src": {
"path": "/var/folders/0b/t9c4hwvx6lg6jk58gw8w_dp80000gn/T/tmp7wxr57wm/src",
"level": 2,
"summary": "Folder contains 1 files: # Application code",
"children": [
"/var/folders/0b/t9c4hwvx6lg6jk58gw8w_dp80000gn/T/tmp7wxr57wm/src/app.py"
],
"token_count": 11,
"created_at": "2026-02-02T17:47:19.243852",
"metadata": {
"folder_name": "src",
"file_count": 1
}
}
}
}
@@ -0,0 +1,21 @@
{
"root_path": "/var/folders/0b/t9c4hwvx6lg6jk58gw8w_dp80000gn/T/tmp8qwfp4zl",
"total_files": 0,
"total_tokens": 11,
"levels": 3,
"built_at": "2026-02-02T17:49:37.074095",
"nodes": {
"/var/folders/0b/t9c4hwvx6lg6jk58gw8w_dp80000gn/T/tmp8qwfp4zl": {
"path": "/var/folders/0b/t9c4hwvx6lg6jk58gw8w_dp80000gn/T/tmp8qwfp4zl",
"level": 3,
"summary": "Project with 0 folders and 0 root files: ",
"children": [],
"token_count": 11,
"created_at": "2026-02-02T17:49:37.074201",
"metadata": {
"folder_count": 0,
"root_file_count": 0
}
}
}
}
@@ -0,0 +1,33 @@
{
"root_path": "/var/folders/0b/t9c4hwvx6lg6jk58gw8w_dp80000gn/T/tmp9nw2kfoj",
"total_files": 2,
"total_tokens": 14,
"levels": 1,
"built_at": "2026-02-02T17:47:19.242486",
"nodes": {
"/var/folders/0b/t9c4hwvx6lg6jk58gw8w_dp80000gn/T/tmp9nw2kfoj/utils.py": {
"path": "/var/folders/0b/t9c4hwvx6lg6jk58gw8w_dp80000gn/T/tmp9nw2kfoj/utils.py",
"level": 1,
"summary": "def helper(): return True",
"children": [],
"token_count": 7,
"created_at": "2026-02-02T17:47:19.242582",
"metadata": {
"filename": "utils.py",
"relative_path": "utils.py"
}
},
"/var/folders/0b/t9c4hwvx6lg6jk58gw8w_dp80000gn/T/tmp9nw2kfoj/main.py": {
"path": "/var/folders/0b/t9c4hwvx6lg6jk58gw8w_dp80000gn/T/tmp9nw2kfoj/main.py",
"level": 1,
"summary": "def main(): print('Hello')",
"children": [],
"token_count": 7,
"created_at": "2026-02-02T17:47:19.242630",
"metadata": {
"filename": "main.py",
"relative_path": "main.py"
}
}
}
}
@@ -0,0 +1,35 @@
{
"root_path": "/var/folders/0b/t9c4hwvx6lg6jk58gw8w_dp80000gn/T/tmp_dbottar",
"total_files": 1,
"total_tokens": 16,
"levels": 2,
"built_at": "2026-02-02T17:49:37.077857",
"nodes": {
"/var/folders/0b/t9c4hwvx6lg6jk58gw8w_dp80000gn/T/tmp_dbottar/src/app.py": {
"path": "/var/folders/0b/t9c4hwvx6lg6jk58gw8w_dp80000gn/T/tmp_dbottar/src/app.py",
"level": 1,
"summary": "# Application code",
"children": [],
"token_count": 5,
"created_at": "2026-02-02T17:49:37.077985",
"metadata": {
"filename": "app.py",
"relative_path": "src/app.py"
}
},
"/var/folders/0b/t9c4hwvx6lg6jk58gw8w_dp80000gn/T/tmp_dbottar/src": {
"path": "/var/folders/0b/t9c4hwvx6lg6jk58gw8w_dp80000gn/T/tmp_dbottar/src",
"level": 2,
"summary": "Folder contains 1 files: # Application code",
"children": [
"/var/folders/0b/t9c4hwvx6lg6jk58gw8w_dp80000gn/T/tmp_dbottar/src/app.py"
],
"token_count": 11,
"created_at": "2026-02-02T17:49:37.078011",
"metadata": {
"folder_name": "src",
"file_count": 1
}
}
}
}
@@ -0,0 +1,33 @@
{
"root_path": "/var/folders/0b/t9c4hwvx6lg6jk58gw8w_dp80000gn/T/tmpp5k48vv4",
"total_files": 2,
"total_tokens": 14,
"levels": 1,
"built_at": "2026-02-02T17:49:37.075925",
"nodes": {
"/var/folders/0b/t9c4hwvx6lg6jk58gw8w_dp80000gn/T/tmpp5k48vv4/utils.py": {
"path": "/var/folders/0b/t9c4hwvx6lg6jk58gw8w_dp80000gn/T/tmpp5k48vv4/utils.py",
"level": 1,
"summary": "def helper(): return True",
"children": [],
"token_count": 7,
"created_at": "2026-02-02T17:49:37.076036",
"metadata": {
"filename": "utils.py",
"relative_path": "utils.py"
}
},
"/var/folders/0b/t9c4hwvx6lg6jk58gw8w_dp80000gn/T/tmpp5k48vv4/main.py": {
"path": "/var/folders/0b/t9c4hwvx6lg6jk58gw8w_dp80000gn/T/tmpp5k48vv4/main.py",
"level": 1,
"summary": "def main(): print('Hello')",
"children": [],
"token_count": 7,
"created_at": "2026-02-02T17:49:37.076227",
"metadata": {
"filename": "main.py",
"relative_path": "main.py"
}
}
}
}
@@ -0,0 +1,33 @@
{
"root_path": "/var/folders/0b/t9c4hwvx6lg6jk58gw8w_dp80000gn/T/tmpwwb01831",
"total_files": 2,
"total_tokens": 16,
"levels": 1,
"built_at": "2026-02-02T17:49:37.079504",
"nodes": {
"/var/folders/0b/t9c4hwvx6lg6jk58gw8w_dp80000gn/T/tmpwwb01831/javascript.js": {
"path": "/var/folders/0b/t9c4hwvx6lg6jk58gw8w_dp80000gn/T/tmpwwb01831/javascript.js",
"level": 1,
"summary": "JavaScript web development",
"children": [],
"token_count": 7,
"created_at": "2026-02-02T17:49:37.079612",
"metadata": {
"filename": "javascript.js",
"relative_path": "javascript.js"
}
},
"/var/folders/0b/t9c4hwvx6lg6jk58gw8w_dp80000gn/T/tmpwwb01831/python.py": {
"path": "/var/folders/0b/t9c4hwvx6lg6jk58gw8w_dp80000gn/T/tmpwwb01831/python.py",
"level": 1,
"summary": "Python programming language code",
"children": [],
"token_count": 9,
"created_at": "2026-02-02T17:49:37.079658",
"metadata": {
"filename": "python.py",
"relative_path": "python.py"
}
}
}
}
@@ -0,0 +1,33 @@
{
"root_path": "/var/folders/0b/t9c4hwvx6lg6jk58gw8w_dp80000gn/T/tmpwzdvf3_7",
"total_files": 2,
"total_tokens": 16,
"levels": 1,
"built_at": "2026-02-02T17:47:19.245229",
"nodes": {
"/var/folders/0b/t9c4hwvx6lg6jk58gw8w_dp80000gn/T/tmpwzdvf3_7/javascript.js": {
"path": "/var/folders/0b/t9c4hwvx6lg6jk58gw8w_dp80000gn/T/tmpwzdvf3_7/javascript.js",
"level": 1,
"summary": "JavaScript web development",
"children": [],
"token_count": 7,
"created_at": "2026-02-02T17:47:19.245344",
"metadata": {
"filename": "javascript.js",
"relative_path": "javascript.js"
}
},
"/var/folders/0b/t9c4hwvx6lg6jk58gw8w_dp80000gn/T/tmpwzdvf3_7/python.py": {
"path": "/var/folders/0b/t9c4hwvx6lg6jk58gw8w_dp80000gn/T/tmpwzdvf3_7/python.py",
"level": 1,
"summary": "Python programming language code",
"children": [],
"token_count": 9,
"created_at": "2026-02-02T17:47:19.245395",
"metadata": {
"filename": "python.py",
"relative_path": "python.py"
}
}
}
}
@@ -0,0 +1,175 @@
"""
Streaming with Reasoning Content Example
This example demonstrates how to handle streaming events with reasoning content
from models like o1, o1-mini, and deepseek-reasoner.
The is_reasoning flag allows you to distinguish between:
- Reasoning/thinking content (is_reasoning=True)
- Final response content (is_reasoning=False)
"""
from praisonaiagents import Agent
from praisonaiagents.streaming import StreamEvent, StreamEventType, StreamMetrics
def create_styled_callback():
"""Create a callback that styles reasoning content differently."""
def callback(event: StreamEvent):
if event.type == StreamEventType.DELTA_TEXT:
if event.is_reasoning:
# Dim gray for reasoning content
print(f"\033[90m{event.content}\033[0m", end="", flush=True)
else:
# Normal for response content
print(event.content, end="", flush=True)
elif event.type == StreamEventType.STREAM_END:
print() # Newline at end
return callback
def create_collecting_callback():
"""Create a callback that collects reasoning and response separately."""
reasoning_parts = []
response_parts = []
def callback(event: StreamEvent):
if event.type == StreamEventType.DELTA_TEXT:
if event.is_reasoning:
reasoning_parts.append(event.content)
else:
response_parts.append(event.content)
def get_results():
return {
"reasoning": "".join(reasoning_parts),
"response": "".join(response_parts)
}
return callback, get_results
def example_basic_streaming():
"""Basic streaming with reasoning detection."""
print("=" * 60)
print("Example 1: Basic Streaming with Reasoning Detection")
print("=" * 60)
agent = Agent(
name="thinker",
instructions="Think step by step before answering.",
llm="gpt-4o-mini" # Change to o1-mini for actual reasoning
)
# Add styled callback
agent.stream_emitter.add_callback(create_styled_callback())
# Enable metrics
agent.stream_emitter.enable_metrics()
# Run with streaming
response = agent.start("What is 15 * 23?", stream=True)
# Show metrics
metrics = agent.stream_emitter.get_metrics()
if metrics:
print(f"\n📊 {metrics.format_summary()}")
return response
def example_collect_separately():
"""Collect reasoning and response content separately."""
print("\n" + "=" * 60)
print("Example 2: Collect Reasoning and Response Separately")
print("=" * 60)
agent = Agent(
name="analyzer",
instructions="Analyze the problem carefully.",
llm="gpt-4o-mini"
)
# Create collecting callback
callback, get_results = create_collecting_callback()
agent.stream_emitter.add_callback(callback)
# Run with streaming
response = agent.start("Explain why the sky is blue.", stream=True)
# Get separated results
results = get_results()
print(f"\n📝 Reasoning length: {len(results['reasoning'])} chars")
print(f"📝 Response length: {len(results['response'])} chars")
return response
def example_with_metrics():
"""Track streaming performance metrics."""
print("\n" + "=" * 60)
print("Example 3: Streaming with Performance Metrics")
print("=" * 60)
metrics = StreamMetrics()
def metrics_callback(event: StreamEvent):
metrics.update_from_event(event)
if event.type == StreamEventType.FIRST_TOKEN:
print("⚡ First token received!")
elif event.type == StreamEventType.DELTA_TEXT:
print(event.content, end="", flush=True)
elif event.type == StreamEventType.STREAM_END:
print()
agent = Agent(name="assistant", llm="gpt-4o-mini")
agent.stream_emitter.add_callback(metrics_callback)
response = agent.start("Write a haiku about coding.", stream=True)
print("\n📊 Performance Metrics:")
print(f" TTFT: {metrics.ttft * 1000:.0f}ms")
print(f" Stream Duration: {metrics.stream_duration * 1000:.0f}ms")
print(f" Total Time: {metrics.total_time * 1000:.0f}ms")
print(f" Tokens: {metrics.token_count}")
print(f" Speed: {metrics.tokens_per_second:.1f} tokens/sec")
return response
def example_multi_agent_context():
"""Show agent context in multi-agent scenarios."""
print("\n" + "=" * 60)
print("Example 4: Multi-Agent Context")
print("=" * 60)
def context_callback(event: StreamEvent):
if event.type == StreamEventType.DELTA_TEXT:
prefix = f"[{event.agent_id}] " if event.agent_id else ""
print(f"{prefix}{event.content}", end="", flush=True)
elif event.type == StreamEventType.STREAM_END:
print()
# Note: agent_id is populated when using multi-agent workflows
agent = Agent(name="helper", llm="gpt-4o-mini")
agent.stream_emitter.add_callback(context_callback)
response = agent.start("Say hello!", stream=True)
return response
if __name__ == "__main__":
print("🚀 PraisonAI Streaming with Reasoning Examples\n")
# Run examples
example_basic_streaming()
example_collect_separately()
example_with_metrics()
example_multi_agent_context()
print("\n✅ All examples completed!")
@@ -457,6 +457,19 @@ _LAZY_IMPORTS = {
'get_default_plugin_dirs': ('praisonaiagents.plugins', 'get_default_plugin_dirs'),
'get_plugin_template': ('praisonaiagents.plugins', 'get_plugin_template'),
'ensure_plugin_dir': ('praisonaiagents.plugins', 'ensure_plugin_dir'),
# Config loader - for config-driven defaults
'get_config': ('praisonaiagents.config.loader', 'get_config'),
'get_default': ('praisonaiagents.config.loader', 'get_default'),
'get_plugins_config': ('praisonaiagents.config.loader', 'get_plugins_config'),
'get_defaults_config': ('praisonaiagents.config.loader', 'get_defaults_config'),
'apply_config_defaults': ('praisonaiagents.config.loader', 'apply_config_defaults'),
'validate_config': ('praisonaiagents.config.loader', 'validate_config'),
'get_config_path': ('praisonaiagents.config.loader', 'get_config_path'),
'ConfigValidationError': ('praisonaiagents.config.loader', 'ConfigValidationError'),
'PraisonConfig': ('praisonaiagents.config.loader', 'PraisonConfig'),
'PluginsConfig': ('praisonaiagents.config.loader', 'PluginsConfig'),
'DefaultsConfig': ('praisonaiagents.config.loader', 'DefaultsConfig'),
}
@@ -9,6 +9,10 @@ Environment Variables:
PRAISONAI_TELEMETRY_ENABLED: Enable telemetry (default: false, opt-in)
PRAISONAI_TELEMETRY_DISABLED: Disable telemetry (takes precedence)
DO_NOT_TRACK: Standard opt-out flag (takes precedence)
PRAISONAI_PLUGINS: Enable plugins (default: false)
- "true" or "1": Enable all discovered plugins
- "logging,metrics": Enable specific plugins (comma-separated)
- "false" or "0": Disable plugins
"""
import os
from typing import Optional
@@ -77,6 +81,38 @@ AUTO_INSTRUMENT = _str_to_bool(
)
# Plugins: Enable background plugins (hooks, metrics, logging)
# Note: Tools and guardrails work WITHOUT this flag - they are explicit
def _get_plugins_enabled() -> bool:
"""Determine if plugins should be auto-enabled based on env var."""
env_value = os.environ.get('PRAISONAI_PLUGINS', '').lower()
if not env_value:
return False
if env_value in ('true', '1', 'yes', 'on'):
return True
if env_value in ('false', '0', 'no', 'off'):
return False
# Treat as comma-separated list of plugin names (implies enabled)
return True
def _get_plugins_list() -> list:
"""Get list of specific plugins to enable from env var.
Returns:
List of plugin names, or empty list if all plugins enabled.
"""
env_value = os.environ.get('PRAISONAI_PLUGINS', '').lower()
if not env_value or env_value in ('true', '1', 'yes', 'on', 'false', '0', 'no', 'off'):
return []
# Parse comma-separated list
return [p.strip() for p in env_value.split(',') if p.strip()]
PLUGINS_ENABLED = _get_plugins_enabled()
PLUGINS_LIST = _get_plugins_list()
def missing_dependency_error(
package_name: str,
extra_name: Optional[str] = None,
@@ -150,6 +150,7 @@ if TYPE_CHECKING:
from ..task.task import Task
from .handoff import Handoff, HandoffConfig, HandoffResult
from ..rag.models import RAGResult, ContextPack
from ..eval.results import EvaluationLoopResult
class Agent:
# Class-level counter for generating unique display names for nameless agents
@@ -540,6 +541,49 @@ class Agent:
Agent._configure_logging()
Agent._logging_configured = True
# ============================================================
# CONFIG-DRIVEN DEFAULTS (apply before parameter resolution)
# Precedence: Explicit params > Config file > Built-in defaults
# Only applies when param is None (not explicitly set)
# ============================================================
from ..config.loader import apply_config_defaults, get_default
# Apply config defaults for LLM if not explicitly set
if llm is None and model is None:
config_model = get_default("model")
if config_model:
llm = config_model
# Apply config defaults for base_url if not explicitly set
if base_url is None:
config_base_url = get_default("base_url")
if config_base_url:
base_url = config_base_url
# Apply config defaults for feature params (memory, knowledge, etc.)
# These use apply_config_defaults which handles enabled=True/False logic
if memory is None:
memory = apply_config_defaults("memory", memory, MemoryConfig)
if knowledge is None:
knowledge = apply_config_defaults("knowledge", knowledge, KnowledgeConfig)
if planning is False: # Only override if explicitly False (default)
config_planning = apply_config_defaults("planning", None, PlanningConfig)
if config_planning:
planning = config_planning
if reflection is None:
reflection = apply_config_defaults("reflection", reflection, ReflectionConfig)
if web is None:
web = apply_config_defaults("web", web, WebConfig)
if output is None:
output = apply_config_defaults("output", output, OutputConfig)
if execution is None:
execution = apply_config_defaults("execution", execution, ExecutionConfig)
if caching is None:
caching = apply_config_defaults("caching", caching, CachingConfig)
if autonomy is None:
# AutonomyConfig is in agent/autonomy.py - use dict for config defaults
autonomy = apply_config_defaults("autonomy", autonomy, None)
# ============================================================
# CONSOLIDATED PARAMS EXTRACTION (agent-centric API)
# Uses unified resolver: Instance > Config > Array > String > Bool > Default
@@ -839,7 +883,10 @@ class Agent:
backend = _memory_config.backend
if hasattr(backend, 'value'):
backend = backend.value
if backend == "file":
# If learn is enabled, pass as dict to trigger full Memory class
if _memory_config.learn:
memory = _memory_config.to_dict()
elif backend == "file":
memory = True
elif _memory_config.config:
memory = _memory_config.config
@@ -851,6 +898,24 @@ class Agent:
elif memory is False:
memory = None
# ─────────────────────────────────────────────────────────────────────
# Resolve HISTORY from MemoryConfig - FAST PATH
# History is enabled via memory="history" preset or MemoryConfig(history=True)
# ─────────────────────────────────────────────────────────────────────
_history_enabled = False
_history_limit = 10
_history_session_id = None
# Check if memory config has history settings
if _memory_config is not None and isinstance(_memory_config, MemoryConfig):
if _memory_config.history:
_history_enabled = True
_history_limit = _memory_config.history_limit
# Use auto_save session if no explicit session and history is enabled
if _history_enabled and _history_session_id is None and auto_save:
_history_session_id = auto_save
# ─────────────────────────────────────────────────────────────────────
# Resolve KNOWLEDGE param - FAST PATH
# ─────────────────────────────────────────────────────────────────────
@@ -1353,6 +1418,11 @@ Your Goal: {self.goal}
# Used when session_id is provided but no DB adapter
self._session_store = None
self._session_store_initialized = False
# History injection settings (for auto-injecting session history into context)
self._history_enabled = _history_enabled
self._history_limit = _history_limit
self._history_session_id = _history_session_id
# Agent-centric feature instances (lazy loaded for zero performance impact)
self._auto_memory = auto_memory
@@ -2298,6 +2368,99 @@ Summary:"""
)
return handoff_obj.execute_programmatic(self, prompt, context)
def run_until(
self,
prompt: str,
criteria: str,
threshold: float = 8.0,
max_iterations: int = 5,
mode: str = "optimize",
on_iteration: Optional[Callable[[Any], None]] = None,
verbose: bool = False,
) -> "EvaluationLoopResult":
"""
Run agent iteratively until output meets quality criteria.
This method implements the "Ralph Loop" pattern: run agent → judge output
→ improve based on feedback → repeat until threshold met.
Args:
prompt: The prompt to send to the agent
criteria: Evaluation criteria for the Judge (e.g., "Response is thorough")
threshold: Score threshold for success (default: 8.0, scale 1-10)
max_iterations: Maximum iterations before stopping (default: 5)
mode: "optimize" (stop on success) or "review" (run all iterations)
on_iteration: Optional callback called after each iteration
verbose: Enable verbose logging
Returns:
EvaluationLoopResult with iteration history and final score
Example:
```python
agent = Agent(name="analyzer", instructions="Analyze systems")
result = agent.run_until(
"Analyze the auth flow",
criteria="Analysis is thorough and actionable",
threshold=8.0,
)
print(result.final_score) # 8.5
print(result.success) # True
```
"""
from ..eval.loop import EvaluationLoop
loop = EvaluationLoop(
agent=self,
criteria=criteria,
threshold=threshold,
max_iterations=max_iterations,
mode=mode,
on_iteration=on_iteration,
verbose=verbose,
)
return loop.run(prompt)
async def run_until_async(
self,
prompt: str,
criteria: str,
threshold: float = 8.0,
max_iterations: int = 5,
mode: str = "optimize",
on_iteration: Optional[Callable[[Any], None]] = None,
verbose: bool = False,
) -> "EvaluationLoopResult":
"""
Async version of run_until().
Run agent iteratively until output meets quality criteria.
Args:
prompt: The prompt to send to the agent
criteria: Evaluation criteria for the Judge
threshold: Score threshold for success (default: 8.0)
max_iterations: Maximum iterations before stopping (default: 5)
mode: "optimize" (stop on success) or "review" (run all iterations)
on_iteration: Optional callback called after each iteration
verbose: Enable verbose logging
Returns:
EvaluationLoopResult with iteration history and final score
"""
from ..eval.loop import EvaluationLoop
loop = EvaluationLoop(
agent=self,
criteria=criteria,
threshold=threshold,
max_iterations=max_iterations,
mode=mode,
on_iteration=on_iteration,
verbose=verbose,
)
return await loop.run_async(prompt)
async def handoff_to_async(
self,
target_agent: 'Agent',
@@ -2504,8 +2667,8 @@ Summary:"""
Initialize RulesManager for persistent rules/instructions.
Automatically discovers rules from:
- ~/.praison/rules/ (global)
- .praison/rules/ (workspace)
- ~/.praisonai/rules/ (global)
- .praisonai/rules/ (workspace)
- Subdirectory rules
NOTE: This is called lazily via the rules_manager property for performance.
@@ -2595,8 +2758,19 @@ Summary:"""
self._memory_instance = FileMemory(user_id=mem_user_id)
elif isinstance(memory, dict):
# Configuration dict
provider = memory.get("provider", "file")
if provider == "file":
provider = memory.get("provider", memory.get("backend", "file"))
learn_enabled = memory.get("learn", False)
# Use full Memory class if learn is enabled (requires LearnManager)
if learn_enabled:
try:
from ..memory.memory import Memory
self._memory_instance = Memory(memory)
except ImportError:
logging.warning("Memory with learn requires additional dependencies. Falling back to FileMemory (learn disabled).")
from ..memory.file_memory import FileMemory
self._memory_instance = FileMemory(user_id=memory.get("user_id", mem_user_id))
elif provider == "file":
from ..memory.file_memory import FileMemory
self._memory_instance = FileMemory(
user_id=memory.get("user_id", mem_user_id),
@@ -2632,6 +2806,24 @@ Summary:"""
return ""
def get_learn_context(self) -> str:
"""
Get learning context for injection into system prompt.
Returns learned preferences, insights, and patterns when memory="learn"
is enabled. Returns empty string when learn is not enabled (zero overhead).
Returns:
Formatted learning context string, or empty string if learn not enabled
"""
if not self._memory_instance:
return ""
if hasattr(self._memory_instance, 'get_learn_context'):
return self._memory_instance.get_learn_context()
return ""
def store_memory(self, content: str, memory_type: str = "short_term", **kwargs):
"""
Store content in memory.
@@ -3272,6 +3464,11 @@ Your Goal: {self.goal}"""
# Display memory info to user if verbose
if self.verbose:
self._display_memory_info()
# Add learn context if learn is enabled (auto-inject when memory="learn")
learn_context = self.get_learn_context()
if learn_context:
system_prompt += f"\n\n## Learned Context (Patterns and insights from past interactions)\n{learn_context}"
# Add skills prompt if skills are configured
if self._skills or self._skills_dirs:
@@ -3442,7 +3639,19 @@ Your Goal: {self.goal}"""
if system_prompt:
messages.append({"role": "system", "content": system_prompt})
# Add chat history
# Inject session history if enabled (from persistent storage)
if self._history_enabled and self._session_store is not None:
try:
session_history = self._session_store.get_chat_history(
self._history_session_id,
max_messages=self._history_limit
)
if session_history:
messages.extend(session_history)
except Exception as e:
logging.debug(f"Failed to load session history: {e}")
# Add in-memory chat history (current conversation)
if self.chat_history:
messages.extend(self.chat_history)
@@ -162,7 +162,7 @@ class ContextAgent(Agent):
# Enhanced logging and output management
self.debug_mode = os.getenv('LOGLEVEL', '').lower() == 'debug'
self.output_dir = Path(".praison/prp") # Save in .praison/prp folder
self.output_dir = Path(".praisonai/prp") # Save in .praisonai/prp folder
self.setup_output_directories() # Create directories first
self.setup_logging() # Then setup logging
@@ -1136,7 +1136,7 @@ detailed, actionable implementation steps."""
filtered = []
exclude_patterns = [
'__pycache__', '.pytest_cache', '.git', 'node_modules',
'.venv', 'venv', 'dist', 'build', '.praison', '.chroma_db',
'.venv', 'venv', 'dist', 'build', '.praisonai', '.chroma_db',
'.ruff_cache', '.mypy_cache', 'context_engineering_output'
]
@@ -2263,7 +2263,7 @@ This PRP provides:
)
print(f"🐛 Debug mode: Comprehensive session report + individual files + final PRP saved")
else:
# Non-debug mode: Save ONLY the final comprehensive PRP directly in .praison/prp
# Non-debug mode: Save ONLY the final comprehensive PRP directly in .praisonai/prp
# Use clean filename: goal name + timestamp
final_output_file = self.output_dir / f"{safe_goal}_prp_{timestamp}.md"
final_output_file.parent.mkdir(parents=True, exist_ok=True)
@@ -513,9 +513,9 @@ class AgentTeam:
"use_embedding": True,
"storage": {
"type": "sqlite",
"path": "./.praison/memory.db"
"path": "./.praisonai/memory.db"
},
"rag_db_path": "./.praison/chroma_db"
"rag_db_path": "./.praisonai/chroma_db"
}
if _embedder:
if isinstance(_embedder, dict):
@@ -2345,7 +2345,7 @@ Context:
context=context if context else None,
# Inherit from original task if available
memory=original_task.memory if original_task else None,
callback=original_task.callback if original_task else None,
on_task_complete=original_task.callback if original_task else None,
guardrails=original_task.guardrail if original_task else None,
max_retries=original_task.max_retries if original_task else 3,
output_json=original_task.output_json if original_task else None,
@@ -20,7 +20,7 @@ Usage:
# Create checkpoint service
service = CheckpointService(
workspace_dir="/path/to/project",
storage_dir="~/.praison/checkpoints"
storage_dir="~/.praisonai/checkpoints"
)
# Initialize shadow git
@@ -51,7 +51,7 @@ class CheckpointService:
Example:
service = CheckpointService(
workspace_dir="/path/to/project",
storage_dir="~/.praison/checkpoints"
storage_dir="~/.praisonai/checkpoints"
)
await service.initialize()
@@ -39,7 +39,7 @@ class CheckpointConfig:
max_checkpoints: int = 100 # Maximum checkpoints to keep
exclude_patterns: List[str] = field(default_factory=lambda: [
".git",
".praison",
".praisonai",
"__pycache__",
"*.pyc",
".env",
@@ -165,6 +165,10 @@ class MemoryConfig:
# Continuous learning configuration
learn: Optional[Union[bool, LearnConfig]] = None
# History injection (auto-inject session history into context)
history: bool = False
history_limit: int = 10
def to_dict(self) -> Dict[str, Any]:
"""Convert to dictionary."""
learn_dict = None
@@ -172,6 +176,9 @@ class MemoryConfig:
learn_dict = LearnConfig().to_dict()
elif isinstance(self.learn, LearnConfig):
learn_dict = self.learn.to_dict()
elif isinstance(self.learn, dict):
# Pass through dict directly (user provided learn config as dict)
learn_dict = self.learn
return {
"backend": self.backend.value if isinstance(self.backend, MemoryBackend) else self.backend,
@@ -181,6 +188,8 @@ class MemoryConfig:
"claude_memory": self.claude_memory,
"config": self.config,
"learn": learn_dict,
"history": self.history,
"history_limit": self.history_limit,
}
@@ -743,7 +752,7 @@ class SkillsConfig:
# With config
Agent(skills=SkillsConfig(
paths=["./my-skill"],
dirs=["~/.praison/skills/"],
dirs=["~/.praisonai/skills/"],
auto_discover=True,
))
"""
@@ -0,0 +1,609 @@
"""
Configuration Loader for PraisonAI Agents.
Loads configuration from multiple sources with precedence:
1. Explicit parameters (highest)
2. Environment variables
3. Config file (.praisonai/config.toml or praisonai.toml)
4. Defaults (lowest)
Zero Performance Impact:
- Config is loaded lazily on first access
- No file I/O at import time
- Cached after first load
Usage:
from praisonaiagents.config.loader import get_config, get_default
# Get entire config
config = get_config()
# Get specific default with fallback
model = get_default("model", "gpt-4o-mini")
memory_config = get_default("memory", {})
# Validate config file
errors = validate_config(config.to_dict())
"""
import os
from pathlib import Path
from typing import Any, Dict, List, Optional, Union
from dataclasses import dataclass, field
# ============================================================================
# CONFIG VALIDATION SCHEMA
# ============================================================================
# Defines valid keys and types for config file validation
VALID_PLUGINS_KEYS = {
"enabled": (bool, list, str), # bool, list of plugin names, or "true"/"false"
"auto_discover": (bool,),
"directories": (list,),
}
VALID_DEFAULTS_KEYS = {
# LLM settings
"model": (str,),
"base_url": (str,),
"api_key": (str,),
# Feature flags
"allow_delegation": (bool,),
"allow_code_execution": (bool,),
"code_execution_mode": (str,),
# Feature configs (nested dicts with 'enabled' key)
"memory": (dict, bool),
"knowledge": (dict, bool),
"planning": (dict, bool),
"reflection": (dict, bool),
"guardrails": (dict, bool),
"web": (dict, bool),
"output": (dict, str), # dict or preset string
"execution": (dict, str), # dict or preset string
"caching": (dict, bool),
"autonomy": (dict, bool),
"skills": (dict, list),
"context": (dict, bool),
"hooks": (dict, list),
"templates": (dict,),
}
VALID_ROOT_KEYS = {"plugins", "defaults"}
class ConfigValidationError(Exception):
"""Raised when config file has validation errors."""
def __init__(self, errors: List[str]):
self.errors = errors
message = "Config validation failed:\n" + "\n".join(f" - {e}" for e in errors)
super().__init__(message)
# Global config cache (lazy loaded)
_config_cache: Optional[Dict[str, Any]] = None
_config_loaded: bool = False
@dataclass
class PluginsConfig:
"""Configuration for the plugin system."""
enabled: Union[bool, List[str]] = False # True, False, or list of plugin names
auto_discover: bool = True
directories: List[str] = field(default_factory=lambda: [
"./.praisonai/plugins/",
"~/.praisonai/plugins/"
])
def to_dict(self) -> Dict[str, Any]:
return {
"enabled": self.enabled,
"auto_discover": self.auto_discover,
"directories": self.directories,
}
@dataclass
class DefaultsConfig:
"""Default configuration for Agent parameters."""
# LLM defaults
model: Optional[str] = None
base_url: Optional[str] = None
# Feature flags
allow_delegation: bool = False
allow_code_execution: bool = False
code_execution_mode: str = "safe"
# Feature configs (nested dicts)
memory: Optional[Dict[str, Any]] = None
knowledge: Optional[Dict[str, Any]] = None
planning: Optional[Dict[str, Any]] = None
reflection: Optional[Dict[str, Any]] = None
web: Optional[Dict[str, Any]] = None
output: Optional[Dict[str, Any]] = None
execution: Optional[Dict[str, Any]] = None
caching: Optional[Dict[str, Any]] = None
autonomy: Optional[Dict[str, Any]] = None
skills: Optional[Dict[str, Any]] = None
def to_dict(self) -> Dict[str, Any]:
return {
"model": self.model,
"base_url": self.base_url,
"allow_delegation": self.allow_delegation,
"allow_code_execution": self.allow_code_execution,
"code_execution_mode": self.code_execution_mode,
"memory": self.memory,
"knowledge": self.knowledge,
"planning": self.planning,
"reflection": self.reflection,
"web": self.web,
"output": self.output,
"execution": self.execution,
"caching": self.caching,
"autonomy": self.autonomy,
"skills": self.skills,
}
@dataclass
class PraisonConfig:
"""Root configuration for PraisonAI Agents."""
plugins: PluginsConfig = field(default_factory=PluginsConfig)
defaults: DefaultsConfig = field(default_factory=DefaultsConfig)
def to_dict(self) -> Dict[str, Any]:
return {
"plugins": self.plugins.to_dict(),
"defaults": self.defaults.to_dict(),
}
def _find_config_file() -> Optional[Path]:
"""Find config file in standard locations.
Search order:
1. .praisonai/config.toml (project-local)
2. praisonai.toml (project root)
3. ~/.praisonai/config.toml (user global)
Returns:
Path to config file if found, None otherwise
"""
# Project-local locations
cwd = Path.cwd()
local_paths = [
cwd / ".praisonai" / "config.toml",
cwd / "praisonai.toml",
]
for path in local_paths:
if path.exists():
return path
# User global location
home = Path.home()
global_path = home / ".praisonai" / "config.toml"
if global_path.exists():
return global_path
return None
def _parse_toml(path: Path) -> Dict[str, Any]:
"""Parse TOML file with fallback for Python < 3.11.
Args:
path: Path to TOML file
Returns:
Parsed config dict
"""
try:
# Python 3.11+ has built-in tomllib
import tomllib
with open(path, "rb") as f:
return tomllib.load(f)
except ImportError:
# Fallback for Python < 3.11
try:
import tomli
with open(path, "rb") as f:
return tomli.load(f)
except ImportError:
# No TOML parser available - return empty config
import logging
logging.debug(
"TOML parser not available. Install tomli for Python < 3.11: "
"pip install tomli"
)
return {}
def _load_config() -> Dict[str, Any]:
"""Load configuration from file.
Returns:
Config dict (empty if no config file found)
"""
config_path = _find_config_file()
if config_path is None:
return {}
try:
return _parse_toml(config_path)
except Exception as e:
import logging
logging.warning(f"Failed to load config from {config_path}: {e}")
return {}
def _dict_to_plugins_config(data: Dict[str, Any]) -> PluginsConfig:
"""Convert dict to PluginsConfig."""
return PluginsConfig(
enabled=data.get("enabled", False),
auto_discover=data.get("auto_discover", True),
directories=data.get("directories", [
"./.praisonai/plugins/",
"~/.praisonai/plugins/"
]),
)
def _dict_to_defaults_config(data: Dict[str, Any]) -> DefaultsConfig:
"""Convert dict to DefaultsConfig."""
return DefaultsConfig(
model=data.get("model"),
base_url=data.get("base_url"),
allow_delegation=data.get("allow_delegation", False),
allow_code_execution=data.get("allow_code_execution", False),
code_execution_mode=data.get("code_execution_mode", "safe"),
memory=data.get("memory"),
knowledge=data.get("knowledge"),
planning=data.get("planning"),
reflection=data.get("reflection"),
web=data.get("web"),
output=data.get("output"),
execution=data.get("execution"),
caching=data.get("caching"),
autonomy=data.get("autonomy"),
skills=data.get("skills"),
)
def _dict_to_praison_config(data: Dict[str, Any]) -> PraisonConfig:
"""Convert dict to PraisonConfig."""
plugins_data = data.get("plugins", {})
defaults_data = data.get("defaults", {})
return PraisonConfig(
plugins=_dict_to_plugins_config(plugins_data),
defaults=_dict_to_defaults_config(defaults_data),
)
def get_config(reload: bool = False) -> PraisonConfig:
"""Get the global configuration.
Loads config lazily on first access and caches it.
Args:
reload: Force reload from file
Returns:
PraisonConfig instance
"""
global _config_cache, _config_loaded
if not _config_loaded or reload:
raw_config = _load_config()
_config_cache = _dict_to_praison_config(raw_config)
_config_loaded = True
return _config_cache
def get_plugins_config() -> PluginsConfig:
"""Get plugins configuration.
Returns:
PluginsConfig instance
"""
return get_config().plugins
def get_defaults_config() -> DefaultsConfig:
"""Get defaults configuration.
Returns:
DefaultsConfig instance
"""
return get_config().defaults
def get_default(key: str, fallback: Any = None) -> Any:
"""Get a specific default value.
Args:
key: Config key (e.g., "model", "memory", "memory.backend")
fallback: Value to return if key not found
Returns:
Config value or fallback
"""
defaults = get_defaults_config()
# Handle nested keys like "memory.backend"
if "." in key:
parts = key.split(".")
value = getattr(defaults, parts[0], None)
if value is None:
return fallback
for part in parts[1:]:
if isinstance(value, dict):
value = value.get(part)
else:
return fallback
if value is None:
return fallback
return value
# Simple key
value = getattr(defaults, key, None)
return value if value is not None else fallback
def is_plugins_enabled() -> bool:
"""Check if plugins are enabled via config or env var.
Returns:
True if plugins should be enabled
"""
# Check env var first (highest precedence)
env_value = os.environ.get("PRAISONAI_PLUGINS", "").lower()
if env_value:
if env_value in ("true", "1", "yes", "on"):
return True
if env_value in ("false", "0", "no", "off"):
return False
# Treat as comma-separated list of plugin names
return True
# Check config file
plugins_config = get_plugins_config()
if isinstance(plugins_config.enabled, bool):
return plugins_config.enabled
if isinstance(plugins_config.enabled, list):
return len(plugins_config.enabled) > 0
return False
def get_enabled_plugins() -> Optional[List[str]]:
"""Get list of enabled plugins (if specific list provided).
Returns:
List of plugin names, or None if all plugins enabled
"""
# Check env var first
env_value = os.environ.get("PRAISONAI_PLUGINS", "").lower()
if env_value and env_value not in ("true", "1", "yes", "on", "false", "0", "no", "off"):
# Treat as comma-separated list
return [p.strip() for p in env_value.split(",") if p.strip()]
# Check config file
plugins_config = get_plugins_config()
if isinstance(plugins_config.enabled, list):
return plugins_config.enabled
return None # All plugins enabled
def clear_config_cache() -> None:
"""Clear the config cache (for testing)."""
global _config_cache, _config_loaded
_config_cache = None
_config_loaded = False
def validate_config(config: Dict[str, Any], raise_on_error: bool = False) -> List[str]:
"""Validate config file structure and types.
Args:
config: Raw config dict (from TOML parsing)
raise_on_error: If True, raise ConfigValidationError on errors
Returns:
List of validation error messages (empty if valid)
Example:
config = {"plugins": {"enabled": True}, "defaults": {"model": "gpt-4o"}}
errors = validate_config(config)
if errors:
print("Config errors:", errors)
"""
errors = []
# Check root keys
for key in config.keys():
if key not in VALID_ROOT_KEYS:
similar = _suggest_similar_key(key, VALID_ROOT_KEYS)
if similar:
errors.append(f"Unknown root key '{key}'. Did you mean '{similar}'?")
else:
errors.append(f"Unknown root key '{key}'. Valid keys: {', '.join(sorted(VALID_ROOT_KEYS))}")
# Validate [plugins] section
if "plugins" in config:
plugins = config["plugins"]
if not isinstance(plugins, dict):
errors.append(f"[plugins] must be a table, got {type(plugins).__name__}")
else:
for key, value in plugins.items():
if key not in VALID_PLUGINS_KEYS:
similar = _suggest_similar_key(key, VALID_PLUGINS_KEYS.keys())
if similar:
errors.append(f"[plugins] Unknown key '{key}'. Did you mean '{similar}'?")
else:
errors.append(f"[plugins] Unknown key '{key}'. Valid keys: {', '.join(sorted(VALID_PLUGINS_KEYS.keys()))}")
else:
valid_types = VALID_PLUGINS_KEYS[key]
if not isinstance(value, valid_types):
errors.append(f"[plugins.{key}] Expected {_format_types(valid_types)}, got {type(value).__name__}")
# Validate [defaults] section
if "defaults" in config:
defaults = config["defaults"]
if not isinstance(defaults, dict):
errors.append(f"[defaults] must be a table, got {type(defaults).__name__}")
else:
for key, value in defaults.items():
if key not in VALID_DEFAULTS_KEYS:
similar = _suggest_similar_key(key, VALID_DEFAULTS_KEYS.keys())
if similar:
errors.append(f"[defaults] Unknown key '{key}'. Did you mean '{similar}'?")
else:
errors.append(f"[defaults] Unknown key '{key}'. Valid keys: {', '.join(sorted(VALID_DEFAULTS_KEYS.keys()))}")
else:
valid_types = VALID_DEFAULTS_KEYS[key]
if not isinstance(value, valid_types):
errors.append(f"[defaults.{key}] Expected {_format_types(valid_types)}, got {type(value).__name__}")
# Validate nested feature configs
if isinstance(value, dict) and key in ("memory", "knowledge", "planning", "reflection",
"guardrails", "web", "output", "execution",
"caching", "autonomy", "context"):
nested_errors = _validate_feature_config(key, value)
errors.extend(nested_errors)
if raise_on_error and errors:
raise ConfigValidationError(errors)
return errors
def _suggest_similar_key(key: str, valid_keys) -> Optional[str]:
"""Suggest a similar valid key using simple string matching."""
key_lower = key.lower()
for valid in valid_keys:
# Exact match after lowercasing
if valid.lower() == key_lower:
return valid
# Prefix match
if valid.lower().startswith(key_lower) or key_lower.startswith(valid.lower()):
return valid
# Substring match
if key_lower in valid.lower() or valid.lower() in key_lower:
return valid
return None
def _format_types(types: tuple) -> str:
"""Format type tuple for error messages."""
type_names = []
for t in types:
if t is bool:
type_names.append("bool")
elif t is str:
type_names.append("str")
elif t is dict:
type_names.append("table")
elif t is list:
type_names.append("array")
else:
type_names.append(t.__name__)
return " or ".join(type_names)
def _validate_feature_config(feature: str, config: Dict[str, Any]) -> List[str]:
"""Validate a nested feature config (memory, knowledge, etc.)."""
errors = []
# Common valid keys for feature configs
common_keys = {"enabled", "backend", "provider", "preset", "verbose"}
# Feature-specific valid keys
feature_keys = {
"memory": {"enabled", "backend", "provider", "use_long_term", "use_short_term",
"user_id", "session_id", "db_path", "collection_name"},
"knowledge": {"enabled", "sources", "provider", "chunk_size", "chunk_overlap"},
"planning": {"enabled", "max_steps", "reasoning"},
"reflection": {"enabled", "max_reflect", "min_reflect", "llm"},
"guardrails": {"enabled", "max_retries", "validator"},
"web": {"enabled", "search", "fetch", "provider"},
"output": {"enabled", "verbose", "markdown", "stream", "metrics", "preset"},
"execution": {"enabled", "max_iter", "max_rpm", "max_execution_time", "preset"},
"caching": {"enabled", "prompt_caching", "ttl"},
"autonomy": {"enabled", "max_steps", "approval_required"},
"context": {"enabled", "max_tokens", "strategy"},
}
valid_keys = feature_keys.get(feature, common_keys)
for key in config.keys():
if key not in valid_keys:
similar = _suggest_similar_key(key, valid_keys)
if similar:
errors.append(f"[defaults.{feature}.{key}] Unknown key. Did you mean '{similar}'?")
# Don't error on unknown nested keys - allow flexibility
return errors
def get_config_path() -> Optional[Path]:
"""Get the path to the config file if it exists.
Returns:
Path to config file, or None if not found
"""
return _find_config_file()
# Convenience function to merge config defaults with explicit params
def apply_config_defaults(
param_name: str,
explicit_value: Any,
config_class: Optional[type] = None,
) -> Any:
"""Apply config defaults to a parameter if not explicitly set.
Args:
param_name: Name of the parameter (e.g., "memory", "knowledge")
explicit_value: Value explicitly passed by user (None if not set)
config_class: Optional config class to instantiate
Returns:
Resolved value with config defaults applied
Example:
# User passes memory=True, config has memory.backend="postgres"
# Result: MemoryConfig with backend="postgres"
memory = apply_config_defaults("memory", True, MemoryConfig)
"""
# If user explicitly passed a value, respect it
if explicit_value is not None:
# If True and we have config defaults, merge them
if explicit_value is True:
config_defaults = get_default(param_name)
if config_defaults and isinstance(config_defaults, dict):
if config_class:
# Check if enabled key exists and is True
if config_defaults.get("enabled", True):
return config_class(**{k: v for k, v in config_defaults.items() if k != "enabled"})
return config_defaults
return explicit_value
# Check if config has defaults for this param
config_defaults = get_default(param_name)
if config_defaults is None:
return None
# If config has enabled=True, return the config
if isinstance(config_defaults, dict):
if config_defaults.get("enabled", False):
if config_class:
return config_class(**{k: v for k, v in config_defaults.items() if k != "enabled"})
return True
return None
@@ -26,6 +26,10 @@ MEMORY_PRESETS: Dict[str, Dict[str, Any]] = {
"mongodb": {"backend": "mongodb"},
# Feature presets
"learn": {"backend": "file", "learn": True}, # Enable Agent Learn with file backend
# History presets (auto-inject session history into context)
"history": {"backend": "file", "history": True, "history_limit": 10},
"session": {"backend": "file", "history": True, "history_limit": 10}, # Alias
"chat": {"backend": "file", "history": True, "history_limit": 20}, # Conversational
}
MEMORY_URL_SCHEMES: Dict[str, str] = {
@@ -0,0 +1,156 @@
# PraisonAI Agents Configuration File
# Copy this file to .praisonai/config.toml in your project directory
# or to ~/.praisonai/config.toml for global settings
# ============================================================================
# PLUGINS CONFIGURATION
# ============================================================================
# Enable background plugins (hooks, metrics, logging)
# Note: Tools and guardrails work WITHOUT enabling plugins
[plugins]
# Enable all plugins: true
# Enable specific plugins: ["logging", "metrics"]
# Disable: false
enabled = false
# Auto-discover plugins from default directories
auto_discover = true
# Plugin directories to scan
directories = [
"./.praisonai/plugins/",
"~/.praisonai/plugins/"
]
# ============================================================================
# DEFAULT SETTINGS
# ============================================================================
# These defaults are used when you pass True to a feature parameter
# Example: Agent(memory=True) will use [defaults.memory] settings
[defaults]
# Default LLM model
model = "gpt-4o-mini"
# Custom LLM endpoint (optional)
# base_url = "http://localhost:11434/v1"
# Feature flags
allow_delegation = false
allow_code_execution = false
code_execution_mode = "safe"
# ============================================================================
# MEMORY DEFAULTS
# ============================================================================
# Used when: Agent(memory=True)
[defaults.memory]
enabled = false
backend = "file" # file, sqlite, redis, postgres, mongodb
# user_id = "" # Auto-generated if empty
auto_memory = false
history = false
history_limit = 10
# Learning configuration
[defaults.memory.learn]
enabled = false
persona = true
insights = true
patterns = false
# ============================================================================
# KNOWLEDGE DEFAULTS
# ============================================================================
# Used when: Agent(knowledge=True)
[defaults.knowledge]
enabled = false
embedder = "openai"
chunking_strategy = "semantic"
chunk_size = 1000
chunk_overlap = 200
retrieval_k = 5
retrieval_threshold = 0.0
rerank = false
# ============================================================================
# PLANNING DEFAULTS
# ============================================================================
# Used when: Agent(planning=True)
[defaults.planning]
enabled = false
reasoning = false
auto_approve = false
read_only = false
# ============================================================================
# REFLECTION DEFAULTS
# ============================================================================
# Used when: Agent(reflection=True)
[defaults.reflection]
enabled = false
min_iterations = 1
max_iterations = 3
# ============================================================================
# WEB DEFAULTS
# ============================================================================
# Used when: Agent(web=True)
[defaults.web]
enabled = false
search = true
fetch = true
search_provider = "duckduckgo"
max_results = 5
# ============================================================================
# OUTPUT DEFAULTS
# ============================================================================
# Used when: Agent(output=True) or default output mode
[defaults.output]
preset = "silent" # silent, actions, verbose, json, stream, status
verbose = false
markdown = false
stream = false
metrics = false
# ============================================================================
# EXECUTION DEFAULTS
# ============================================================================
# Used when: Agent(execution=True) or default execution limits
[defaults.execution]
max_iter = 20
max_retry_limit = 2
# max_rpm = null
# max_execution_time = null
# ============================================================================
# CACHING DEFAULTS
# ============================================================================
# Used when: Agent(caching=True)
[defaults.caching]
enabled = true
prompt_caching = false
# ============================================================================
# AUTONOMY DEFAULTS
# ============================================================================
# Used when: Agent(autonomy=True)
[defaults.autonomy]
level = "suggest" # suggest, auto_edit, full_auto
escalation_enabled = true
escalation_threshold = 3
doom_loop_detection = true
max_consecutive_failures = 3
require_approval_for_writes = true
require_approval_for_shell = true
@@ -181,7 +181,7 @@ class ArtifactStoreProtocol(Protocol):
Example implementation in wrapper:
class FileSystemArtifactStore:
def store(self, content, metadata):
# Write to ~/.praison/runs/{run_id}/artifacts/
# Write to ~/.praisonai/runs/{run_id}/artifacts/
...
"""
@@ -76,6 +76,13 @@ __all__ = [
"count_tokens",
"needs_chunking",
"get_recommended_chunk_size",
# EvaluationLoop (iterative improvement)
"EvaluationLoop",
"EvaluationLoopConfig",
"IterationResult",
"EvaluationLoopResult",
"EvaluationLoopProtocol",
"EvaluationLoopResultProtocol",
]
_LAZY_IMPORTS = {
@@ -134,6 +141,13 @@ _LAZY_IMPORTS = {
"count_tokens": ("tokens", "count_tokens"),
"needs_chunking": ("tokens", "needs_chunking"),
"get_recommended_chunk_size": ("tokens", "get_recommended_chunk_size"),
# EvaluationLoop (iterative improvement)
"EvaluationLoop": ("loop", "EvaluationLoop"),
"EvaluationLoopConfig": ("loop", "EvaluationLoopConfig"),
"IterationResult": ("results", "IterationResult"),
"EvaluationLoopResult": ("results", "EvaluationLoopResult"),
"EvaluationLoopProtocol": ("protocols", "EvaluationLoopProtocol"),
"EvaluationLoopResultProtocol": ("protocols", "EvaluationLoopResultProtocol"),
}
@@ -0,0 +1,276 @@
"""
EvaluationLoop - Iterative evaluation and improvement loop for agents.
This module provides the EvaluationLoop class which implements the "Ralph Loop"
pattern: run agent → judge output → improve → repeat until threshold met.
Example:
from praisonaiagents import Agent
from praisonaiagents.eval import EvaluationLoop
agent = Agent(name="analyzer", instructions="Analyze systems")
loop = EvaluationLoop(agent=agent, criteria="Analysis is thorough")
result = loop.run("Analyze the auth flow")
print(result.final_score) # 8.5
print(result.success) # True
"""
import time
import logging
from typing import TYPE_CHECKING, Optional, Callable, Any, List
from dataclasses import dataclass
if TYPE_CHECKING:
from ..agent.agent import Agent
logger = logging.getLogger(__name__)
@dataclass
class EvaluationLoopConfig:
"""Configuration for EvaluationLoop."""
criteria: str = ""
threshold: float = 8.0
max_iterations: int = 5
mode: str = "optimize"
model: str = "gpt-4o-mini"
verbose: bool = False
class EvaluationLoop:
"""
Iterative evaluation loop that runs an agent, judges output, and improves.
The loop continues until either:
- The score meets or exceeds the threshold (success)
- Max iterations is reached (failure in optimize mode)
- All iterations complete (review mode always runs all)
Args:
agent: The Agent instance to evaluate
criteria: Evaluation criteria for the Judge
threshold: Score threshold for success (default: 8.0)
max_iterations: Maximum number of iterations (default: 5)
mode: "optimize" (stop on success) or "review" (run all iterations)
judge: Optional custom Judge instance
on_iteration: Optional callback called after each iteration
verbose: Enable verbose logging
model: LLM model for Judge (default: gpt-4o-mini)
Example:
loop = EvaluationLoop(
agent=my_agent,
criteria="Response is helpful and accurate",
threshold=8.0,
max_iterations=5,
on_iteration=lambda r: print(f"Score: {r.score}")
)
result = loop.run("Analyze the codebase")
"""
def __init__(
self,
agent: "Agent",
criteria: str,
threshold: float = 8.0,
max_iterations: int = 5,
mode: str = "optimize",
judge: Optional[Any] = None,
on_iteration: Optional[Callable[[Any], None]] = None,
verbose: bool = False,
model: str = "gpt-4o-mini",
):
self.agent = agent
self.criteria = criteria
self.threshold = threshold
self.max_iterations = max_iterations
self.mode = mode
self._judge = judge
self.on_iteration = on_iteration
self.verbose = verbose
self.model = model
if mode not in ("optimize", "review"):
raise ValueError(f"mode must be 'optimize' or 'review', got '{mode}'")
@property
def judge(self):
"""Lazy-load Judge to avoid import-time overhead."""
if self._judge is None:
from .judge import Judge
self._judge = Judge(
criteria=self.criteria,
model=self.model,
)
return self._judge
def _get_agent_output(self, prompt: str, iteration: int, feedback: str = "") -> str:
"""Get output from agent, optionally including feedback from previous iteration."""
if iteration == 1 or not feedback:
return str(self.agent.chat(prompt))
improved_prompt = f"{prompt}\n\nPrevious feedback to address:\n{feedback}"
return str(self.agent.chat(improved_prompt))
async def _get_agent_output_async(self, prompt: str, iteration: int, feedback: str = "") -> str:
"""Async version of _get_agent_output."""
if iteration == 1 or not feedback:
if hasattr(self.agent, 'chat_async'):
return str(await self.agent.chat_async(prompt))
return str(self.agent.chat(prompt))
improved_prompt = f"{prompt}\n\nPrevious feedback to address:\n{feedback}"
if hasattr(self.agent, 'chat_async'):
return str(await self.agent.chat_async(improved_prompt))
return str(self.agent.chat(improved_prompt))
def run(self, prompt: str) -> "EvaluationLoopResult":
"""
Run the evaluation loop synchronously.
Args:
prompt: The prompt to send to the agent
Returns:
EvaluationLoopResult with iteration results
"""
from .results import IterationResult, EvaluationLoopResult
start_time = time.time()
iterations: List[IterationResult] = []
feedback = ""
for i in range(1, self.max_iterations + 1):
if self.verbose:
logger.info(f"EvaluationLoop iteration {i}/{self.max_iterations}")
output = self._get_agent_output(prompt, i, feedback)
judge_result = self.judge.run(
output=output,
criteria=self.criteria,
)
findings = getattr(judge_result, 'suggestions', []) or []
iteration_result = IterationResult(
iteration=i,
output=output,
score=judge_result.score,
reasoning=judge_result.reasoning,
findings=findings,
)
iterations.append(iteration_result)
if self.on_iteration:
try:
self.on_iteration(iteration_result)
except Exception as e:
logger.warning(f"on_iteration callback error: {e}")
if self.mode == "optimize" and judge_result.score >= self.threshold:
if self.verbose:
logger.info(f"Threshold met at iteration {i}: {judge_result.score} >= {self.threshold}")
break
feedback = judge_result.reasoning
if findings:
feedback += "\nSuggestions:\n" + "\n".join(f"- {s}" for s in findings)
total_duration = time.time() - start_time
success = iterations[-1].score >= self.threshold if iterations else False
result = EvaluationLoopResult(
iterations=iterations,
success=success,
total_duration_seconds=total_duration,
threshold=self.threshold,
mode=self.mode,
)
if self.verbose:
result.print_summary()
return result
async def run_async(self, prompt: str) -> "EvaluationLoopResult":
"""
Run the evaluation loop asynchronously.
Args:
prompt: The prompt to send to the agent
Returns:
EvaluationLoopResult with iteration results
"""
from .results import IterationResult, EvaluationLoopResult
start_time = time.time()
iterations: List[IterationResult] = []
feedback = ""
for i in range(1, self.max_iterations + 1):
if self.verbose:
logger.info(f"EvaluationLoop iteration {i}/{self.max_iterations}")
output = await self._get_agent_output_async(prompt, i, feedback)
if hasattr(self.judge, 'run_async'):
judge_result = await self.judge.run_async(
output=output,
criteria=self.criteria,
)
else:
judge_result = self.judge.run(
output=output,
criteria=self.criteria,
)
findings = getattr(judge_result, 'suggestions', []) or []
iteration_result = IterationResult(
iteration=i,
output=output,
score=judge_result.score,
reasoning=judge_result.reasoning,
findings=findings,
)
iterations.append(iteration_result)
if self.on_iteration:
try:
self.on_iteration(iteration_result)
except Exception as e:
logger.warning(f"on_iteration callback error: {e}")
if self.mode == "optimize" and judge_result.score >= self.threshold:
if self.verbose:
logger.info(f"Threshold met at iteration {i}: {judge_result.score} >= {self.threshold}")
break
feedback = judge_result.reasoning
if findings:
feedback += "\nSuggestions:\n" + "\n".join(f"- {s}" for s in findings)
total_duration = time.time() - start_time
success = iterations[-1].score >= self.threshold if iterations else False
result = EvaluationLoopResult(
iterations=iterations,
success=success,
total_duration_seconds=total_duration,
threshold=self.threshold,
mode=self.mode,
)
if self.verbose:
result.print_summary()
return result
__all__ = [
'EvaluationLoop',
'EvaluationLoopConfig',
]
@@ -243,4 +243,58 @@ __all__ = [
'OptimizationRuleProtocol',
'JudgeProtocol',
'JudgeResultProtocol',
'EvaluationLoopProtocol',
'EvaluationLoopResultProtocol',
]
class EvaluationLoopResultProtocol(Protocol):
"""Protocol for evaluation loop results."""
@property
def success(self) -> bool:
"""Whether the loop achieved the threshold."""
...
@property
def final_score(self) -> float:
"""Get the final iteration's score."""
...
@property
def score_history(self) -> List[float]:
"""Get list of all scores across iterations."""
...
@property
def final_output(self) -> str:
"""Get the final iteration's output."""
...
class EvaluationLoopProtocol(Protocol):
"""Protocol for evaluation loops that iteratively improve agent outputs."""
def run(self, prompt: str) -> EvaluationLoopResultProtocol:
"""
Run the evaluation loop synchronously.
Args:
prompt: The prompt to send to the agent
Returns:
EvaluationLoopResultProtocol with iteration results
"""
...
async def run_async(self, prompt: str) -> EvaluationLoopResultProtocol:
"""
Run the evaluation loop asynchronously.
Args:
prompt: The prompt to send to the agent
Returns:
EvaluationLoopResultProtocol with iteration results
"""
...
@@ -547,3 +547,189 @@ class CriteriaResult:
print(f" Average Score: {self.avg_score:.2f}")
print(f" Pass Rate: {self.pass_rate:.1%}")
print(f" Status: {'PASSED' if self.passed else 'FAILED'}")
@dataclass
class IterationResult:
"""
Result from a single iteration of an evaluation loop.
Attributes:
iteration: The iteration number (1-indexed)
output: The agent's output for this iteration
score: Quality score (1-10)
reasoning: Explanation for the score
findings: List of findings/suggestions from this iteration
timestamp: When this iteration completed
metadata: Additional metadata
"""
iteration: int
output: str
score: float
reasoning: str
findings: List[str] = field(default_factory=list)
timestamp: str = field(default_factory=lambda: datetime.now().isoformat())
metadata: Dict[str, Any] = field(default_factory=dict)
@property
def passed(self) -> bool:
"""Check if this iteration passed (score >= 7.0)."""
return self.score >= 7.0
def to_dict(self) -> Dict[str, Any]:
"""Convert result to dictionary."""
return {
"iteration": self.iteration,
"output": self.output,
"score": self.score,
"reasoning": self.reasoning,
"findings": self.findings,
"timestamp": self.timestamp,
"passed": self.passed,
"metadata": self.metadata,
}
@dataclass
class EvaluationLoopResult:
"""
Result from running an evaluation loop.
Attributes:
iterations: List of IterationResult from each iteration
success: Whether the loop achieved the threshold
total_duration_seconds: Total time taken
threshold: The score threshold used
mode: The mode used ('optimize' or 'review')
metadata: Additional metadata
"""
iterations: List[IterationResult] = field(default_factory=list)
success: bool = False
total_duration_seconds: float = 0.0
threshold: float = 8.0
mode: str = "optimize"
metadata: Dict[str, Any] = field(default_factory=dict)
@property
def final_score(self) -> float:
"""Get the final iteration's score."""
if not self.iterations:
return 0.0
return self.iterations[-1].score
@property
def score_history(self) -> List[float]:
"""Get list of all scores across iterations."""
return [it.score for it in self.iterations]
@property
def final_output(self) -> str:
"""Get the final iteration's output."""
if not self.iterations:
return ""
return self.iterations[-1].output
@property
def accumulated_findings(self) -> List[str]:
"""Get all findings accumulated across iterations."""
findings = []
for it in self.iterations:
findings.extend(it.findings)
return findings
@property
def num_iterations(self) -> int:
"""Get the number of iterations completed."""
return len(self.iterations)
def to_dict(self) -> Dict[str, Any]:
"""Convert result to dictionary."""
return {
"success": self.success,
"final_score": self.final_score,
"score_history": self.score_history,
"final_output": self.final_output,
"accumulated_findings": self.accumulated_findings,
"num_iterations": self.num_iterations,
"total_duration_seconds": self.total_duration_seconds,
"threshold": self.threshold,
"mode": self.mode,
"iterations": [it.to_dict() for it in self.iterations],
"metadata": self.metadata,
}
def to_json(self) -> str:
"""Convert result to JSON string."""
return json.dumps(self.to_dict(), indent=2)
@property
def final_report(self) -> str:
"""Generate a markdown report of the evaluation loop."""
lines = [
"# Evaluation Loop Report",
"",
f"**Status**: {'✅ Success' if self.success else '❌ Failed'}",
f"**Final Score**: {self.final_score:.1f}/10",
f"**Threshold**: {self.threshold}/10",
f"**Iterations**: {self.num_iterations}",
f"**Duration**: {self.total_duration_seconds:.2f}s",
f"**Mode**: {self.mode}",
"",
"## Score History",
"",
]
for i, score in enumerate(self.score_history, 1):
status = "✅" if score >= self.threshold else "⬜"
lines.append(f"- Iteration {i}: {score:.1f}/10 {status}")
lines.append("")
lines.append("## Iteration Details")
lines.append("")
for it in self.iterations:
lines.append(f"### Iteration {it.iteration}")
lines.append(f"**Score**: {it.score:.1f}/10")
lines.append(f"**Reasoning**: {it.reasoning}")
if it.findings:
lines.append("**Findings**:")
for f in it.findings:
lines.append(f" - {f}")
lines.append("")
if self.accumulated_findings:
lines.append("## All Findings")
lines.append("")
for f in self.accumulated_findings:
lines.append(f"- {f}")
return "\n".join(lines)
def print_summary(self) -> None:
"""Print a summary of the evaluation loop results."""
try:
from rich.console import Console
from rich.table import Table
console = Console()
table = Table(title="Evaluation Loop Summary")
table.add_column("Metric", style="cyan")
table.add_column("Value", style="green" if self.success else "red")
table.add_row("Status", "✅ SUCCESS" if self.success else "❌ FAILED")
table.add_row("Final Score", f"{self.final_score:.1f}/10")
table.add_row("Threshold", f"{self.threshold}/10")
table.add_row("Iterations", str(self.num_iterations))
table.add_row("Duration", f"{self.total_duration_seconds:.2f}s")
table.add_row("Mode", self.mode)
table.add_row("Score History", " → ".join(f"{s:.1f}" for s in self.score_history))
console.print(table)
except ImportError:
print("Evaluation Loop Summary")
print(f" Status: {'SUCCESS' if self.success else 'FAILED'}")
print(f" Final Score: {self.final_score:.1f}/10")
print(f" Threshold: {self.threshold}/10")
print(f" Iterations: {self.num_iterations}")
print(f" Duration: {self.total_duration_seconds:.2f}s")
@@ -11,7 +11,15 @@ from typing import Optional, Dict, Any, List, Callable, Union, Literal
class HookEvent(str, Enum):
"""Event names for the hook system."""
"""Event names for the hook system.
This enum is also aliased as PluginHook for backward compatibility.
All plugin lifecycle events are included here for DRY compliance.
"""
# Plugin/System lifecycle
ON_INIT = "on_init"
ON_SHUTDOWN = "on_shutdown"
# Tool lifecycle
BEFORE_TOOL = "before_tool"
AFTER_TOOL = "after_tool"
@@ -33,6 +41,8 @@ class HookEvent(str, Enum):
ON_RETRY = "on_retry"
# Message lifecycle (for bot/channel integrations)
BEFORE_MESSAGE = "before_message"
AFTER_MESSAGE = "after_message"
MESSAGE_RECEIVED = "message_received"
MESSAGE_SENDING = "message_sending"
MESSAGE_SENT = "message_sent"
@@ -48,6 +58,11 @@ class HookEvent(str, Enum):
# Tool result persistence (for modifying tool results before storage)
TOOL_RESULT_PERSIST = "tool_result_persist"
# Permission/Config/Auth hooks
ON_PERMISSION_ASK = "on_permission_ask"
ON_CONFIG = "on_config"
ON_AUTH = "on_auth"
# Claude Code parity events
USER_PROMPT_SUBMIT = "user_prompt_submit" # When user submits a prompt
NOTIFICATION = "notification" # When notification is sent
@@ -416,7 +416,7 @@ class Knowledge:
def config(self):
# Generate unique collection name for each instance (only if not provided in config)
default_collection = f"test_{int(time.time())}_{str(uuid.uuid4())[:8]}"
persist_dir = ".praison"
persist_dir = ".praisonai"
# Create persistent client config
base_config = {
@@ -862,7 +862,7 @@ class Knowledge:
result = IndexResult()
# Initialize file tracker for incremental indexing
state_dir = os.path.join(os.path.dirname(path) if os.path.isfile(path) else path, ".praison")
state_dir = os.path.join(os.path.dirname(path) if os.path.isfile(path) else path, ".praisonai")
os.makedirs(state_dir, exist_ok=True)
state_file = os.path.join(state_dir, ".index_state.json")
@@ -1681,6 +1681,23 @@ Now provide your final answer using this result. Summarize the information natur
response_text = resp["choices"][0]["message"]["content"]
final_response = resp
# Emit StreamEvent for reasoning content if callback provided
if _emit and reasoning_content:
stream_callback(StreamEvent(
type=StreamEventType.DELTA_TEXT,
timestamp=time.perf_counter(),
content=reasoning_content,
is_reasoning=True
))
# Emit StreamEvent for response content if callback provided
if _emit and response_text:
stream_callback(StreamEvent(
type=StreamEventType.DELTA_TEXT,
timestamp=time.perf_counter(),
content=response_text,
is_reasoning=False
))
# Track token usage
if self.metrics:
self._track_token_usage(final_response, self.model)
@@ -764,6 +764,14 @@ class OpenAIClient:
if rc:
reasoning_content += rc
live.update(display_fn(f"{full_response_text}\n[Reasoning: {reasoning_content}]", start_time))
# Emit reasoning content as StreamEvent with is_reasoning=True
if _emit:
stream_callback(StreamEvent(
type=StreamEventType.DELTA_TEXT,
timestamp=time.perf_counter(),
content=rc,
is_reasoning=True
))
# Clear the last generating display with a blank line
console.print()
@@ -978,6 +986,14 @@ class OpenAIClient:
if rc:
reasoning_content += rc
live.update(display_fn(f"{full_response_text}\n[Reasoning: {reasoning_content}]", start_time))
# Emit reasoning content as StreamEvent with is_reasoning=True
if _emit:
stream_callback(StreamEvent(
type=StreamEventType.DELTA_TEXT,
timestamp=time.perf_counter(),
content=rc,
is_reasoning=True
))
# Clear the last generating display with a blank line
console.print()
@@ -1543,6 +1559,8 @@ class OpenAIClient:
reasoning_steps: bool = False,
verbose: bool = True,
max_iterations: int = 10,
stream_callback: Optional[Callable] = None,
emit_events: bool = False,
**kwargs
):
"""
@@ -1560,6 +1578,8 @@ class OpenAIClient:
reasoning_steps: Whether to show reasoning
verbose: Whether to show verbose output
max_iterations: Maximum tool calling iterations
stream_callback: Optional callback for StreamEvent emission
emit_events: Whether to emit StreamEvents via callback
**kwargs: Additional API parameters
Yields:
@@ -1568,8 +1588,19 @@ class OpenAIClient:
# Format tools for OpenAI API
formatted_tools = self.format_tools(tools)
# Setup StreamEvent emission if enabled
_emit = emit_events and stream_callback is not None
if _emit:
from ..streaming.events import StreamEvent, StreamEventType
stream_callback(StreamEvent(
type=StreamEventType.REQUEST_START,
timestamp=time.perf_counter(),
metadata={"model": model, "provider": "openai"}
))
# Continue tool execution loop until no more tool calls are needed
iteration_count = 0
_first_token_emitted = False
while iteration_count < max_iterations:
try:
@@ -1594,6 +1625,20 @@ class OpenAIClient:
content = chunk.choices[0].delta.content
full_response_text += content
yield content
# Emit StreamEvent for text delta
if _emit:
if not _first_token_emitted:
stream_callback(StreamEvent(
type=StreamEventType.FIRST_TOKEN,
timestamp=time.perf_counter()
))
_first_token_emitted = True
stream_callback(StreamEvent(
type=StreamEventType.DELTA_TEXT,
timestamp=time.perf_counter(),
content=content,
is_reasoning=False
))
# Handle reasoning content if enabled
if reasoning_steps and chunk.choices and hasattr(chunk.choices[0].delta, "reasoning_content"):
@@ -1601,6 +1646,14 @@ class OpenAIClient:
if rc:
reasoning_content += rc
yield f"[Reasoning: {rc}]"
# Emit StreamEvent for reasoning content
if _emit:
stream_callback(StreamEvent(
type=StreamEventType.DELTA_TEXT,
timestamp=time.perf_counter(),
content=rc,
is_reasoning=True
))
# Process the complete response to check for tool calls
final_response = process_stream_chunks(chunks)
@@ -63,7 +63,7 @@ class MCPAuthStorage:
Initialize auth storage.
Args:
filepath: Path to auth storage file. Defaults to ~/.praison/mcp-auth.json
filepath: Path to auth storage file. Defaults to ~/.praisonai/mcp-auth.json
"""
self.filepath = filepath or get_default_auth_filepath()
self._ensure_directory()
@@ -7,14 +7,14 @@ Provides reusable documentation context support similar to:
- Project documentation for AI context
Features:
- Auto-discovery of docs from .praison/docs/
- Auto-discovery of docs from .praisonai/docs/
- Support for markdown, text, and code files
- @doc mentions for including specific docs
- Priority ordering for docs
- Character limits for context window management
Storage Structure:
.praison/docs/
.praisonai/docs/
├── project-overview.md # Project overview
├── architecture.md # Architecture decisions
├── api-reference.md # API documentation
@@ -74,13 +74,13 @@ class DocsManager:
Manages documentation context for AI agents.
Provides:
- Auto-discovery of docs from .praison/docs/
- Global docs from ~/.praison/docs/
- Auto-discovery of docs from .praisonai/docs/
- Global docs from ~/.praisonai/docs/
- @doc mentions for including specific docs
- Priority-based ordering
"""
DOCS_DIR_NAME = ".praison/docs"
DOCS_DIR_NAME = ".praisonai/docs"
SUPPORTED_EXTENSIONS = [".md", ".txt", ".rst"]
# Maximum characters per doc (context window management)
@@ -97,11 +97,11 @@ class DocsManager:
Args:
workspace_path: Path to workspace/project root
global_docs_path: Path to global docs (default: ~/.praison/docs)
global_docs_path: Path to global docs (default: ~/.praisonai/docs)
verbose: Verbosity level
"""
self.workspace_path = Path(workspace_path) if workspace_path else Path.cwd()
self.global_docs_path = Path(global_docs_path) if global_docs_path else Path.home() / ".praison" / "docs"
self.global_docs_path = Path(global_docs_path) if global_docs_path else Path.home() / ".praisonai" / "docs"
self.verbose = verbose
self._docs: Dict[str, Doc] = {}
@@ -6,7 +6,7 @@ Provides short-term, long-term, entity, and episodic memory
without requiring any external packages.
Storage Structure:
.praison/memory/{user_id}/
.praisonai/memory/{user_id}/
├── config.json # Memory configuration
├── short_term.json # Rolling buffer (recent context)
├── long_term.json # Persistent facts/knowledge
@@ -129,7 +129,7 @@ class FileMemory:
Args:
user_id: Unique identifier for the user/session
base_path: Base directory for memory storage (default: .praison/memory)
base_path: Base directory for memory storage (default: .praisonai/memory)
config: Configuration overrides
verbose: Verbosity level (0=quiet, 1=info, 2+=debug)
"""
@@ -140,7 +140,7 @@ class FileMemory:
if base_path:
self.base_path = Path(base_path)
else:
self.base_path = Path(".praison/memory")
self.base_path = Path(".praisonai/memory")
self.user_path = self.base_path / user_id
self.episodic_path = self.user_path / "episodic"
@@ -6,12 +6,12 @@ allowing custom actions before/after agent operations.
Features:
- Pre/post hooks for read, write, command, prompt operations
- JSON configuration (.praison/hooks.json)
- JSON configuration (.praisonai/hooks.json)
- Script execution with timeout
- Exit code handling
- Lazy loading for performance
Configuration (.praison/hooks.json):
Configuration (.praisonai/hooks.json):
{
"hooks": {
"pre_write_code": "./scripts/lint.sh",
@@ -85,7 +85,7 @@ class HooksManager:
"""
Manages hook discovery, loading, and execution.
Hooks are configured in .praison/hooks.json and can execute
Hooks are configured in .praisonai/hooks.json and can execute
scripts before/after agent operations.
Example:
@@ -103,7 +103,7 @@ class HooksManager:
```
"""
CONFIG_FILE = ".praison/hooks.json"
CONFIG_FILE = ".praisonai/hooks.json"
def __init__(
self,
@@ -384,7 +384,7 @@ class HooksManager:
timeout: Global timeout in seconds
enabled: Whether hooks are enabled
"""
config_dir = self.workspace_path / ".praison"
config_dir = self.workspace_path / ".praisonai"
config_dir.mkdir(parents=True, exist_ok=True)
config = {
@@ -220,10 +220,11 @@ class LearnManager:
"""
Generate context suitable for injection into system prompt.
Returns a formatted string with relevant learnings.
Returns a formatted string with relevant learnings from all enabled stores.
"""
parts = []
# 1. Persona - User preferences and profile
if "persona" in self._stores:
personas = self.get_persona_context(limit=5)
if personas:
@@ -231,6 +232,7 @@ class LearnManager:
for p in personas:
parts.append(f"- {p}")
# 2. Insights - Observations and learnings
if "insights" in self._stores:
insights = self.get_insights_context(limit=5)
if insights:
@@ -238,4 +240,38 @@ class LearnManager:
for i in insights:
parts.append(f"- {i}")
# 3. Patterns - Reusable knowledge patterns
if "patterns" in self._stores:
entries = self._stores["patterns"].list_all(limit=3)
if entries:
parts.append("\nKnown Patterns:")
for e in entries:
parts.append(f"- {e.content}")
# 4. Decisions - Past decision records (for consistency)
if "decisions" in self._stores:
entries = self._stores["decisions"].list_all(limit=3)
if entries:
parts.append("\nPast Decisions:")
for e in entries:
parts.append(f"- {e.content}")
# 5. Feedback - User feedback signals
if "feedback" in self._stores:
entries = self._stores["feedback"].list_all(limit=3)
if entries:
parts.append("\nUser Feedback:")
for e in entries:
parts.append(f"- {e.content}")
# 6. Improvements - Self-improvement proposals
if "improvements" in self._stores:
entries = self._stores["improvements"].list_all(limit=3)
if entries:
parts.append("\nImprovement Areas:")
for e in entries:
parts.append(f"- {e.content}")
# Note: threads are session-specific and not included in system prompt
return "\n".join(parts) if parts else ""
@@ -6,13 +6,13 @@ Provides JSON-based MCP server configuration support similar to:
- Windsurf MCP configuration
Features:
- Auto-discovery of MCP configs from .praison/mcp/
- Auto-discovery of MCP configs from .praisonai/mcp/
- JSON config files for MCP servers
- Environment variable support
- Easy server management
Storage Structure:
.praison/mcp/
.praisonai/mcp/
├── filesystem.json # Filesystem MCP server
├── brave-search.json # Brave search MCP server
└── postgres.json # PostgreSQL MCP server
@@ -108,12 +108,12 @@ class MCPConfigManager:
Manages MCP server configurations for AI agents.
Provides:
- Auto-discovery of MCP configs from .praison/mcp/
- Global configs from ~/.praison/mcp/
- Auto-discovery of MCP configs from .praisonai/mcp/
- Global configs from ~/.praisonai/mcp/
- Easy server management
"""
MCP_DIR_NAME = ".praison/mcp"
MCP_DIR_NAME = ".praisonai/mcp"
SUPPORTED_EXTENSIONS = [".json", ".mcpjson"]
def __init__(
@@ -127,11 +127,11 @@ class MCPConfigManager:
Args:
workspace_path: Path to workspace/project root
global_mcp_path: Path to global MCP configs (default: ~/.praison/mcp)
global_mcp_path: Path to global MCP configs (default: ~/.praisonai/mcp)
verbose: Verbosity level
"""
self.workspace_path = Path(workspace_path) if workspace_path else Path.cwd()
self.global_mcp_path = Path(global_mcp_path) if global_mcp_path else Path.home() / ".praison" / "mcp"
self.global_mcp_path = Path(global_mcp_path) if global_mcp_path else Path.home() / ".praisonai" / "mcp"
self.verbose = verbose
self._configs: Dict[str, MCPConfig] = {}
@@ -226,15 +226,15 @@ class Memory:
self.embedding_dimensions = self._get_embedding_dimensions(self.embedding_model)
self._log_verbose(f"Using embedding dimensions: {self.embedding_dimensions}")
# Create .praison directory if it doesn't exist
os.makedirs(".praison", exist_ok=True)
# Create .praisonai directory if it doesn't exist
os.makedirs(".praisonai", exist_ok=True)
# Short-term DB
self.short_db = self.cfg.get("short_db", ".praison/short_term.db")
self.short_db = self.cfg.get("short_db", ".praisonai/short_term.db")
self._init_stm()
# Long-term DB
self.long_db = self.cfg.get("long_db", ".praison/long_term.db")
self.long_db = self.cfg.get("long_db", ".praisonai/long_term.db")
self._init_ltm()
# Conditionally init Mem0, MongoDB, or local RAG
@@ -18,7 +18,7 @@ Features:
- Symlink support for shared rules
Storage Structure:
.praison/rules/
.praisonai/rules/
├── global.md # Always applied (global)
├── python.md # Glob: **/*.py
├── testing.md # Glob: **/*.test.*
@@ -99,9 +99,9 @@ class RulesManager:
Manages rules/instructions for AI agents.
Supports multiple rule sources:
- Global rules (~/.praison/rules/)
- Workspace rules (.praison/rules/)
- Subdirectory rules (subdir/.praison/rules/)
- Global rules (~/.praisonai/rules/)
- Workspace rules (.praisonai/rules/)
- Subdirectory rules (subdir/.praisonai/rules/)
- Root instruction files (CLAUDE.md, AGENTS.md, GEMINI.md, .cursorrules, .windsurfrules)
Example:
@@ -122,7 +122,7 @@ class RulesManager:
```
"""
RULES_DIR_NAME = ".praison/rules"
RULES_DIR_NAME = ".praisonai/rules"
SUPPORTED_EXTENSIONS = [".md", ".mdc", ".txt"]
# Root-level instruction files (like Claude, Codex, Gemini, Cursor, Windsurf)
@@ -161,11 +161,11 @@ class RulesManager:
Args:
workspace_path: Path to workspace/project root
global_rules_path: Path to global rules (default: ~/.praison/rules)
global_rules_path: Path to global rules (default: ~/.praisonai/rules)
verbose: Verbosity level
"""
self.workspace_path = Path(workspace_path) if workspace_path else Path.cwd()
self.global_rules_path = Path(global_rules_path) if global_rules_path else Path.home() / ".praison" / "rules"
self.global_rules_path = Path(global_rules_path) if global_rules_path else Path.home() / ".praisonai" / "rules"
self.verbose = verbose
self._rules: Dict[str, Rule] = {}
@@ -457,7 +457,7 @@ class RulesManager:
if rule:
self._rules[f"root:{rule.name}"] = rule
# 4. Load workspace rules from .praison/rules/
# 4. Load workspace rules from .praisonai/rules/
workspace_rules_dir = self.workspace_path / self.RULES_DIR_NAME.replace("/", os.sep)
workspace_rules = self._discover_rules_in_dir(workspace_rules_dir)
for rule in workspace_rules:
@@ -485,7 +485,7 @@ class RulesManager:
stop_at = git_root if git_root else self.workspace_path
current = Path.cwd()
while current != stop_at and current != current.parent:
# Check .praison/rules/
# Check .praisonai/rules/
subdir_rules_dir = current / self.RULES_DIR_NAME.replace("/", os.sep)
subdir_rules = self._discover_rules_in_dir(subdir_rules_dir)
for rule in subdir_rules:
@@ -6,14 +6,14 @@ allowing users to define and execute complex task sequences.
Features:
- YAML/Markdown workflow definitions
- Auto-discovery from .praison/workflows/
- Auto-discovery from .praisonai/workflows/
- Step-by-step execution with context passing
- Variable substitution
- Conditional steps
- Lazy loading for performance
Storage Structure:
.praison/workflows/
.praisonai/workflows/
├── deploy.md # Deployment workflow
├── test.md # Testing workflow
└── review.md # Code review workflow
@@ -731,7 +731,7 @@ class WorkflowManager:
Workflows are defined in markdown files with YAML frontmatter
and step definitions.
Example workflow file (.praison/workflows/deploy.md):
Example workflow file (.praisonai/workflows/deploy.md):
```markdown
---
name: Deploy to Production
@@ -782,7 +782,7 @@ class WorkflowManager:
```
"""
WORKFLOWS_DIR = ".praison/workflows"
WORKFLOWS_DIR = ".praisonai/workflows"
SUPPORTED_EXTENSIONS = [".md", ".yaml", ".yml"]
def __init__(
@@ -1857,7 +1857,7 @@ class WorkflowManager:
def _get_checkpoints_dir(self) -> Path:
"""Get the checkpoints directory path."""
checkpoints_dir = self.workspace_path / ".praison" / "checkpoints"
checkpoints_dir = self.workspace_path / ".praisonai" / "checkpoints"
checkpoints_dir.mkdir(parents=True, exist_ok=True)
return checkpoints_dir
@@ -234,6 +234,56 @@ def get_mcp_auth_path() -> Path:
return get_data_dir() / "mcp-auth.json"
def get_memory_dir() -> Path:
"""
Get memory directory for short/long term databases.
Returns:
Path to ~/.praisonai/memory/
"""
return get_data_dir() / "memory"
def get_workflows_dir() -> Path:
"""
Get workflows directory.
Returns:
Path to ~/.praisonai/workflows/
"""
return get_data_dir() / "workflows"
def get_summaries_dir() -> Path:
"""
Get summaries directory for RAG.
Returns:
Path to ~/.praisonai/summaries/
"""
return get_data_dir() / "summaries"
def get_prp_dir() -> Path:
"""
Get PRP (Prompt Response Pair) output directory.
Returns:
Path to ~/.praisonai/prp/
"""
return get_data_dir() / "prp"
def get_runs_dir() -> Path:
"""
Get runs directory for artifacts.
Returns:
Path to ~/.praisonai/runs/
"""
return get_data_dir() / "runs"
def get_project_data_dir(project_path: Optional[Union[str, Path]] = None) -> Path:
"""
Get project-level data directory.
@@ -307,4 +357,9 @@ def get_all_paths() -> Dict[str, Path]:
"learn": get_learn_dir(),
"cache": get_cache_dir(),
"mcp_auth": get_mcp_auth_path(),
"memory": get_memory_dir(),
"workflows": get_workflows_dir(),
"summaries": get_summaries_dir(),
"prp": get_prp_dir(),
"runs": get_runs_dir(),
}
@@ -37,10 +37,16 @@ Usage:
"""
__all__ = [
# Easy enable API
"enable",
"disable",
"list_plugins",
"is_enabled",
# Core
"PluginManager",
"Plugin",
"PluginHook",
"PluginType",
"PluginInfo",
"FunctionPlugin",
"get_plugin_manager",
@@ -64,6 +70,170 @@ __all__ = [
]
# ============================================================================
# EASY ENABLE API
# ============================================================================
# These functions provide a simple way to enable/disable plugins globally.
# Zero performance impact when not called - plugins only load on enable().
# ============================================================================
# Global state for plugin system (lazy initialized)
_plugins_enabled: bool = False
_enabled_plugin_names: list = None # None = all, list = specific
def enable(plugins: list = None) -> None:
"""Enable the plugin system.
Discovers and enables plugins from default directories.
This is the main entry point for using plugins.
Args:
plugins: Optional list of plugin names to enable.
If None, enables all discovered plugins.
Examples:
# Enable all discovered plugins
from praisonaiagents import plugins
plugins.enable()
# Enable specific plugins only
plugins.enable(["logging", "metrics"])
Note:
- Tools and guardrails work WITHOUT calling enable()
- Only background plugins (hooks, metrics, logging) need enable()
- Can also be enabled via PRAISONAI_PLUGINS env var
- Can also be enabled via .praisonai/config.toml
"""
global _plugins_enabled, _enabled_plugin_names
_plugins_enabled = True
_enabled_plugin_names = plugins # None = all, list = specific
# Get plugin manager and auto-discover
from .manager import get_plugin_manager
manager = get_plugin_manager()
# Auto-discover plugins from default directories
manager.auto_discover_plugins()
# Enable specific plugins or all
if plugins is not None:
# Enable only specified plugins
for name in plugins:
manager.enable(name)
else:
# Enable all discovered plugins
for plugin_info in manager.list_plugins():
manager.enable(plugin_info.get("name", ""))
import logging
logging.debug(f"Plugins enabled: {plugins if plugins else 'all'}")
def disable(plugins: list = None) -> None:
"""Disable plugins.
Args:
plugins: Optional list of plugin names to disable.
If None, disables all plugins.
Examples:
# Disable all plugins
plugins.disable()
# Disable specific plugins
plugins.disable(["logging"])
"""
global _plugins_enabled, _enabled_plugin_names
from .manager import get_plugin_manager
manager = get_plugin_manager()
if plugins is not None:
# Disable specific plugins
for name in plugins:
manager.disable(name)
else:
# Disable all plugins
_plugins_enabled = False
_enabled_plugin_names = None
for plugin_info in manager.list_plugins():
manager.disable(plugin_info.get("name", ""))
def list_plugins() -> list:
"""List all discovered plugins.
Returns:
List of plugin info dicts with name, version, enabled status.
Example:
from praisonaiagents import plugins
all_plugins = plugins.list_plugins()
for p in all_plugins:
print(f"{p['name']} v{p['version']} - {'enabled' if p['enabled'] else 'disabled'}")
"""
from .manager import get_plugin_manager
manager = get_plugin_manager()
# Auto-discover if not already done
if not manager._plugins and not manager._single_file_plugins:
manager.auto_discover_plugins()
# Combine registered plugins and single-file plugins
result = []
# Add registered plugins (Plugin class instances)
for info in manager.list_plugins():
if hasattr(info, 'name'):
# PluginInfo object
result.append({
"name": info.name,
"version": getattr(info, 'version', '1.0.0'),
"description": getattr(info, 'description', ''),
"enabled": manager.is_enabled(info.name),
"type": "registered",
})
elif isinstance(info, dict):
# Already a dict
info["enabled"] = manager.is_enabled(info.get("name", ""))
info["type"] = "registered"
result.append(info)
# Add single-file plugins
for name, meta in manager._single_file_plugins.items():
result.append({
"name": name,
"version": meta.get("version", "1.0.0"),
"description": meta.get("description", ""),
"enabled": manager.is_enabled(name),
"type": "single_file",
})
return result
def is_enabled(name: str = None) -> bool:
"""Check if plugins are enabled.
Args:
name: Optional plugin name to check. If None, checks if system is enabled.
Returns:
True if enabled, False otherwise.
"""
global _plugins_enabled
if name is None:
return _plugins_enabled
from .manager import get_plugin_manager
manager = get_plugin_manager()
return manager.is_enabled(name)
def __getattr__(name: str):
"""Lazy load module components."""
# Core classes
@@ -91,6 +261,10 @@ def __getattr__(name: str):
from .plugin import FunctionPlugin
return FunctionPlugin
if name == "PluginType":
from .plugin import PluginType
return PluginType
# Protocols
if name == "PluginProtocol":
from .protocols import PluginProtocol
@@ -5,8 +5,8 @@ Discovers and loads plugins from directories.
Plugins are simple Python files with WordPress-style docstring headers.
Default plugin directories (in precedence order):
1. Project: ./.praison/plugins/
2. User: ~/.praison/plugins/
1. Project: ./.praisonai/plugins/
2. User: ~/.praisonai/plugins/
Usage:
from praisonaiagents.plugins.discovery import discover_plugins, load_plugin
@@ -25,6 +25,7 @@ from pathlib import Path
from typing import Any, Dict, List, Optional
from .parser import parse_plugin_header_from_file, PluginParseError
from ..paths import get_plugins_dir, get_project_data_dir
logger = logging.getLogger(__name__)
@@ -32,25 +33,26 @@ logger = logging.getLogger(__name__)
def get_default_plugin_dirs() -> List[Path]:
"""Get default plugin directory locations.
Uses centralized paths.py for consistent path management.
Returns directories in precedence order (high to low):
1. Project: ./.praison/plugins/
2. User: ~/.praison/plugins/
1. Project: ./.praisonai/plugins/
2. User: ~/.praisonai/plugins/
Returns:
List of existing plugin directories
"""
dirs = []
cwd = Path.cwd()
# Project-level directory
project_dir = cwd / ".praison" / "plugins"
if project_dir.exists() and project_dir.is_dir():
dirs.append(project_dir)
# Project-level directory (use centralized path)
project_data_dir = get_project_data_dir()
project_plugins = project_data_dir / "plugins"
if project_plugins.exists() and project_plugins.is_dir():
dirs.append(project_plugins)
# User-level directory
user_dir = Path.home() / ".praison" / "plugins"
if user_dir.exists() and user_dir.is_dir():
dirs.append(user_dir)
# User-level directory (use centralized path)
user_plugins = get_plugins_dir()
if user_plugins.exists() and user_plugins.is_dir():
dirs.append(user_plugins)
return dirs
@@ -253,12 +255,13 @@ def unload_plugin(module_name: str) -> bool:
def ensure_plugin_dir() -> Path:
"""Ensure the user plugin directory exists.
Creates ~/.praison/plugins/ if it doesn't exist.
Creates ~/.praisonai/plugins/ if it doesn't exist.
Uses centralized paths.py for consistent path management.
Returns:
Path to the user plugin directory
"""
user_dir = Path.home() / ".praison" / "plugins"
user_dir = get_plugins_dir()
user_dir.mkdir(parents=True, exist_ok=True)
return user_dir
@@ -2,14 +2,16 @@
Plugin Manager for PraisonAI Agents.
Provides dynamic plugin discovery, loading, and lifecycle management.
Thread-safe and async-safe for multi-agent environments.
"""
import asyncio
import importlib.util
import logging
import os
import sys
import threading
from pathlib import Path
from typing import Any, Callable, Dict, List, Optional, Type
from typing import Any, Dict, List, Optional
from .plugin import Plugin, PluginHook, PluginInfo, FunctionPlugin
@@ -20,6 +22,9 @@ class PluginManager:
"""
Manages plugin discovery, loading, and execution.
Thread-safe for concurrent access in multi-agent environments.
Supports both sync and async hook execution.
Example:
manager = PluginManager()
@@ -29,14 +34,18 @@ class PluginManager:
# Register a plugin directly
manager.register(MyPlugin())
# Execute hooks
# Execute hooks (sync)
args = manager.execute_hook(PluginHook.BEFORE_TOOL, "bash", {"cmd": "ls"})
# Execute hooks (async)
args = await manager.async_execute_hook(PluginHook.BEFORE_TOOL, "bash", {"cmd": "ls"})
"""
def __init__(self):
self._plugins: Dict[str, Plugin] = {}
self._enabled: Dict[str, bool] = {}
self._single_file_plugins: Dict[str, Dict[str, Any]] = {} # WordPress-style plugins
self._lock = threading.RLock() # Thread safety for multi-agent environments
def register(self, plugin: Plugin) -> bool:
"""
@@ -266,6 +275,33 @@ class PluginManager:
return result
async def async_execute_hook(
self,
hook: PluginHook,
*args,
**kwargs
) -> Any:
"""
Execute a hook across all enabled plugins asynchronously.
Async-safe for use in async agent contexts.
Args:
hook: The hook to execute
*args: Arguments to pass to hook
**kwargs: Keyword arguments to pass to hook
Returns:
The result (may be modified by plugins)
"""
# Use the sync version in an executor to avoid blocking
# This is safe because execute_hook is thread-safe with _lock
loop = asyncio.get_event_loop()
return await loop.run_in_executor(
None,
lambda: self.execute_hook(hook, *args, **kwargs)
)
def get_all_tools(self) -> List[Dict[str, Any]]:
"""Get all tools from all enabled plugins."""
tools = []
@@ -364,8 +400,8 @@ class PluginManager:
Auto-discover and load plugins from default directories.
Scans:
- ./.praison/plugins/ (project-level)
- ~/.praison/plugins/ (user-level)
- ./.praisonai/plugins/ (project-level)
- ~/.praisonai/plugins/ (user-level)
Returns:
Number of plugins loaded
@@ -2,6 +2,7 @@
Plugin definitions for PraisonAI Agents.
Provides base plugin class and hook definitions.
PluginHook is now an alias for HookEvent (DRY compliance).
"""
import logging
@@ -10,68 +11,26 @@ from dataclasses import dataclass, field
from enum import Enum
from typing import Any, Callable, Dict, List, Optional
# Import HookEvent at module level for alias (DRY)
from ..hooks.types import HookEvent
logger = logging.getLogger(__name__)
# PluginHook is now an alias for HookEvent - single source of truth
PluginHook = HookEvent
class PluginHook(str, Enum):
"""Available plugin hooks."""
class PluginType(str, Enum):
"""Types of plugins that can be registered.
# Lifecycle hooks
ON_INIT = "on_init"
ON_SHUTDOWN = "on_shutdown"
# Agent hooks
BEFORE_AGENT = "before_agent"
AFTER_AGENT = "after_agent"
# Tool hooks
BEFORE_TOOL = "before_tool"
AFTER_TOOL = "after_tool"
# Message hooks (for bot/channel integrations)
BEFORE_MESSAGE = "before_message"
AFTER_MESSAGE = "after_message"
MESSAGE_RECEIVED = "message_received"
MESSAGE_SENDING = "message_sending"
MESSAGE_SENT = "message_sent"
# LLM hooks
BEFORE_LLM = "before_llm"
AFTER_LLM = "after_llm"
# Session hooks
SESSION_START = "session_start"
SESSION_END = "session_end"
# Gateway hooks
GATEWAY_START = "gateway_start"
GATEWAY_STOP = "gateway_stop"
# Compaction hooks (memory management)
BEFORE_COMPACTION = "before_compaction"
AFTER_COMPACTION = "after_compaction"
# Tool result persistence (for modifying tool results before storage)
TOOL_RESULT_PERSIST = "tool_result_persist"
# Permission hooks
ON_PERMISSION_ASK = "on_permission_ask"
# Config hooks
ON_CONFIG = "on_config"
# Auth hooks
ON_AUTH = "on_auth"
# Error hooks
ON_ERROR = "on_error"
ON_RETRY = "on_retry"
# Claude Code parity hooks
USER_PROMPT_SUBMIT = "user_prompt_submit" # When user submits a prompt
NOTIFICATION = "notification" # When notification is sent
SUBAGENT_STOP = "subagent_stop" # When subagent completes
SETUP = "setup" # On initialization/maintenance
Used to categorize plugins and enable type-specific discovery.
"""
TOOL = "tool" # Provides tools for agents
HOOK = "hook" # Intercepts lifecycle events
SKILL = "skill" # Provides agent capabilities (SKILL.md)
POLICY = "policy" # Provides execution rules
GUARDRAIL = "guardrail" # Provides validation rules
INTEGRATION = "integration" # External tool integrations
@dataclass
@@ -18,21 +18,42 @@ class Process:
DEFAULT_RETRY_LIMIT = 3 # Predefined retry limit in a common place
VALIDATION_FAILURE_DECISIONS = ["invalid", "retry", "failed", "error", "unsuccessful", "fail", "errors", "reject", "rejected", "incomplete"] # Decision strings that trigger validation feedback
def __init__(self, tasks: Dict[str, Task], agents: List[Agent], manager_llm: Optional[str] = None, verbose: bool = False, max_iter: int = 10):
def __init__(
self,
tasks: Dict[str, Task],
agents: List[Agent],
manager_llm: Optional[str] = None,
verbose: bool = False,
max_iter: int = 10,
output: Optional[str] = None,
):
logging.debug(f"=== Initializing Process ===")
logging.debug(f"Number of tasks: {len(tasks)}")
logging.debug(f"Number of agents: {len(agents)}")
logging.debug(f"Manager LLM: {manager_llm}")
logging.debug(f"Verbose mode: {verbose}")
logging.debug(f"Max iterations: {max_iter}")
self.tasks = tasks
self.agents = agents
self.manager_llm = manager_llm
self.verbose = verbose
self.max_iter = max_iter
self.task_retry_counter: Dict[str, int] = {} # Initialize retry counter
self.workflow_finished = False # ADDED: Workflow finished flag
# Resolve verbose from output= param (takes precedence) or legacy verbose= param
if output is not None:
# output= takes precedence over verbose=
from ..config.presets import OUTPUT_PRESETS
preset = OUTPUT_PRESETS.get(output, {})
self._verbose = preset.get("verbose", False)
else:
# Backward compat: use legacy verbose= param
self._verbose = verbose
# Keep self.verbose as alias for backward compat
self.verbose = self._verbose
logging.debug(f"Verbose mode: {self._verbose}")
def _create_llm_instance(self):
"""Create and return a configured LLM instance for manager tasks."""
@@ -95,7 +116,7 @@ class Process:
agent=loop_task.agent,
name=f"{loop_task.name}_{task_count}" if loop_task.name else task_desc,
expected_output=getattr(loop_task, 'expected_output', None),
callback=loop_task.callback, # Inherit callback from parent loop task
on_task_complete=loop_task.callback, # Inherit callback from parent loop task
is_start=(task_count == 1),
task_type="task"
)
@@ -121,7 +142,7 @@ class Process:
agent=loop_task.agent,
name=f"{loop_task.name}_{i+1}" if loop_task.name else line.strip(),
expected_output=getattr(loop_task, 'expected_output', None),
callback=loop_task.callback, # Inherit callback from parent loop task
on_task_complete=loop_task.callback, # Inherit callback from parent loop task
is_start=(i == 0),
task_type="task"
)
@@ -915,7 +936,7 @@ Provide a JSON with the structure:
agent=start_task.agent,
name=f"{start_task.name}_{task_count}" if start_task.name else task_desc,
expected_output=getattr(start_task, 'expected_output', None),
callback=start_task.callback, # Inherit callback from parent loop task
on_task_complete=start_task.callback, # Inherit callback from parent loop task
is_start=(task_count == 1),
task_type="decision", # Change to decision type
next_tasks=inherited_next_tasks, # Inherit parent's next tasks
@@ -949,7 +970,7 @@ Provide a JSON with the structure:
agent=start_task.agent,
name=f"{start_task.name}_{i+1}" if start_task.name else line.strip(),
expected_output=getattr(start_task, 'expected_output', None),
callback=start_task.callback, # Inherit callback from parent loop task
on_task_complete=start_task.callback, # Inherit callback from parent loop task
is_start=(i == 0),
task_type="task",
condition={
@@ -1033,7 +1054,7 @@ Tasks by type:
agent=current_task.agent,
name=f"{current_task.name}_{i+1}" if current_task.name else task_desc,
expected_output=getattr(current_task, 'expected_output', None),
callback=current_task.callback, # Inherit callback from parent loop task
on_task_complete=current_task.callback, # Inherit callback from parent loop task
is_start=(i == 0),
task_type="task",
condition={
@@ -1058,7 +1079,7 @@ Tasks by type:
agent=current_task.agent,
name=f"{current_task.name}_{i+1}" if current_task.name else line.strip(),
expected_output=getattr(current_task, 'expected_output', None),
callback=current_task.callback, # Inherit callback from parent loop task
on_task_complete=current_task.callback, # Inherit callback from parent loop task
is_start=(i == 0),
task_type="task",
condition={
@@ -92,7 +92,7 @@ class RetrievalConfig:
vector_store_provider: str = "chroma"
vector_store_config: Dict[str, Any] = field(default_factory=dict)
collection_name: Optional[str] = None
persist_path: str = ".praison"
persist_path: str = ".praisonai"
# Embedding configuration
embedder_provider: Optional[str] = None
@@ -233,7 +233,7 @@ class RetrievalConfig:
vector_store_provider=data.get("vector_store_provider", "chroma"),
vector_store_config=data.get("vector_store_config", {}),
collection_name=data.get("collection_name"),
persist_path=data.get("persist_path", ".praison"),
persist_path=data.get("persist_path", ".praisonai"),
embedder_provider=data.get("embedder_provider"),
embedder_model=data.get("embedder_model"),
auto_keywords=auto_keywords,
@@ -99,7 +99,7 @@ class HierarchicalSummarizer:
def __init__(
self,
llm=None,
persist_path: str = ".praison/summaries",
persist_path: str = ".praisonai/summaries",
verbose: bool = False,
):
"""
@@ -83,7 +83,7 @@ class Session:
default_memory_config = {
"provider": "rag",
"use_embedding": False, # Disable embeddings to avoid OpenAI API key requirement
"rag_db_path": f".praison/sessions/{self.session_id}/chroma_db"
"rag_db_path": f".praisonai/sessions/{self.session_id}/chroma_db"
}
if memory_config:
default_memory_config.update(memory_config)
@@ -94,7 +94,7 @@ class Session:
self.knowledge_config = default_knowledge_config
# Create session directory
os.makedirs(f".praison/sessions/{self.session_id}", exist_ok=True)
os.makedirs(f".praisonai/sessions/{self.session_id}", exist_ok=True)
# Initialize components lazily
self._memory = None
@@ -22,7 +22,7 @@ Usage:
)
agent.start("What did I say before?") # Remembers!
Default storage: ~/.praison/sessions/{session_id}.json
Default storage: ~/.praisonai/sessions/{session_id}.json
"""
from typing import TYPE_CHECKING
@@ -86,7 +86,7 @@ class Session:
default_memory_config = {
"provider": "rag",
"use_embedding": False,
"rag_db_path": f".praison/sessions/{self.session_id}/chroma_db"
"rag_db_path": f".praisonai/sessions/{self.session_id}/chroma_db"
}
if memory_config:
default_memory_config.update(memory_config)
@@ -95,7 +95,7 @@ class Session:
default_knowledge_config = knowledge_config or {}
self.knowledge_config = default_knowledge_config
os.makedirs(f".praison/sessions/{self.session_id}", exist_ok=True)
os.makedirs(f".praisonai/sessions/{self.session_id}", exist_ok=True)
self._memory = None
self._knowledge = None
@@ -213,7 +213,7 @@ class DefaultSessionStore:
Initialize session store.
Args:
session_dir: Directory for session files. Defaults to ~/.praison/sessions/
session_dir: Directory for session files. Defaults to ~/.praisonai/sessions/
max_messages: Maximum messages to keep per session.
lock_timeout: Timeout for file lock acquisition.
"""
@@ -5,14 +5,16 @@ from typing import List, Optional
from .parser import find_skill_md, read_properties
from .models import SkillProperties
from ..paths import get_skills_dir, get_project_data_dir
def get_default_skill_dirs() -> List[Path]:
"""Get default skill directory locations.
Uses centralized paths.py for consistent path management.
Returns directories in precedence order (high to low):
1. Project: ./.praison/skills/ or ./.claude/skills/
2. User: ~/.praison/skills/
1. Project: ./.praisonai/skills/ or ./.claude/skills/
2. User: ~/.praisonai/skills/
3. System: /etc/praison/skills/ (admin-managed)
Returns:
@@ -21,19 +23,21 @@ def get_default_skill_dirs() -> List[Path]:
dirs = []
cwd = Path.cwd()
# Project-level directories
project_dirs = [
cwd / ".praison" / "skills",
cwd / ".claude" / "skills",
]
for d in project_dirs:
if d.exists() and d.is_dir():
dirs.append(d)
# Project-level directories (use centralized path)
project_data_dir = get_project_data_dir()
project_skills = project_data_dir / "skills"
if project_skills.exists() and project_skills.is_dir():
dirs.append(project_skills)
# Also check .claude/skills for compatibility
claude_skills = cwd / ".claude" / "skills"
if claude_skills.exists() and claude_skills.is_dir():
dirs.append(claude_skills)
# User-level directory
user_dir = Path.home() / ".praison" / "skills"
if user_dir.exists() and user_dir.is_dir():
dirs.append(user_dir)
# User-level directory (use centralized path)
user_skills = get_skills_dir()
if user_skills.exists() and user_skills.is_dir():
dirs.append(user_skills)
# System-level directory (Unix-like systems)
system_dir = Path("/etc/praison/skills")
@@ -19,7 +19,7 @@ class SkillManager:
manager = SkillManager()
# Discover skills from directories
manager.discover(["./skills", "~/.praison/skills"])
manager.discover(["./skills", "~/.praisonai/skills"])
# Get XML for system prompt
prompt_xml = manager.to_prompt()
@@ -167,7 +167,7 @@ class SQLiteBackend:
Example:
```python
backend = SQLiteBackend(db_path="~/.praison/data.db")
backend = SQLiteBackend(db_path="~/.praisonai/data.db")
backend.save("session_123", {"messages": []})
data = backend.load("session_123")
```
@@ -175,7 +175,7 @@ class SQLiteBackend:
def __init__(
self,
db_path: str = "~/.praison/storage.db",
db_path: str = "~/.praisonai/storage.db",
table_name: str = "praison_storage",
auto_create: bool = True,
):
@@ -473,10 +473,10 @@ def get_backend(
Example:
```python
# File backend (default)
backend = get_backend("file", storage_dir="~/.praison/data")
backend = get_backend("file", storage_dir="~/.praisonai/data")
# SQLite backend
backend = get_backend("sqlite", db_path="~/.praison/data.db")
backend = get_backend("sqlite", db_path="~/.praisonai/data.db")
# Redis backend
backend = get_backend("redis", url="redis://localhost:6379")
@@ -43,6 +43,10 @@ class StreamEvent:
tool_call: Tool call data for DELTA_TOOL_CALL events
metadata: Additional event-specific metadata
error: Error message for ERROR events
is_reasoning: Whether this content is reasoning/thinking (default False)
agent_id: Optional agent identifier for multi-agent scenarios
session_id: Optional session identifier for tracking
run_id: Optional run identifier for correlation
"""
type: StreamEventType
timestamp: float = field(default_factory=time.perf_counter)
@@ -50,6 +54,10 @@ class StreamEvent:
tool_call: Optional[Dict[str, Any]] = None
metadata: Optional[Dict[str, Any]] = None
error: Optional[str] = None
is_reasoning: bool = False
agent_id: Optional[str] = None
session_id: Optional[str] = None
run_id: Optional[str] = None
def __repr__(self) -> str:
parts = [f"StreamEvent({self.type.value}"]
@@ -54,6 +54,7 @@ class Task:
output_json: Optional[Type[BaseModel]] = None,
output_pydantic: Optional[Type[BaseModel]] = None,
callback: Optional[Union[Callable[[TaskOutput], Any], Callable[[TaskOutput], Coroutine[Any, Any, Any]]]] = None,
on_task_complete: Optional[Union[Callable[[TaskOutput], Any], Callable[[TaskOutput], Coroutine[Any, Any, Any]]]] = None,
status: str = "not started",
result: Optional[TaskOutput] = None,
create_directory: Optional[bool] = False,
@@ -158,7 +159,24 @@ class Task:
self.output_file = output_file
self.output_json = output_json
self.output_pydantic = output_pydantic
self.callback = callback
# Handle callback/on_task_complete: on_task_complete is canonical, callback is deprecated
if callback is not None and on_task_complete is None:
import warnings
warnings.warn(
"Parameter 'callback' is deprecated, use 'on_task_complete' instead. "
"Example: Task(on_task_complete=my_fn) instead of Task(callback=my_fn)",
DeprecationWarning,
stacklevel=2
)
self.callback = callback
elif callback is not None and on_task_complete is not None:
raise ValueError(
"Cannot specify both 'callback' and 'on_task_complete'. "
"Use 'on_task_complete' only (callback is deprecated)."
)
else:
# on_task_complete takes precedence (or both are None)
self.callback = on_task_complete
self.status = status
self.result = result
self.create_directory = create_directory
@@ -4,7 +4,7 @@ Mentions Parser for PraisonAI Agents.
Provides @mention syntax support similar to Cursor IDE:
- @file:path/to/file.py - Include file content
- @web:query - Search the web
- @doc:name - Include doc from .praison/docs/
- @doc:name - Include doc from .praisonai/docs/
- @rule:name - Include specific rule
Usage:
@@ -30,7 +30,7 @@ class MentionsParser:
Supports:
- @file:path - Include file content
- @web:query - Web search (requires web search tool)
- @doc:name - Include doc from .praison/docs/
- @doc:name - Include doc from .praisonai/docs/
- @rule:name - Include specific rule
- @url:https://... - Fetch URL content
"""
@@ -213,8 +213,8 @@ class ToolRegistry:
"""Discover and load tools from single-file plugins.
Scans default plugin directories for WordPress-style plugins:
- ./.praison/plugins/ (project-level)
- ~/.praison/plugins/ (user-level)
- ./.praisonai/plugins/ (project-level)
- ~/.praisonai/plugins/ (user-level)
Returns:
Number of plugins loaded
@@ -333,8 +333,8 @@ def discover_plugins() -> int:
"""Discover and load tools from single-file plugins.
Scans default plugin directories for WordPress-style plugins:
- ./.praison/plugins/ (project-level)
- ~/.praison/plugins/ (user-level)
- ./.praisonai/plugins/ (project-level)
- ~/.praisonai/plugins/ (user-level)
Returns:
Number of plugins loaded
@@ -6,14 +6,14 @@ allowing users to define and execute complex task sequences.
Features:
- YAML/Markdown workflow definitions
- Auto-discovery from .praison/workflows/
- Auto-discovery from .praisonai/workflows/
- Step-by-step execution with context passing
- Variable substitution
- Conditional steps
- Lazy loading for performance
Storage Structure:
.praison/workflows/
.praisonai/workflows/
├── deploy.md # Deployment workflow
├── test.md # Testing workflow
└── review.md # Code review workflow
@@ -2904,7 +2904,7 @@ class WorkflowManager:
Workflows are defined in markdown files with YAML frontmatter
and step definitions.
Example workflow file (.praison/workflows/deploy.md):
Example workflow file (.praisonai/workflows/deploy.md):
```markdown
---
name: Deploy to Production
@@ -2955,7 +2955,7 @@ class WorkflowManager:
```
"""
WORKFLOWS_DIR = ".praison/workflows"
WORKFLOWS_DIR = ".praisonai/workflows"
SUPPORTED_EXTENSIONS = [".md", ".yaml", ".yml"]
def __init__(
@@ -4246,7 +4246,7 @@ class WorkflowManager:
def _get_checkpoints_dir(self) -> Path:
"""Get the checkpoints directory path."""
checkpoints_dir = self.workspace_path / ".praison" / "checkpoints"
checkpoints_dir = self.workspace_path / ".praisonai" / "checkpoints"
checkpoints_dir.mkdir(parents=True, exist_ok=True)
return checkpoints_dir
+1 -1
View File
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
[project]
name = "praisonaiagents"
version = "1.3.0"
version = "1.4.2"
description = "Praison AI agents for completing complex tasks with Self Reflection Agents"
readme = "README.md"
requires-python = ">=3.10"
@@ -1,6 +1,7 @@
from praisonaiagents import (
Agent,
Task,
AgentTeam,
Agents,
error_logs,
register_display_callback,
@@ -147,7 +148,7 @@ def main():
markdown=True
)
# Create tasks with callbacks
# Create tasks with on_task_complete callbacks
task1 = Task(
name="research_task",
description="""Analyze 2024's AI advancements.
@@ -155,7 +156,7 @@ def main():
expected_output="""A detailed report on 2024 AI advancements""",
agent=researcher,
tools=[internet_search_tool],
callback=task_callback
on_task_complete=task_callback
)
task2 = Task(
@@ -166,7 +167,7 @@ def main():
expected_output="A blog post of at least 4 paragraphs",
agent=writer,
context=[task1],
callback=task_callback,
on_task_complete=task_callback,
tools=[]
)
@@ -175,7 +176,7 @@ def main():
description="""Create a json object with a title of "My Task" and content of "My content".""",
expected_output="""JSON output with title and content""",
agent=researcher,
callback=task_callback
on_task_complete=task_callback
)
task4 = Task(
@@ -186,7 +187,7 @@ def main():
context=[task2],
output_file='test.txt',
create_directory=True,
callback=task_callback
on_task_complete=task_callback
)
# Create and run agents manager
@@ -0,0 +1,275 @@
"""Integration tests for EvaluationLoop with mocked LLM."""
import pytest
from unittest.mock import MagicMock, patch
class TestEvaluationLoopIntegration:
"""Integration tests for EvaluationLoop."""
def test_evaluation_loop_with_agent_and_judge(self):
"""Test full integration with Agent and Judge (mocked LLM)."""
from praisonaiagents import Agent
from praisonaiagents.eval import EvaluationLoop, EvaluationLoopResult
with patch('praisonaiagents.agent.agent.Agent.chat') as mock_chat, \
patch('litellm.completion') as mock_completion:
mock_chat.side_effect = [
"First attempt at analysis",
"Improved analysis with more detail",
"Final comprehensive analysis with all requirements",
]
mock_response = MagicMock()
mock_response.choices = [MagicMock()]
mock_response.choices[0].message.content = """
SCORE: 9
REASONING: Excellent comprehensive analysis
SUGGESTIONS: None needed
"""
mock_completion.return_value = mock_response
agent = Agent(name="test_agent", instructions="Analyze systems")
loop = EvaluationLoop(
agent=agent,
criteria="Analysis is thorough",
threshold=8.0,
max_iterations=3,
)
result = loop.run("Analyze the auth flow")
assert isinstance(result, EvaluationLoopResult)
assert result.final_score == 9.0
assert result.success is True
def test_agent_run_until_method(self):
"""Test Agent.run_until() convenience method."""
from praisonaiagents import Agent
from praisonaiagents.eval.results import EvaluationLoopResult
with patch('praisonaiagents.agent.agent.Agent.chat') as mock_chat, \
patch('litellm.completion') as mock_completion:
mock_chat.return_value = "Good analysis output"
mock_response = MagicMock()
mock_response.choices = [MagicMock()]
mock_response.choices[0].message.content = """
SCORE: 8.5
REASONING: Good quality analysis
SUGGESTIONS: Consider edge cases
"""
mock_completion.return_value = mock_response
agent = Agent(name="test_agent", instructions="Analyze systems")
result = agent.run_until(
"Analyze the codebase",
criteria="Analysis is thorough",
threshold=8.0,
)
assert isinstance(result, EvaluationLoopResult)
assert result.success is True
assert result.final_score == 8.5
def test_evaluation_loop_iteration_callback(self):
"""Test that iteration callback is called correctly."""
from praisonaiagents import Agent
from praisonaiagents.eval import EvaluationLoop
with patch('praisonaiagents.agent.agent.Agent.chat') as mock_chat, \
patch('litellm.completion') as mock_completion:
mock_chat.return_value = "Output"
mock_response = MagicMock()
mock_response.choices = [MagicMock()]
mock_response.choices[0].message.content = "SCORE: 9\nREASONING: Good"
mock_completion.return_value = mock_response
callback_results = []
def on_iteration(result):
callback_results.append(result)
agent = Agent(name="test", instructions="Test")
loop = EvaluationLoop(
agent=agent,
criteria="Be good",
threshold=8.0,
on_iteration=on_iteration,
)
loop.run("Test prompt")
assert len(callback_results) == 1
assert callback_results[0].score == 9.0
def test_evaluation_loop_multiple_iterations(self):
"""Test loop runs multiple iterations when score is below threshold."""
from praisonaiagents import Agent
from praisonaiagents.eval import EvaluationLoop
with patch('praisonaiagents.agent.agent.Agent.chat') as mock_chat, \
patch('litellm.completion') as mock_completion:
mock_chat.side_effect = ["Output 1", "Output 2", "Output 3"]
responses = [
"SCORE: 5\nREASONING: Needs work\nSUGGESTIONS: Improve detail",
"SCORE: 7\nREASONING: Better\nSUGGESTIONS: Add examples",
"SCORE: 9\nREASONING: Excellent\nSUGGESTIONS: None",
]
mock_response = MagicMock()
mock_response.choices = [MagicMock()]
mock_response.choices[0].message.content = responses[0]
call_count = [0]
def side_effect(*args, **kwargs):
idx = min(call_count[0], len(responses) - 1)
mock_response.choices[0].message.content = responses[idx]
call_count[0] += 1
return mock_response
mock_completion.side_effect = side_effect
agent = Agent(name="test", instructions="Test")
loop = EvaluationLoop(
agent=agent,
criteria="Be thorough",
threshold=8.0,
max_iterations=5,
)
result = loop.run("Test prompt")
assert result.success is True
assert len(result.iterations) == 3
assert result.score_history == [5.0, 7.0, 9.0]
def test_evaluation_loop_max_iterations_reached(self):
"""Test loop stops at max iterations even if threshold not met."""
from praisonaiagents import Agent
from praisonaiagents.eval import EvaluationLoop
with patch('praisonaiagents.agent.agent.Agent.chat') as mock_chat, \
patch('litellm.completion') as mock_completion:
mock_chat.return_value = "Output"
mock_response = MagicMock()
mock_response.choices = [MagicMock()]
mock_response.choices[0].message.content = "SCORE: 5\nREASONING: Poor"
mock_completion.return_value = mock_response
agent = Agent(name="test", instructions="Test")
loop = EvaluationLoop(
agent=agent,
criteria="Be thorough",
threshold=8.0,
max_iterations=3,
)
result = loop.run("Test prompt")
assert result.success is False
assert len(result.iterations) == 3
assert result.final_score == 5.0
def test_evaluation_loop_review_mode(self):
"""Test review mode runs all iterations regardless of score."""
from praisonaiagents import Agent
from praisonaiagents.eval import EvaluationLoop
with patch('praisonaiagents.agent.agent.Agent.chat') as mock_chat, \
patch('litellm.completion') as mock_completion:
mock_chat.return_value = "Output"
mock_response = MagicMock()
mock_response.choices = [MagicMock()]
mock_response.choices[0].message.content = "SCORE: 10\nREASONING: Perfect"
mock_completion.return_value = mock_response
agent = Agent(name="test", instructions="Test")
loop = EvaluationLoop(
agent=agent,
criteria="Be thorough",
threshold=8.0,
max_iterations=3,
mode="review",
)
result = loop.run("Test prompt")
assert result.success is True
assert len(result.iterations) == 3
class TestEvaluationLoopResultSerialization:
"""Tests for result serialization."""
def test_result_to_json(self):
"""Test EvaluationLoopResult.to_json()."""
from praisonaiagents.eval.results import IterationResult, EvaluationLoopResult
import json
iterations = [
IterationResult(iteration=1, output="Out1", score=7.0, reasoning="OK"),
IterationResult(iteration=2, output="Out2", score=9.0, reasoning="Good"),
]
result = EvaluationLoopResult(
iterations=iterations,
success=True,
total_duration_seconds=5.0,
)
json_str = result.to_json()
parsed = json.loads(json_str)
assert parsed["success"] is True
assert parsed["final_score"] == 9.0
assert parsed["score_history"] == [7.0, 9.0]
assert len(parsed["iterations"]) == 2
def test_result_final_report(self):
"""Test EvaluationLoopResult.final_report property."""
from praisonaiagents.eval.results import IterationResult, EvaluationLoopResult
iterations = [
IterationResult(
iteration=1,
output="First output",
score=6.0,
reasoning="Needs improvement",
findings=["Add more detail"]
),
IterationResult(
iteration=2,
output="Better output",
score=8.5,
reasoning="Good quality",
findings=[]
),
]
result = EvaluationLoopResult(
iterations=iterations,
success=True,
total_duration_seconds=3.5,
threshold=8.0,
)
report = result.final_report
assert "# Evaluation Loop Report" in report
assert "Success" in report or "✅" in report
assert "8.5" in report
assert "Iteration 1" in report
assert "Iteration 2" in report
assert "Add more detail" in report
@@ -0,0 +1,528 @@
"""Unit tests for EvaluationLoop class and related dataclasses."""
import pytest
import time
from unittest.mock import MagicMock, patch, AsyncMock
from datetime import datetime
class TestIterationResult:
"""Tests for IterationResult dataclass."""
def test_iteration_result_creation(self):
"""Test basic IterationResult creation."""
from praisonaiagents.eval.results import IterationResult
result = IterationResult(
iteration=1,
output="Test output",
score=8.5,
reasoning="Good quality",
findings=["Finding 1", "Finding 2"],
)
assert result.iteration == 1
assert result.output == "Test output"
assert result.score == 8.5
assert result.reasoning == "Good quality"
assert result.findings == ["Finding 1", "Finding 2"]
assert result.timestamp is not None
def test_iteration_result_defaults(self):
"""Test IterationResult with default values."""
from praisonaiagents.eval.results import IterationResult
result = IterationResult(
iteration=1,
output="Test",
score=7.0,
reasoning="OK",
)
assert result.findings == []
assert result.metadata == {}
def test_iteration_result_to_dict(self):
"""Test IterationResult to_dict method."""
from praisonaiagents.eval.results import IterationResult
result = IterationResult(
iteration=1,
output="Test",
score=8.0,
reasoning="Good",
findings=["F1"],
)
d = result.to_dict()
assert d["iteration"] == 1
assert d["output"] == "Test"
assert d["score"] == 8.0
assert d["reasoning"] == "Good"
assert d["findings"] == ["F1"]
assert "timestamp" in d
def test_iteration_result_passed_property(self):
"""Test IterationResult passed property."""
from praisonaiagents.eval.results import IterationResult
# Default threshold is 7.0
result_pass = IterationResult(iteration=1, output="", score=8.0, reasoning="")
result_fail = IterationResult(iteration=1, output="", score=5.0, reasoning="")
assert result_pass.passed is True
assert result_fail.passed is False
class TestEvaluationLoopResult:
"""Tests for EvaluationLoopResult dataclass."""
def test_evaluation_loop_result_creation(self):
"""Test basic EvaluationLoopResult creation."""
from praisonaiagents.eval.results import EvaluationLoopResult, IterationResult
iterations = [
IterationResult(iteration=1, output="Out1", score=6.0, reasoning="R1"),
IterationResult(iteration=2, output="Out2", score=7.5, reasoning="R2"),
IterationResult(iteration=3, output="Out3", score=8.5, reasoning="R3"),
]
result = EvaluationLoopResult(
iterations=iterations,
success=True,
total_duration_seconds=5.5,
)
assert len(result.iterations) == 3
assert result.success is True
assert result.total_duration_seconds == 5.5
def test_evaluation_loop_result_final_score(self):
"""Test final_score property returns last iteration score."""
from praisonaiagents.eval.results import EvaluationLoopResult, IterationResult
iterations = [
IterationResult(iteration=1, output="", score=6.0, reasoning=""),
IterationResult(iteration=2, output="", score=8.5, reasoning=""),
]
result = EvaluationLoopResult(iterations=iterations, success=True)
assert result.final_score == 8.5
def test_evaluation_loop_result_score_history(self):
"""Test score_history property."""
from praisonaiagents.eval.results import EvaluationLoopResult, IterationResult
iterations = [
IterationResult(iteration=1, output="", score=5.0, reasoning=""),
IterationResult(iteration=2, output="", score=6.5, reasoning=""),
IterationResult(iteration=3, output="", score=8.0, reasoning=""),
]
result = EvaluationLoopResult(iterations=iterations, success=True)
assert result.score_history == [5.0, 6.5, 8.0]
def test_evaluation_loop_result_findings(self):
"""Test accumulated_findings property."""
from praisonaiagents.eval.results import EvaluationLoopResult, IterationResult
iterations = [
IterationResult(iteration=1, output="", score=5.0, reasoning="", findings=["F1", "F2"]),
IterationResult(iteration=2, output="", score=7.0, reasoning="", findings=["F3"]),
]
result = EvaluationLoopResult(iterations=iterations, success=True)
assert result.accumulated_findings == ["F1", "F2", "F3"]
def test_evaluation_loop_result_final_output(self):
"""Test final_output property."""
from praisonaiagents.eval.results import EvaluationLoopResult, IterationResult
iterations = [
IterationResult(iteration=1, output="First", score=5.0, reasoning=""),
IterationResult(iteration=2, output="Final", score=8.0, reasoning=""),
]
result = EvaluationLoopResult(iterations=iterations, success=True)
assert result.final_output == "Final"
def test_evaluation_loop_result_to_dict(self):
"""Test to_dict method."""
from praisonaiagents.eval.results import EvaluationLoopResult, IterationResult
iterations = [
IterationResult(iteration=1, output="Out", score=8.0, reasoning="Good"),
]
result = EvaluationLoopResult(
iterations=iterations,
success=True,
total_duration_seconds=2.5,
)
d = result.to_dict()
assert d["success"] is True
assert d["final_score"] == 8.0
assert d["score_history"] == [8.0]
assert d["total_duration_seconds"] == 2.5
assert len(d["iterations"]) == 1
def test_evaluation_loop_result_to_json(self):
"""Test to_json method."""
from praisonaiagents.eval.results import EvaluationLoopResult, IterationResult
import json
iterations = [
IterationResult(iteration=1, output="Out", score=8.0, reasoning="Good"),
]
result = EvaluationLoopResult(iterations=iterations, success=True)
json_str = result.to_json()
parsed = json.loads(json_str)
assert parsed["success"] is True
assert parsed["final_score"] == 8.0
def test_evaluation_loop_result_final_report(self):
"""Test final_report property generates markdown."""
from praisonaiagents.eval.results import EvaluationLoopResult, IterationResult
iterations = [
IterationResult(iteration=1, output="Out1", score=6.0, reasoning="Needs work", findings=["F1"]),
IterationResult(iteration=2, output="Out2", score=8.5, reasoning="Good", findings=["F2"]),
]
result = EvaluationLoopResult(
iterations=iterations,
success=True,
total_duration_seconds=3.0,
)
report = result.final_report
assert "# Evaluation Loop Report" in report
assert "8.5" in report # Final score
assert "Success" in report or "✅" in report
assert "Iteration 1" in report
assert "Iteration 2" in report
def test_evaluation_loop_result_empty_iterations(self):
"""Test with empty iterations list."""
from praisonaiagents.eval.results import EvaluationLoopResult
result = EvaluationLoopResult(iterations=[], success=False)
assert result.final_score == 0.0
assert result.score_history == []
assert result.final_output == ""
assert result.accumulated_findings == []
class TestEvaluationLoop:
"""Tests for EvaluationLoop class."""
def test_evaluation_loop_init_defaults(self):
"""Test EvaluationLoop initialization with defaults."""
from praisonaiagents.eval.loop import EvaluationLoop
agent = MagicMock()
loop = EvaluationLoop(
agent=agent,
criteria="Be helpful",
)
assert loop.agent == agent
assert loop.criteria == "Be helpful"
assert loop.threshold == 8.0
assert loop.max_iterations == 5
assert loop.mode == "optimize"
def test_evaluation_loop_init_custom(self):
"""Test EvaluationLoop initialization with custom values."""
from praisonaiagents.eval.loop import EvaluationLoop
agent = MagicMock()
judge = MagicMock()
callback = MagicMock()
loop = EvaluationLoop(
agent=agent,
criteria="Custom criteria",
threshold=9.0,
max_iterations=10,
mode="review",
judge=judge,
on_iteration=callback,
verbose=True,
)
assert loop.threshold == 9.0
assert loop.max_iterations == 10
assert loop.mode == "review"
assert loop.judge == judge
assert loop.on_iteration == callback
assert loop.verbose is True
def test_evaluation_loop_run_single_iteration_passes(self):
"""Test run() when first iteration passes threshold."""
from praisonaiagents.eval.loop import EvaluationLoop
from praisonaiagents.eval.results import EvaluationLoopResult
agent = MagicMock()
agent.chat.return_value = "Good output"
judge = MagicMock()
judge_result = MagicMock()
judge_result.score = 9.0
judge_result.reasoning = "Excellent"
judge_result.suggestions = []
judge.run.return_value = judge_result
loop = EvaluationLoop(
agent=agent,
criteria="Be helpful",
threshold=8.0,
judge=judge,
)
result = loop.run("Test prompt")
assert isinstance(result, EvaluationLoopResult)
assert result.success is True
assert result.final_score == 9.0
assert len(result.iterations) == 1
def test_evaluation_loop_run_multiple_iterations(self):
"""Test run() with multiple iterations before passing."""
from praisonaiagents.eval.loop import EvaluationLoop
agent = MagicMock()
agent.chat.side_effect = ["Output 1", "Output 2", "Output 3"]
judge = MagicMock()
judge_results = [
MagicMock(score=5.0, reasoning="Poor", suggestions=["Improve"]),
MagicMock(score=7.0, reasoning="Better", suggestions=["More"]),
MagicMock(score=8.5, reasoning="Good", suggestions=[]),
]
judge.run.side_effect = judge_results
loop = EvaluationLoop(
agent=agent,
criteria="Be helpful",
threshold=8.0,
max_iterations=5,
judge=judge,
)
result = loop.run("Test prompt")
assert result.success is True
assert result.final_score == 8.5
assert len(result.iterations) == 3
assert result.score_history == [5.0, 7.0, 8.5]
def test_evaluation_loop_run_max_iterations_reached(self):
"""Test run() when max iterations reached without passing."""
from praisonaiagents.eval.loop import EvaluationLoop
agent = MagicMock()
agent.chat.return_value = "Output"
judge = MagicMock()
judge_result = MagicMock(score=5.0, reasoning="Poor", suggestions=["Improve"])
judge.run.return_value = judge_result
loop = EvaluationLoop(
agent=agent,
criteria="Be helpful",
threshold=8.0,
max_iterations=3,
judge=judge,
)
result = loop.run("Test prompt")
assert result.success is False
assert len(result.iterations) == 3
assert result.final_score == 5.0
def test_evaluation_loop_run_review_mode(self):
"""Test run() in review mode (runs all iterations)."""
from praisonaiagents.eval.loop import EvaluationLoop
agent = MagicMock()
agent.chat.return_value = "Output"
judge = MagicMock()
# Even high score shouldn't stop in review mode
judge_result = MagicMock(score=9.0, reasoning="Great", suggestions=[])
judge.run.return_value = judge_result
loop = EvaluationLoop(
agent=agent,
criteria="Be helpful",
threshold=8.0,
max_iterations=3,
mode="review",
judge=judge,
)
result = loop.run("Test prompt")
# Review mode runs all iterations
assert len(result.iterations) == 3
assert result.success is True
def test_evaluation_loop_on_iteration_callback(self):
"""Test on_iteration callback is called."""
from praisonaiagents.eval.loop import EvaluationLoop
agent = MagicMock()
agent.chat.return_value = "Output"
judge = MagicMock()
judge_result = MagicMock(score=9.0, reasoning="Good", suggestions=[])
judge.run.return_value = judge_result
callback_calls = []
def on_iteration(iteration_result):
callback_calls.append(iteration_result)
loop = EvaluationLoop(
agent=agent,
criteria="Be helpful",
threshold=8.0,
judge=judge,
on_iteration=on_iteration,
)
loop.run("Test prompt")
assert len(callback_calls) == 1
assert callback_calls[0].score == 9.0
def test_evaluation_loop_extracts_findings(self):
"""Test that findings are extracted from suggestions."""
from praisonaiagents.eval.loop import EvaluationLoop
agent = MagicMock()
agent.chat.return_value = "Output"
judge = MagicMock()
judge_result = MagicMock(
score=9.0,
reasoning="Good",
suggestions=["Finding 1", "Finding 2"]
)
judge.run.return_value = judge_result
loop = EvaluationLoop(
agent=agent,
criteria="Be helpful",
threshold=8.0,
judge=judge,
)
result = loop.run("Test prompt")
assert result.iterations[0].findings == ["Finding 1", "Finding 2"]
class TestEvaluationLoopAsync:
"""Tests for EvaluationLoop async methods."""
@pytest.mark.asyncio
async def test_evaluation_loop_run_async(self):
"""Test run_async() method."""
from praisonaiagents.eval.loop import EvaluationLoop
from praisonaiagents.eval.results import EvaluationLoopResult
agent = MagicMock()
agent.chat_async = AsyncMock(return_value="Output")
judge = MagicMock()
judge_result = MagicMock(score=9.0, reasoning="Good", suggestions=[])
judge.run_async = AsyncMock(return_value=judge_result)
loop = EvaluationLoop(
agent=agent,
criteria="Be helpful",
threshold=8.0,
judge=judge,
)
result = await loop.run_async("Test prompt")
assert isinstance(result, EvaluationLoopResult)
assert result.success is True
assert result.final_score == 9.0
class TestAgentRunUntil:
"""Tests for Agent.run_until() method."""
def test_agent_run_until_exists(self):
"""Test that Agent has run_until method."""
from praisonaiagents import Agent
assert hasattr(Agent, 'run_until')
def test_agent_run_until_basic(self):
"""Test Agent.run_until() basic usage."""
from praisonaiagents import Agent
from praisonaiagents.eval.results import EvaluationLoopResult
with patch('praisonaiagents.eval.loop.EvaluationLoop') as MockLoop:
mock_result = MagicMock(spec=EvaluationLoopResult)
mock_result.success = True
mock_result.final_score = 8.5
mock_loop_instance = MagicMock()
mock_loop_instance.run.return_value = mock_result
MockLoop.return_value = mock_loop_instance
agent = Agent(name="test", instructions="Be helpful")
result = agent.run_until(
"Test prompt",
criteria="Be helpful",
threshold=8.0,
)
assert result.success is True
MockLoop.assert_called_once()
class TestEvaluationLoopProtocol:
"""Tests for EvaluationLoopProtocol."""
def test_protocol_exists(self):
"""Test that EvaluationLoopProtocol exists."""
from praisonaiagents.eval.protocols import EvaluationLoopProtocol
assert EvaluationLoopProtocol is not None
def test_evaluation_loop_implements_protocol(self):
"""Test that EvaluationLoop implements the protocol."""
from praisonaiagents.eval.loop import EvaluationLoop
from praisonaiagents.eval.protocols import EvaluationLoopProtocol
# Check that EvaluationLoop has required methods
assert hasattr(EvaluationLoop, 'run')
assert hasattr(EvaluationLoop, 'run_async')
class TestLazyLoading:
"""Tests for lazy loading of EvaluationLoop."""
def test_lazy_import_from_eval(self):
"""Test that EvaluationLoop can be imported from eval module."""
from praisonaiagents.eval import EvaluationLoop
assert EvaluationLoop is not None
def test_lazy_import_results(self):
"""Test that result classes can be imported from eval module."""
from praisonaiagents.eval import IterationResult, EvaluationLoopResult
assert IterationResult is not None
assert EvaluationLoopResult is not None
@@ -0,0 +1,229 @@
"""
TDD Tests for Learn Context Injection.
Tests that when memory="learn" is used, the learn context is automatically
injected into the system prompt without requiring a new Agent parameter.
Key behaviors:
1. memory="learn" preset enables LearnManager
2. Learn context is auto-injected into system prompt
3. No new Agent params needed (uses existing memory= param)
4. Zero overhead when learn is not enabled
"""
import pytest # noqa: F401 - used for test markers
class TestLearnPresetResolution:
"""Test that memory='learn' resolves correctly."""
def test_learn_preset_exists_in_memory_presets(self):
"""MEMORY_PRESETS should have 'learn' preset with learn=True."""
from praisonaiagents.config.presets import MEMORY_PRESETS
assert "learn" in MEMORY_PRESETS
assert MEMORY_PRESETS["learn"]["learn"] is True
assert MEMORY_PRESETS["learn"]["backend"] == "file"
def test_memory_learn_string_resolves_to_config_with_learn_true(self):
"""memory='learn' should resolve to MemoryConfig with learn=True."""
from praisonaiagents.config.param_resolver import resolve
from praisonaiagents.config.feature_configs import MemoryConfig
from praisonaiagents.config.presets import MEMORY_PRESETS, MEMORY_URL_SCHEMES
result = resolve(
value="learn",
param_name="memory",
config_class=MemoryConfig,
presets=MEMORY_PRESETS,
url_schemes=MEMORY_URL_SCHEMES,
)
assert isinstance(result, MemoryConfig)
assert result.learn is True
class TestAgentLearnContextInjection:
"""Test that Agent injects learn context into system prompt."""
def test_agent_with_memory_learn_has_learn_manager(self):
"""Agent with memory='learn' should have access to LearnManager."""
from praisonaiagents import Agent
# Create agent with memory="learn"
agent = Agent(
name="test_agent",
instructions="Test instructions",
memory="learn",
)
# Memory instance should exist
assert agent._memory_instance is not None
# Memory should have learn property that returns LearnManager
if hasattr(agent._memory_instance, 'learn'):
learn_manager = agent._memory_instance.learn
assert learn_manager is not None
def test_get_learn_context_returns_empty_when_no_learnings(self):
"""get_learn_context should return empty string when no learnings exist."""
from praisonaiagents import Agent
import uuid
# Use unique user_id to ensure clean state
unique_user_id = f"test_user_{uuid.uuid4().hex[:8]}"
agent = Agent(
name="test_agent",
instructions="Test instructions",
memory={"backend": "file", "learn": True, "user_id": unique_user_id},
)
# Should have get_learn_context method
assert hasattr(agent, 'get_learn_context')
# Should return empty string when no learnings (fresh user)
context = agent.get_learn_context()
assert context == "" or context is None or (isinstance(context, str) and len(context) == 0)
def test_get_learn_context_returns_learnings_when_present(self):
"""get_learn_context should return formatted learnings when present."""
from praisonaiagents import Agent
agent = Agent(
name="test_agent",
instructions="Test instructions",
memory="learn",
)
# Add some learnings via the memory's learn manager
if agent._memory_instance and hasattr(agent._memory_instance, 'learn'):
learn_manager = agent._memory_instance.learn
if learn_manager:
learn_manager.capture_persona("User prefers concise responses")
learn_manager.capture_insight("User works in data science")
# get_learn_context should now return formatted context
context = agent.get_learn_context()
# Should contain the captured learnings
if context:
assert "concise" in context.lower() or "User Preferences" in context
def test_build_system_prompt_includes_learn_context(self):
"""_build_system_prompt should include learn context when memory='learn'."""
from praisonaiagents import Agent
agent = Agent(
name="test_agent",
role="Test Role",
goal="Test Goal",
backstory="Test Backstory",
instructions="Test instructions",
memory="learn",
)
# Add some learnings
if agent._memory_instance and hasattr(agent._memory_instance, 'learn'):
learn_manager = agent._memory_instance.learn
if learn_manager:
learn_manager.capture_persona("User prefers detailed explanations")
# Build system prompt
system_prompt = agent._build_system_prompt()
# System prompt should include learn context section
# Note: This test will fail until we implement the fix
if agent._memory_instance and hasattr(agent._memory_instance, 'learn'):
learn_manager = agent._memory_instance.learn
if learn_manager and learn_manager.get_persona_context():
assert "Learned" in system_prompt or "User Preferences" in system_prompt or "detailed explanations" in system_prompt.lower()
class TestLearnContextZeroOverhead:
"""Test that learn context has zero overhead when not enabled."""
def test_no_learn_context_when_memory_true(self):
"""memory=True should NOT enable learn context."""
from praisonaiagents import Agent
agent = Agent(
name="test_agent",
instructions="Test instructions",
memory=True,
)
# get_learn_context should return empty or None
context = agent.get_learn_context()
assert context == "" or context is None
def test_no_learn_context_when_memory_false(self):
"""memory=False should NOT enable learn context."""
from praisonaiagents import Agent
agent = Agent(
name="test_agent",
instructions="Test instructions",
memory=False,
)
# get_learn_context should return empty or None
context = agent.get_learn_context()
assert context == "" or context is None
def test_no_learn_context_when_memory_none(self):
"""memory=None should NOT enable learn context."""
from praisonaiagents import Agent
agent = Agent(
name="test_agent",
instructions="Test instructions",
)
# get_learn_context should return empty or None
context = agent.get_learn_context()
assert context == "" or context is None
class TestMemoryLearnProperty:
"""Test Memory.learn property and get_learn_context method."""
def test_memory_learn_property_returns_learn_manager_when_enabled(self):
"""Memory.learn should return LearnManager when learn=True in config."""
from praisonaiagents.memory.memory import Memory
memory = Memory({"learn": True})
# learn property should return LearnManager
learn_manager = memory.learn
assert learn_manager is not None
# Should have capture methods
assert hasattr(learn_manager, 'capture_persona')
assert hasattr(learn_manager, 'capture_insight')
assert hasattr(learn_manager, 'to_system_prompt_context')
def test_memory_learn_property_returns_none_when_disabled(self):
"""Memory.learn should return None when learn is not enabled."""
from praisonaiagents.memory.memory import Memory
memory = Memory({}) # No learn config
# learn property should return None
assert memory.learn is None
def test_memory_get_learn_context_returns_formatted_string(self):
"""Memory.get_learn_context should return formatted context string."""
from praisonaiagents.memory.memory import Memory
memory = Memory({"learn": True})
# Add some learnings
memory.learn.capture_persona("User prefers Python")
memory.learn.capture_insight("User is a developer")
# get_learn_context should return formatted string
context = memory.get_learn_context()
assert isinstance(context, str)
assert "Python" in context or "User Preferences" in context
@@ -0,0 +1,283 @@
"""
Unit tests for plugins.enable() API and config-driven defaults.
Tests:
- plugins.enable() / plugins.disable()
- PRAISONAI_PLUGINS env var
- Config file loading
- Config-driven defaults for Agent parameters
"""
import os
import pytest
import tempfile
from pathlib import Path
from unittest.mock import patch
class TestPluginsEnableAPI:
"""Test the plugins.enable() API."""
def test_enable_all_plugins(self):
"""Test enabling all plugins."""
from praisonaiagents import plugins
# Reset state
plugins._plugins_enabled = False
plugins._enabled_plugin_names = None
# Enable all
plugins.enable()
assert plugins.is_enabled() is True
assert plugins._enabled_plugin_names is None # None means all
def test_enable_specific_plugins(self):
"""Test enabling specific plugins by name."""
from praisonaiagents import plugins
# Reset state
plugins._plugins_enabled = False
plugins._enabled_plugin_names = None
# Enable specific plugins
plugins.enable(["logging", "metrics"])
assert plugins.is_enabled() is True
assert plugins._enabled_plugin_names == ["logging", "metrics"]
def test_disable_all_plugins(self):
"""Test disabling all plugins."""
from praisonaiagents import plugins
# First enable
plugins._plugins_enabled = True
plugins._enabled_plugin_names = None
# Then disable
plugins.disable()
assert plugins.is_enabled() is False
def test_list_plugins_returns_list(self):
"""Test that list_plugins returns a list."""
from praisonaiagents import plugins
result = plugins.list_plugins()
assert isinstance(result, list)
def test_is_enabled_default_false(self):
"""Test that plugins are disabled by default."""
from praisonaiagents import plugins
# Reset state
plugins._plugins_enabled = False
assert plugins.is_enabled() is False
class TestPluginsEnvVar:
"""Test PRAISONAI_PLUGINS environment variable."""
def test_env_var_true_enables_plugins(self):
"""Test that PRAISONAI_PLUGINS=true enables plugins."""
from praisonaiagents._config import _get_plugins_enabled
with patch.dict(os.environ, {"PRAISONAI_PLUGINS": "true"}):
assert _get_plugins_enabled() is True
def test_env_var_false_disables_plugins(self):
"""Test that PRAISONAI_PLUGINS=false disables plugins."""
from praisonaiagents._config import _get_plugins_enabled
with patch.dict(os.environ, {"PRAISONAI_PLUGINS": "false"}):
assert _get_plugins_enabled() is False
def test_env_var_list_enables_specific(self):
"""Test that PRAISONAI_PLUGINS=logging,metrics enables specific plugins."""
from praisonaiagents._config import _get_plugins_enabled, _get_plugins_list
with patch.dict(os.environ, {"PRAISONAI_PLUGINS": "logging,metrics"}):
assert _get_plugins_enabled() is True
assert _get_plugins_list() == ["logging", "metrics"]
def test_env_var_empty_disables(self):
"""Test that empty PRAISONAI_PLUGINS disables plugins."""
from praisonaiagents._config import _get_plugins_enabled
with patch.dict(os.environ, {"PRAISONAI_PLUGINS": ""}):
assert _get_plugins_enabled() is False
class TestConfigLoader:
"""Test config file loading."""
def test_get_config_returns_praison_config(self):
"""Test that get_config returns a PraisonConfig."""
from praisonaiagents.config.loader import get_config, PraisonConfig, clear_config_cache
clear_config_cache()
config = get_config()
assert isinstance(config, PraisonConfig)
def test_get_plugins_config(self):
"""Test getting plugins config."""
from praisonaiagents.config.loader import get_plugins_config, PluginsConfig, clear_config_cache
clear_config_cache()
plugins_config = get_plugins_config()
assert isinstance(plugins_config, PluginsConfig)
def test_get_defaults_config(self):
"""Test getting defaults config."""
from praisonaiagents.config.loader import get_defaults_config, DefaultsConfig, clear_config_cache
clear_config_cache()
defaults_config = get_defaults_config()
assert isinstance(defaults_config, DefaultsConfig)
def test_get_default_with_fallback(self):
"""Test get_default returns fallback when key not found."""
from praisonaiagents.config.loader import get_default, clear_config_cache
clear_config_cache()
result = get_default("nonexistent_key", "fallback_value")
assert result == "fallback_value"
def test_is_plugins_enabled_from_config(self):
"""Test is_plugins_enabled checks config file."""
from praisonaiagents.config.loader import is_plugins_enabled, clear_config_cache
clear_config_cache()
# Without env var, should check config (which defaults to False)
with patch.dict(os.environ, {}, clear=True):
# Remove PRAISONAI_PLUGINS if present
os.environ.pop("PRAISONAI_PLUGINS", None)
result = is_plugins_enabled()
assert isinstance(result, bool)
def test_config_file_parsing(self):
"""Test parsing a TOML config file."""
from praisonaiagents.config.loader import _parse_toml, clear_config_cache
# Create a temp TOML file
with tempfile.NamedTemporaryFile(mode='w', suffix='.toml', delete=False) as f:
f.write("""
[plugins]
enabled = true
auto_discover = true
[defaults]
model = "gpt-4o-mini"
[defaults.memory]
enabled = true
backend = "file"
""")
f.flush()
try:
result = _parse_toml(Path(f.name))
assert result.get("plugins", {}).get("enabled") is True
assert result.get("defaults", {}).get("model") == "gpt-4o-mini"
assert result.get("defaults", {}).get("memory", {}).get("backend") == "file"
finally:
os.unlink(f.name)
class TestConfigDrivenDefaults:
"""Test config-driven defaults for Agent parameters."""
def test_apply_config_defaults_with_true(self):
"""Test that True uses config defaults."""
from praisonaiagents.config.loader import apply_config_defaults, clear_config_cache
from praisonaiagents.config.feature_configs import MemoryConfig
clear_config_cache()
# When user passes True, should use config defaults
# Without config file, should just return True
result = apply_config_defaults("memory", True, MemoryConfig)
# Should return True or a MemoryConfig
assert result is True or isinstance(result, MemoryConfig)
def test_apply_config_defaults_with_none(self):
"""Test that None checks config for enabled."""
from praisonaiagents.config.loader import apply_config_defaults, clear_config_cache
clear_config_cache()
# When user passes None, should check config
result = apply_config_defaults("memory", None, None)
# Without config, should return None
assert result is None
def test_apply_config_defaults_explicit_value(self):
"""Test that explicit values are preserved."""
from praisonaiagents.config.loader import apply_config_defaults, clear_config_cache
from praisonaiagents.config.feature_configs import MemoryConfig
clear_config_cache()
# When user passes explicit config, should preserve it
explicit_config = MemoryConfig(backend="postgres")
result = apply_config_defaults("memory", explicit_config, MemoryConfig)
assert result == explicit_config
class TestToolsGuardrailsExempt:
"""Test that tools and guardrails work without plugins.enable()."""
def test_tools_work_without_enable(self):
"""Test that tools work without calling plugins.enable()."""
from praisonaiagents import plugins
# Ensure plugins are disabled
plugins._plugins_enabled = False
# Tools should still work - this is tested by Agent creation
# Just verify the flag is false
assert plugins.is_enabled() is False
def test_guardrails_work_without_enable(self):
"""Test that guardrails work without calling plugins.enable()."""
from praisonaiagents import plugins
# Ensure plugins are disabled
plugins._plugins_enabled = False
# Guardrails should still work - this is tested by Agent creation
# Just verify the flag is false
assert plugins.is_enabled() is False
class TestPluginManagerIntegration:
"""Test PluginManager integration with enable API."""
def test_get_plugin_manager_singleton(self):
"""Test that get_plugin_manager returns singleton."""
from praisonaiagents.plugins import get_plugin_manager
manager1 = get_plugin_manager()
manager2 = get_plugin_manager()
assert manager1 is manager2
def test_plugin_manager_has_required_methods(self):
"""Test that PluginManager has all required methods."""
from praisonaiagents.plugins import get_plugin_manager
manager = get_plugin_manager()
# Check required methods exist
assert hasattr(manager, 'register')
assert hasattr(manager, 'unregister')
assert hasattr(manager, 'enable')
assert hasattr(manager, 'disable')
assert hasattr(manager, 'is_enabled')
assert hasattr(manager, 'list_plugins')
assert hasattr(manager, 'auto_discover_plugins')
assert hasattr(manager, 'execute_hook')
if __name__ == "__main__":
pytest.main([__file__, "-v"])
@@ -0,0 +1,257 @@
"""
Tests for Unified Plugin System.
TDD tests for:
1. Path centralization (using paths.py instead of hardcoded paths)
2. Hook unification (PluginHook = HookEvent)
3. PluginType enum
4. Async hook execution
5. Thread safety
"""
import pytest
import threading
import asyncio
from pathlib import Path
from unittest.mock import patch, MagicMock
class TestPathCentralization:
"""Tests for centralized path usage in discovery modules."""
def test_plugins_discovery_uses_paths_module(self):
"""plugins/discovery.py should use paths.get_plugins_dir()."""
from praisonaiagents.plugins.discovery import get_default_plugin_dirs
from praisonaiagents.paths import get_plugins_dir, get_project_data_dir
# The function should use paths.py functions internally
# We verify by checking the returned paths match the expected format
dirs = get_default_plugin_dirs()
# Should return paths that end with 'plugins'
for d in dirs:
assert d.name == "plugins", f"Expected 'plugins' dir, got {d}"
def test_skills_discovery_uses_paths_module(self):
"""skills/discovery.py should use paths.get_skills_dir()."""
from praisonaiagents.skills.discovery import get_default_skill_dirs
from praisonaiagents.paths import get_skills_dir
dirs = get_default_skill_dirs()
# Should return paths that end with 'skills'
for d in dirs:
assert d.name == "skills", f"Expected 'skills' dir, got {d}"
def test_paths_module_returns_praisonai_dir(self):
"""paths.py should return ~/.praisonai/ as default."""
from praisonaiagents.paths import get_data_dir, DEFAULT_DIR_NAME
assert DEFAULT_DIR_NAME == ".praisonai"
# Without env override, should use .praisonai
data_dir = get_data_dir()
assert data_dir.name in [".praisonai", ".praison"] # .praison for legacy
class TestHookUnification:
"""Tests for PluginHook = HookEvent unification."""
def test_plugin_hook_is_hook_event_alias(self):
"""PluginHook should be an alias for HookEvent."""
from praisonaiagents.plugins import PluginHook
from praisonaiagents.hooks import HookEvent
# They should be the same enum
assert PluginHook is HookEvent, "PluginHook should be HookEvent alias"
def test_hook_event_has_plugin_lifecycle_events(self):
"""HookEvent should have ON_INIT, ON_SHUTDOWN, etc."""
from praisonaiagents.hooks import HookEvent
# Plugin lifecycle events
assert hasattr(HookEvent, 'ON_INIT'), "Missing ON_INIT"
assert hasattr(HookEvent, 'ON_SHUTDOWN'), "Missing ON_SHUTDOWN"
# Permission/config events
assert hasattr(HookEvent, 'ON_PERMISSION_ASK'), "Missing ON_PERMISSION_ASK"
assert hasattr(HookEvent, 'ON_CONFIG'), "Missing ON_CONFIG"
assert hasattr(HookEvent, 'ON_AUTH'), "Missing ON_AUTH"
# Message events
assert hasattr(HookEvent, 'BEFORE_MESSAGE'), "Missing BEFORE_MESSAGE"
assert hasattr(HookEvent, 'AFTER_MESSAGE'), "Missing AFTER_MESSAGE"
def test_hook_event_values_match_plugin_hook_values(self):
"""HookEvent values should match expected plugin hook values."""
from praisonaiagents.hooks import HookEvent
assert HookEvent.ON_INIT.value == "on_init"
assert HookEvent.ON_SHUTDOWN.value == "on_shutdown"
assert HookEvent.BEFORE_TOOL.value == "before_tool"
assert HookEvent.AFTER_TOOL.value == "after_tool"
class TestPluginType:
"""Tests for PluginType enum."""
def test_plugin_type_enum_exists(self):
"""PluginType enum should exist."""
from praisonaiagents.plugins import PluginType
assert PluginType is not None
def test_plugin_type_has_expected_values(self):
"""PluginType should have TOOL, HOOK, SKILL, POLICY."""
from praisonaiagents.plugins import PluginType
assert hasattr(PluginType, 'TOOL')
assert hasattr(PluginType, 'HOOK')
assert hasattr(PluginType, 'SKILL')
assert hasattr(PluginType, 'POLICY')
assert PluginType.TOOL.value == "tool"
assert PluginType.HOOK.value == "hook"
assert PluginType.SKILL.value == "skill"
assert PluginType.POLICY.value == "policy"
class TestAsyncHookExecution:
"""Tests for async hook execution."""
@pytest.mark.asyncio
async def test_async_execute_hook_exists(self):
"""PluginManager should have async_execute_hook method."""
from praisonaiagents.plugins import PluginManager
manager = PluginManager()
assert hasattr(manager, 'async_execute_hook'), "Missing async_execute_hook"
assert asyncio.iscoroutinefunction(manager.async_execute_hook)
@pytest.mark.asyncio
async def test_async_execute_hook_works(self):
"""async_execute_hook should execute hooks asynchronously."""
from praisonaiagents.plugins import PluginManager, Plugin, PluginInfo
from praisonaiagents.hooks import HookEvent
class AsyncTestPlugin(Plugin):
@property
def info(self):
return PluginInfo(
name="async_test",
hooks=[HookEvent.BEFORE_TOOL]
)
def before_tool(self, tool_name, args):
args["async_modified"] = True
return args
manager = PluginManager()
manager.register(AsyncTestPlugin())
result = await manager.async_execute_hook(
HookEvent.BEFORE_TOOL,
"test_tool",
{"original": True}
)
assert result.get("async_modified") is True
class TestThreadSafety:
"""Tests for thread safety in PluginManager."""
def test_plugin_manager_has_lock(self):
"""PluginManager should have a threading lock."""
from praisonaiagents.plugins import PluginManager
manager = PluginManager()
assert hasattr(manager, '_lock'), "Missing _lock attribute"
assert isinstance(manager._lock, type(threading.RLock()))
def test_concurrent_registration_is_safe(self):
"""Concurrent plugin registration should be thread-safe."""
from praisonaiagents.plugins import PluginManager, Plugin, PluginInfo
from praisonaiagents.hooks import HookEvent
class TestPlugin(Plugin):
def __init__(self, name):
self._name = name
@property
def info(self):
return PluginInfo(name=self._name, hooks=[])
manager = PluginManager()
errors = []
def register_plugin(i):
try:
manager.register(TestPlugin(f"plugin_{i}"))
except Exception as e:
errors.append(e)
threads = [threading.Thread(target=register_plugin, args=(i,)) for i in range(10)]
for t in threads:
t.start()
for t in threads:
t.join()
assert len(errors) == 0, f"Thread safety errors: {errors}"
assert len(manager.list_plugins()) == 10
class TestBackwardCompatibility:
"""Tests for backward compatibility."""
def test_legacy_plugin_hook_import_works(self):
"""Importing PluginHook from plugins should still work."""
from praisonaiagents.plugins import PluginHook
# Should be able to use PluginHook values
assert PluginHook.BEFORE_TOOL.value == "before_tool"
def test_legacy_paths_fallback(self):
"""Legacy ~/.praison/ path should work with warning."""
from praisonaiagents.paths import get_data_dir, LEGACY_DIR_NAME
assert LEGACY_DIR_NAME == ".praison"
def test_plugin_info_still_works(self):
"""PluginInfo should still work with PluginHook."""
from praisonaiagents.plugins import PluginInfo, PluginHook
info = PluginInfo(
name="test",
hooks=[PluginHook.BEFORE_TOOL, PluginHook.AFTER_TOOL]
)
assert len(info.hooks) == 2
class TestExports:
"""Tests for proper exports in __init__.py files."""
def test_plugins_exports_plugin_type(self):
"""plugins/__init__.py should export PluginType."""
from praisonaiagents.plugins import PluginType
assert PluginType is not None
def test_plugins_exports_plugin_hook(self):
"""plugins/__init__.py should export PluginHook."""
from praisonaiagents.plugins import PluginHook
assert PluginHook is not None
def test_hooks_exports_all_events(self):
"""hooks/__init__.py should export HookEvent with all values."""
from praisonaiagents.hooks import HookEvent
# Core events
assert hasattr(HookEvent, 'BEFORE_TOOL')
assert hasattr(HookEvent, 'AFTER_TOOL')
assert hasattr(HookEvent, 'BEFORE_AGENT')
assert hasattr(HookEvent, 'AFTER_AGENT')
# Plugin lifecycle events (new)
assert hasattr(HookEvent, 'ON_INIT')
assert hasattr(HookEvent, 'ON_SHUTDOWN')
@@ -0,0 +1,156 @@
"""
TDD Tests for verbose= to output= migration in Process class.
These tests verify:
1. Process class accepts output= parameter
2. Process class still accepts verbose= for backward compat (with deprecation)
3. output= takes precedence over verbose=
"""
import pytest
from unittest.mock import MagicMock
class TestProcessOutputMigration:
"""Test Process class migration from verbose= to output=."""
def test_process_accepts_output_parameter(self):
"""Process should accept output= parameter."""
from praisonaiagents.process.process import Process
from praisonaiagents.agent.agent import Agent
from praisonaiagents.task.task import Task
# Create minimal mocks
agent = MagicMock(spec=Agent)
agent.name = "test_agent"
task = MagicMock(spec=Task)
task.id = "task1"
task.name = "test_task"
# This should work with output= parameter
process = Process(
tasks={"task1": task},
agents=[agent],
output="verbose"
)
# Verify verbose was set from output preset
assert process._verbose
def test_process_output_silent_sets_verbose_false(self):
"""output='silent' should set verbose=False."""
from praisonaiagents.process.process import Process
from praisonaiagents.agent.agent import Agent
from praisonaiagents.task.task import Task
agent = MagicMock(spec=Agent)
agent.name = "test_agent"
task = MagicMock(spec=Task)
task.id = "task1"
task.name = "test_task"
process = Process(
tasks={"task1": task},
agents=[agent],
output="silent"
)
assert not process._verbose
def test_process_backward_compat_verbose_true(self):
"""Process should still accept verbose=True for backward compat."""
from praisonaiagents.process.process import Process
from praisonaiagents.agent.agent import Agent
from praisonaiagents.task.task import Task
agent = MagicMock(spec=Agent)
agent.name = "test_agent"
task = MagicMock(spec=Task)
task.id = "task1"
task.name = "test_task"
# Old API should still work
process = Process(
tasks={"task1": task},
agents=[agent],
verbose=True
)
assert process._verbose
def test_process_output_takes_precedence_over_verbose(self):
"""output= should take precedence over verbose= if both provided."""
from praisonaiagents.process.process import Process
from praisonaiagents.agent.agent import Agent
from praisonaiagents.task.task import Task
agent = MagicMock(spec=Agent)
agent.name = "test_agent"
task = MagicMock(spec=Task)
task.id = "task1"
task.name = "test_task"
# output= should win
process = Process(
tasks={"task1": task},
agents=[agent],
output="silent",
verbose=True # This should be ignored
)
assert not process._verbose
class TestAgentVerboseRejection:
"""Test that Agent class rejects verbose= parameter."""
def test_agent_rejects_verbose_parameter(self):
"""Agent(verbose=True) should raise TypeError."""
from praisonaiagents import Agent
with pytest.raises(TypeError) as exc_info:
Agent(instructions="Test", verbose=True)
assert "verbose" in str(exc_info.value).lower()
def test_agent_accepts_output_verbose(self):
"""Agent(output='verbose') should work."""
from praisonaiagents import Agent
agent = Agent(instructions="Test", output="verbose")
assert agent.verbose
assert agent.markdown
class TestAgentTeamOutputParameter:
"""Test that AgentTeam uses output= not verbose=."""
def test_agent_team_accepts_output_parameter(self):
"""AgentTeam should accept output= parameter."""
from praisonaiagents import Agent, AgentTeam, Task
agent = Agent(instructions="Test agent")
task = Task(description="Test task", agent=agent)
team = AgentTeam(
agents=[agent],
tasks=[task],
output="verbose"
)
assert team.verbose
def test_agent_team_output_silent(self):
"""AgentTeam with output='silent' should have verbose=0 or False."""
from praisonaiagents import Agent, AgentTeam, Task
agent = Agent(instructions="Test agent")
task = Task(description="Test task", agent=agent)
team = AgentTeam(
agents=[agent],
tasks=[task],
output="silent"
)
assert not team.verbose
@@ -0,0 +1,295 @@
"""
Tests for Task.on_task_complete parameter (renamed from callback).
TDD: These tests are written FIRST before implementation.
The rename from `callback` to `on_task_complete` aligns with:
- WorkflowHooksConfig.on_step_complete
- WorkflowHooksConfig.on_workflow_complete
- MultiAgentHooksConfig.on_task_complete
This ensures consistent naming across the SDK.
"""
import warnings
import pytest
import asyncio
class TestOnTaskCompleteParameter:
"""Test that on_task_complete parameter works correctly."""
def test_on_task_complete_parameter_accepted(self):
"""Task should accept on_task_complete parameter."""
from praisonaiagents import Task
def my_callback(output):
pass
task = Task(
description="Test task",
on_task_complete=my_callback
)
# The callback should be stored (internally as self.callback for minimal diff)
assert task.callback is my_callback
def test_on_task_complete_default_none(self):
"""on_task_complete should default to None."""
from praisonaiagents import Task
task = Task(description="Test task")
assert task.callback is None
def test_on_task_complete_with_sync_function(self):
"""on_task_complete should work with sync functions."""
from praisonaiagents import Task
results = []
def sync_callback(output):
results.append(output)
task = Task(
description="Test task",
on_task_complete=sync_callback
)
assert task.callback is sync_callback
assert callable(task.callback)
def test_on_task_complete_with_async_function(self):
"""on_task_complete should work with async functions."""
from praisonaiagents import Task
async def async_callback(output):
pass
task = Task(
description="Test task",
on_task_complete=async_callback
)
assert task.callback is async_callback
assert asyncio.iscoroutinefunction(task.callback)
class TestCallbackDeprecation:
"""Test that callback parameter is deprecated but still works."""
def test_callback_emits_deprecation_warning(self):
"""Using callback= should emit DeprecationWarning."""
from praisonaiagents import Task
def my_callback(output):
pass
with warnings.catch_warnings(record=True) as w:
warnings.simplefilter("always")
task = Task(
description="Test task",
callback=my_callback
)
# Should have exactly one deprecation warning about callback
deprecation_warnings = [
x for x in w
if issubclass(x.category, DeprecationWarning)
and 'callback' in str(x.message).lower()
]
assert len(deprecation_warnings) == 1
assert 'on_task_complete' in str(deprecation_warnings[0].message)
def test_callback_still_works(self):
"""callback= should still work (backward compatibility)."""
from praisonaiagents import Task
def my_callback(output):
pass
with warnings.catch_warnings():
warnings.simplefilter("ignore")
task = Task(
description="Test task",
callback=my_callback
)
# Should still be stored
assert task.callback is my_callback
def test_on_task_complete_no_deprecation_warning(self):
"""Using on_task_complete= should NOT emit deprecation warning."""
from praisonaiagents import Task
def my_callback(output):
pass
with warnings.catch_warnings(record=True) as w:
warnings.simplefilter("always")
task = Task(
description="Test task",
on_task_complete=my_callback
)
# Should have no deprecation warnings about callback
deprecation_warnings = [
x for x in w
if issubclass(x.category, DeprecationWarning)
and 'callback' in str(x.message).lower()
]
assert len(deprecation_warnings) == 0
class TestBothParametersProvided:
"""Test behavior when both callback and on_task_complete are provided."""
def test_both_parameters_raises_error(self):
"""Providing both callback and on_task_complete should raise ValueError."""
from praisonaiagents import Task
def callback1(output):
pass
def callback2(output):
pass
with warnings.catch_warnings():
warnings.simplefilter("ignore")
with pytest.raises(ValueError) as exc_info:
Task(
description="Test task",
callback=callback1,
on_task_complete=callback2
)
assert "callback" in str(exc_info.value).lower()
assert "on_task_complete" in str(exc_info.value).lower()
class TestCallbackExecution:
"""Test that callbacks are executed correctly."""
def test_sync_callback_receives_task_output(self):
"""Sync callback should receive TaskOutput."""
from praisonaiagents import Task
from praisonaiagents.main import TaskOutput
received = []
def my_callback(output):
received.append(output)
task = Task(
description="Test task",
on_task_complete=my_callback
)
# Create a mock TaskOutput
task_output = TaskOutput(
description="Test",
raw="Test output",
agent="TestAgent"
)
# Execute callback synchronously
task.execute_callback_sync(task_output)
# Give async task time to complete
import time
time.sleep(0.1)
assert len(received) == 1
assert received[0].raw == "Test output"
@pytest.mark.asyncio
async def test_async_callback_receives_task_output(self):
"""Async callback should receive TaskOutput."""
from praisonaiagents import Task
from praisonaiagents.main import TaskOutput
received = []
async def my_callback(output):
received.append(output)
task = Task(
description="Test task",
on_task_complete=my_callback
)
# Create a mock TaskOutput
task_output = TaskOutput(
description="Test",
raw="Test output",
agent="TestAgent"
)
# Execute callback
await task.execute_callback(task_output)
assert len(received) == 1
assert received[0].raw == "Test output"
class TestNamingConsistency:
"""Test that naming is consistent with other hooks/callbacks in the SDK."""
def test_matches_workflow_hooks_config_pattern(self):
"""on_task_complete follows the on_<entity>_complete pattern."""
from praisonaiagents.workflows.workflow_configs import WorkflowHooksConfig
# WorkflowHooksConfig has on_step_complete, on_workflow_complete
config = WorkflowHooksConfig()
assert hasattr(config, 'on_step_complete')
assert hasattr(config, 'on_workflow_complete')
def test_matches_multi_agent_hooks_config_pattern(self):
"""on_task_complete matches MultiAgentHooksConfig.on_task_complete."""
from praisonaiagents.config.feature_configs import MultiAgentHooksConfig
# MultiAgentHooksConfig already has on_task_complete
config = MultiAgentHooksConfig()
assert hasattr(config, 'on_task_complete')
class TestCallbackWithMetadata:
"""Test callback with metadata support (2-parameter callbacks)."""
@pytest.mark.asyncio
async def test_two_param_callback_receives_metadata(self):
"""Two-parameter callback should receive TaskOutput and metadata."""
from praisonaiagents import Task
from praisonaiagents.main import TaskOutput
received_output = []
received_metadata = []
async def my_callback(output, metadata):
received_output.append(output)
received_metadata.append(metadata)
task = Task(
name="test_task",
description="Test task",
on_task_complete=my_callback
)
task_output = TaskOutput(
description="Test",
raw="Test output",
agent="TestAgent"
)
await task.execute_callback(task_output)
assert len(received_output) == 1
assert len(received_metadata) == 1
assert 'task_id' in received_metadata[0]
assert 'task_name' in received_metadata[0]
assert received_metadata[0]['task_name'] == "test_task"
@@ -0,0 +1,205 @@
"""
Unit tests for history injection via memory presets.
History is enabled via:
- memory="history" preset (10 messages)
- memory="chat" preset (20 messages)
- memory=MemoryConfig(history=True, history_limit=N)
The history= parameter and HistoryConfig class have been removed
to keep the API simple and consistent with other memory presets.
"""
import pytest
from unittest.mock import Mock
class TestDefaultHistoryDisabled:
"""Test that history is disabled by default."""
def test_default_history_disabled(self):
"""Agent() without memory should have history disabled."""
from praisonaiagents import Agent
agent = Agent(
name="test",
instructions="Test agent"
)
assert agent._history_enabled is False
def test_memory_true_does_not_enable_history(self):
"""Agent(memory=True) should NOT enable history by default."""
from praisonaiagents import Agent
agent = Agent(
name="test",
instructions="Test agent",
memory=True
)
# memory=True enables memory backend, not history injection
assert agent._history_enabled is False
class TestMemoryHistoryPreset:
"""Test memory='history' preset."""
def test_memory_history_preset_exists(self):
"""memory='history' should be a valid preset."""
from praisonaiagents.config.presets import MEMORY_PRESETS
assert "history" in MEMORY_PRESETS
def test_memory_history_enables_history(self):
"""Agent(memory='history') should enable history injection."""
from praisonaiagents import Agent
agent = Agent(
name="test",
instructions="Test agent",
memory="history"
)
# Memory should be enabled
assert agent.memory is not None or agent._memory_instance is not None
# History should be enabled
assert agent._history_enabled is True
def test_memory_session_preset_exists(self):
"""memory='session' should be a valid preset (alias for history)."""
from praisonaiagents.config.presets import MEMORY_PRESETS
assert "session" in MEMORY_PRESETS
def test_memory_chat_preset_exists(self):
"""memory='chat' should be a valid preset for conversational use."""
from praisonaiagents.config.presets import MEMORY_PRESETS
assert "chat" in MEMORY_PRESETS
class TestMemoryConfigHistoryFields:
"""Test new history fields in MemoryConfig."""
def test_memory_config_has_history_field(self):
"""MemoryConfig should have a history field."""
from praisonaiagents.config import MemoryConfig
config = MemoryConfig(history=True)
assert config.history is True
def test_memory_config_has_history_limit_field(self):
"""MemoryConfig should have a history_limit field."""
from praisonaiagents.config import MemoryConfig
config = MemoryConfig(history=True, history_limit=20)
assert config.history_limit == 20
def test_memory_config_history_defaults(self):
"""MemoryConfig history fields should have sensible defaults."""
from praisonaiagents.config import MemoryConfig
config = MemoryConfig()
assert config.history is False # Disabled by default
assert config.history_limit == 10 # Default limit
def test_agent_memory_config_with_history(self):
"""Agent with MemoryConfig(history=True) should enable history."""
from praisonaiagents import Agent
from praisonaiagents.config import MemoryConfig
agent = Agent(
name="test",
instructions="Test agent",
memory=MemoryConfig(history=True, history_limit=15)
)
assert agent._history_enabled is True
assert agent._history_limit == 15
class TestHistoryInjection:
"""Test that history is actually injected into messages."""
def test_history_injected_into_messages(self):
"""When memory='history', session history should be in messages."""
from praisonaiagents import Agent
# Create agent with history enabled via preset
agent = Agent(
name="test",
instructions="Test agent",
memory="history"
)
# Set _using_custom_llm to bypass OpenAI client initialization
agent._using_custom_llm = True
# Mock session store with some history
mock_store = Mock()
mock_store.get_chat_history.return_value = [
{"role": "user", "content": "Previous question"},
{"role": "assistant", "content": "Previous answer"}
]
agent._session_store = mock_store
agent._history_session_id = "test-session"
# Build messages
messages, _ = agent._build_messages("New question")
# Should contain the injected history
message_contents = [m.get("content", "") for m in messages]
assert "Previous question" in str(message_contents)
assert "Previous answer" in str(message_contents)
def test_history_respects_limit(self):
"""History injection should respect the limit from MemoryConfig."""
from praisonaiagents import Agent
from praisonaiagents.config import MemoryConfig
agent = Agent(
name="test",
instructions="Test agent",
memory=MemoryConfig(history=True, history_limit=2)
)
# Set _using_custom_llm to bypass OpenAI client initialization
agent._using_custom_llm = True
# Mock session store
mock_store = Mock()
mock_store.get_chat_history.return_value = [
{"role": "user", "content": "Msg 1"},
{"role": "assistant", "content": "Msg 2"}
]
agent._session_store = mock_store
agent._history_session_id = "test-session"
# Build messages
agent._build_messages("New question")
# Should have called with limit=2
mock_store.get_chat_history.assert_called_with("test-session", max_messages=2)
class TestHistoryWithSession:
"""Test history integration with session management."""
def test_history_uses_auto_save_session(self):
"""If auto_save is set, history should use that session."""
from praisonaiagents import Agent
agent = Agent(
name="test",
instructions="Test agent",
auto_save="my-auto-session",
memory="history"
)
# Should use auto_save session for history
assert agent._history_session_id == "my-auto-session"
if __name__ == "__main__":
pytest.main([__file__, "-v"])
@@ -0,0 +1,332 @@
"""
Tests for StreamEvent dataclass and streaming events.
TDD: These tests are written FIRST to define expected behavior.
"""
import time
class TestStreamEventDataclass:
"""Tests for StreamEvent dataclass fields and defaults."""
def test_stream_event_has_is_reasoning_field(self):
"""StreamEvent should have is_reasoning field."""
from praisonaiagents.streaming.events import StreamEvent, StreamEventType
event = StreamEvent(type=StreamEventType.DELTA_TEXT)
assert hasattr(event, 'is_reasoning'), "StreamEvent missing 'is_reasoning' field"
def test_stream_event_is_reasoning_defaults_to_false(self):
"""is_reasoning should default to False for backward compatibility."""
from praisonaiagents.streaming.events import StreamEvent, StreamEventType
event = StreamEvent(type=StreamEventType.DELTA_TEXT, content="Hello")
assert event.is_reasoning is False, "is_reasoning should default to False"
def test_stream_event_is_reasoning_can_be_set_true(self):
"""is_reasoning can be explicitly set to True."""
from praisonaiagents.streaming.events import StreamEvent, StreamEventType
event = StreamEvent(
type=StreamEventType.DELTA_TEXT,
content="Thinking...",
is_reasoning=True
)
assert event.is_reasoning is True
def test_stream_event_has_agent_id_field(self):
"""StreamEvent should have optional agent_id field."""
from praisonaiagents.streaming.events import StreamEvent, StreamEventType
event = StreamEvent(type=StreamEventType.DELTA_TEXT)
assert hasattr(event, 'agent_id'), "StreamEvent missing 'agent_id' field"
def test_stream_event_agent_id_defaults_to_none(self):
"""agent_id should default to None."""
from praisonaiagents.streaming.events import StreamEvent, StreamEventType
event = StreamEvent(type=StreamEventType.DELTA_TEXT)
assert event.agent_id is None, "agent_id should default to None"
def test_stream_event_agent_id_can_be_set(self):
"""agent_id can be explicitly set."""
from praisonaiagents.streaming.events import StreamEvent, StreamEventType
event = StreamEvent(
type=StreamEventType.DELTA_TEXT,
content="Hello",
agent_id="agent-123"
)
assert event.agent_id == "agent-123"
def test_stream_event_has_session_id_field(self):
"""StreamEvent should have optional session_id field."""
from praisonaiagents.streaming.events import StreamEvent, StreamEventType
event = StreamEvent(type=StreamEventType.DELTA_TEXT)
assert hasattr(event, 'session_id'), "StreamEvent missing 'session_id' field"
def test_stream_event_session_id_defaults_to_none(self):
"""session_id should default to None."""
from praisonaiagents.streaming.events import StreamEvent, StreamEventType
event = StreamEvent(type=StreamEventType.DELTA_TEXT)
assert event.session_id is None, "session_id should default to None"
def test_stream_event_has_run_id_field(self):
"""StreamEvent should have optional run_id field."""
from praisonaiagents.streaming.events import StreamEvent, StreamEventType
event = StreamEvent(type=StreamEventType.DELTA_TEXT)
assert hasattr(event, 'run_id'), "StreamEvent missing 'run_id' field"
def test_stream_event_run_id_defaults_to_none(self):
"""run_id should default to None."""
from praisonaiagents.streaming.events import StreamEvent, StreamEventType
event = StreamEvent(type=StreamEventType.DELTA_TEXT)
assert event.run_id is None, "run_id should default to None"
class TestStreamEventBackwardCompatibility:
"""Tests to ensure backward compatibility with existing code."""
def test_existing_code_works_without_new_fields(self):
"""Existing code that doesn't use new fields should work unchanged."""
from praisonaiagents.streaming.events import StreamEvent, StreamEventType
# This is how existing code creates events
event = StreamEvent(
type=StreamEventType.DELTA_TEXT,
content="Hello world"
)
assert event.type == StreamEventType.DELTA_TEXT
assert event.content == "Hello world"
assert event.is_reasoning is False # Default
assert event.agent_id is None # Default
def test_stream_event_repr_works_with_new_fields(self):
"""__repr__ should work with new fields."""
from praisonaiagents.streaming.events import StreamEvent, StreamEventType
event = StreamEvent(
type=StreamEventType.DELTA_TEXT,
content="Hello",
is_reasoning=True,
agent_id="agent-1"
)
repr_str = repr(event)
assert "DELTA_TEXT" in repr_str or "delta_text" in repr_str
def test_stream_metrics_update_from_event_works(self):
"""StreamMetrics.update_from_event should work with new fields."""
from praisonaiagents.streaming.events import StreamEvent, StreamEventType, StreamMetrics
metrics = StreamMetrics()
event = StreamEvent(
type=StreamEventType.DELTA_TEXT,
content="Hello",
is_reasoning=True
)
# Should not raise
metrics.update_from_event(event)
assert metrics.token_count == 1
class TestStreamEventEmitterWithNewFields:
"""Tests for StreamEventEmitter with new StreamEvent fields."""
def test_emitter_handles_reasoning_events(self):
"""StreamEventEmitter should handle events with is_reasoning=True."""
from praisonaiagents.streaming.events import StreamEvent, StreamEventType, StreamEventEmitter
emitter = StreamEventEmitter()
received_events = []
def callback(event):
received_events.append(event)
emitter.add_callback(callback)
# Emit a reasoning event
event = StreamEvent(
type=StreamEventType.DELTA_TEXT,
content="Thinking...",
is_reasoning=True
)
emitter.emit(event)
assert len(received_events) == 1
assert received_events[0].is_reasoning is True
def test_emitter_handles_agent_context(self):
"""StreamEventEmitter should handle events with agent context."""
from praisonaiagents.streaming.events import StreamEvent, StreamEventType, StreamEventEmitter
emitter = StreamEventEmitter()
received_events = []
def callback(event):
received_events.append(event)
emitter.add_callback(callback)
# Emit an event with agent context
event = StreamEvent(
type=StreamEventType.DELTA_TEXT,
content="Hello",
agent_id="agent-123",
session_id="session-456",
run_id="run-789"
)
emitter.emit(event)
assert len(received_events) == 1
assert received_events[0].agent_id == "agent-123"
assert received_events[0].session_id == "session-456"
assert received_events[0].run_id == "run-789"
class TestStreamEventReasoningCallback:
"""Tests for handling reasoning content in callbacks."""
def test_callback_can_distinguish_reasoning_content(self):
"""Callback should be able to distinguish reasoning from regular content."""
from praisonaiagents.streaming.events import StreamEvent, StreamEventType
regular_events = []
reasoning_events = []
def callback(event):
if event.type == StreamEventType.DELTA_TEXT:
if event.is_reasoning:
reasoning_events.append(event)
else:
regular_events.append(event)
# Simulate receiving events
events = [
StreamEvent(type=StreamEventType.DELTA_TEXT, content="Hello", is_reasoning=False),
StreamEvent(type=StreamEventType.DELTA_TEXT, content="Thinking...", is_reasoning=True),
StreamEvent(type=StreamEventType.DELTA_TEXT, content="World", is_reasoning=False),
]
for event in events:
callback(event)
assert len(regular_events) == 2
assert len(reasoning_events) == 1
assert reasoning_events[0].content == "Thinking..."
class TestCreateTextPrinterCallbackWithReasoning:
"""Tests for text printer callback with reasoning support."""
def test_create_text_printer_callback_exists(self):
"""create_text_printer_callback should exist."""
from praisonaiagents.streaming.events import create_text_printer_callback
callback = create_text_printer_callback()
assert callable(callback)
class TestStreamEventAllFields:
"""Test all StreamEvent fields together."""
def test_stream_event_all_fields(self):
"""StreamEvent should support all fields together."""
from praisonaiagents.streaming.events import StreamEvent, StreamEventType
event = StreamEvent(
type=StreamEventType.DELTA_TEXT,
timestamp=time.perf_counter(),
content="Hello",
tool_call=None,
metadata={"key": "value"},
error=None,
is_reasoning=True,
agent_id="agent-1",
session_id="session-1",
run_id="run-1"
)
assert event.type == StreamEventType.DELTA_TEXT
assert event.content == "Hello"
assert event.is_reasoning is True
assert event.agent_id == "agent-1"
assert event.session_id == "session-1"
assert event.run_id == "run-1"
assert event.metadata == {"key": "value"}
class TestAgentIdPopulation:
"""Tests for agent_id population in streaming events."""
def test_agent_has_agent_id_property(self):
"""Agent should have agent_id property for streaming context."""
from praisonaiagents import Agent
agent = Agent(name="test-agent")
# Agent should have an agent_id (either explicit or derived from name)
assert hasattr(agent, 'agent_id') or hasattr(agent, 'name')
def test_stream_emitter_can_receive_agent_context(self):
"""StreamEventEmitter should handle events with agent context."""
from praisonaiagents.streaming.events import StreamEvent, StreamEventType, StreamEventEmitter
emitter = StreamEventEmitter()
received = []
emitter.add_callback(lambda e: received.append(e))
event = StreamEvent(
type=StreamEventType.DELTA_TEXT,
content="test",
agent_id="my-agent",
session_id="sess-1",
run_id="run-1"
)
emitter.emit(event)
assert len(received) == 1
assert received[0].agent_id == "my-agent"
class TestLiteLLMReasoningSupport:
"""Tests for LiteLLM reasoning content support."""
def test_llm_class_has_stream_callback_param(self):
"""LLM.get_response should accept stream_callback parameter."""
from praisonaiagents.llm import LLM
import inspect
# Check if get_response method exists and has stream_callback param
sig = inspect.signature(LLM.get_response)
params = list(sig.parameters.keys())
assert 'stream_callback' in params, "LLM.get_response should have stream_callback parameter"
def test_llm_class_has_emit_events_param(self):
"""LLM.get_response should accept emit_events parameter."""
from praisonaiagents.llm import LLM
import inspect
sig = inspect.signature(LLM.get_response)
params = list(sig.parameters.keys())
assert 'emit_events' in params, "LLM.get_response should have emit_events parameter"
class TestGeneratorStreamingPath:
"""Tests for generator-based streaming with callbacks."""
def test_openai_client_generator_has_stream_callback(self):
"""OpenAI client generator should support stream_callback."""
from praisonaiagents.llm.openai_client import OpenAIClient
import inspect
# Check chat_completion_with_tools_stream signature
sig = inspect.signature(OpenAIClient.chat_completion_with_tools_stream)
params = list(sig.parameters.keys())
assert 'stream_callback' in params, "chat_completion_with_tools_stream should have stream_callback"
+1 -1
View File
@@ -2549,7 +2549,7 @@ wheels = [
[[package]]
name = "praisonaiagents"
version = "1.3.0"
version = "1.4.2"
source = { editable = "." }
dependencies = [
{ name = "aiohttp" },
+2 -2
View File
@@ -3,8 +3,8 @@ class Praisonai < Formula
desc "AI tools for various AI applications"
homepage "https://github.com/MervinPraison/PraisonAI"
url "https://github.com/MervinPraison/PraisonAI/archive/refs/tags/v4.3.0.tar.gz"
sha256 `curl -sL https://github.com/MervinPraison/PraisonAI/archive/refs/tags/v4.3.0.tar.gz | shasum -a 256`.split.first
url "https://github.com/MervinPraison/PraisonAI/archive/refs/tags/v4.4.2.tar.gz"
sha256 `curl -sL https://github.com/MervinPraison/PraisonAI/archive/refs/tags/v4.4.2.tar.gz | shasum -a 256`.split.first
license "MIT"
depends_on "[email protected]"
+178 -2
View File
@@ -314,9 +314,22 @@ def train_list(
@app.command("show")
def train_show(
session_id: str = typer.Argument(..., help="Session ID to show"),
iterations_flag: bool = typer.Option(False, "--iterations", "-i", help="Show detailed iteration info"),
json_output: bool = typer.Option(False, "--json", "-j", help="Output as JSON"),
):
"""Show details of a training session."""
"""
Show details of a training session.
Examples:
# Show session summary
praisonai train show train-abc123
# Show with detailed iterations
praisonai train show train-abc123 --iterations
# Output as JSON
praisonai train show train-abc123 --json
"""
from ..output.console import get_output_controller
output = get_output_controller()
@@ -346,11 +359,174 @@ def train_show(
if report:
report.print_summary()
# Show best iteration
best = report.get_best_iteration()
if best:
output.print(f"\n✨ Best Iteration: #{best.iteration_num} (Score: {best.score}/10)")
if iterations:
output.print("\nIterations:")
# Find best score for highlighting
best_score = max(it.score for it in iterations) if iterations else 0
for it in iterations:
output.print(f" [{it.iteration_num}] Score: {it.score}/10 - {it.feedback[:50]}...")
# Highlight best iteration
marker = "★" if it.score == best_score else " "
feedback_preview = it.feedback[:50] + "..." if len(it.feedback) > 50 else it.feedback
output.print(f" {marker} [{it.iteration_num}] Score: {it.score}/10 - {feedback_preview}")
# Show suggestions if --iterations flag
if iterations_flag and it.suggestions:
for suggestion in it.suggestions[:3]: # Show top 3 suggestions
output.print(f" → {suggestion}")
@app.command("apply")
def train_apply(
session_id: str = typer.Argument(..., help="Training session ID to apply"),
agent_file: Optional[str] = typer.Option(
None,
"--agent", "-a",
help="Path to agent YAML file"
),
iteration: Optional[int] = typer.Option(
None,
"--iteration", "-n",
help="Specific iteration number (default: best score)"
),
run_prompt: Optional[str] = typer.Option(
None,
"--run", "-r",
help="Run agent with this prompt after applying training"
),
json_output: bool = typer.Option(False, "--json", "-j", help="Output as JSON"),
):
"""
Apply training to an agent.
Loads training suggestions from a session and applies them to an agent
via hooks. By default, uses the iteration with the best score.
Examples:
# Apply best iteration to default agent
praisonai train apply train-abc123
# Apply specific iteration
praisonai train apply train-abc123 --iteration 2
# Apply to agent from YAML file
praisonai train apply train-abc123 --agent my_agent.yaml
# Apply and run immediately
praisonai train apply train-abc123 --run "Hello, how are you?"
"""
from ..output.console import get_output_controller
output = get_output_controller()
try:
from praisonai.train.agents import apply_training, get_training_profile
from praisonai.train.agents.storage import TrainingStorage
except ImportError as e:
output.print_error(f"Training module not available: {e}")
raise typer.Exit(1)
# Verify session exists
storage = TrainingStorage(session_id=session_id)
if not storage.storage_path.exists():
output.print_error(f"Session not found: {session_id}")
raise typer.Exit(1)
# Get the training profile
profile = get_training_profile(
session_id=session_id,
iteration=iteration,
agent_name="agent",
)
if profile is None:
output.print_error(f"Could not load training profile from session {session_id}")
raise typer.Exit(1)
if json_output or output.is_json_mode:
output.print_json({
"session_id": session_id,
"iteration": profile.iteration_num,
"score": profile.quality_score,
"suggestions": profile.suggestions,
"summary": profile.summary,
})
if not run_prompt:
return
# Show profile info
output.print_panel(
f"Session: {profile.session_id}\n"
f"Iteration: {profile.iteration_num}\n"
f"Score: {profile.quality_score}/10\n"
f"Suggestions: {len(profile.suggestions)}",
title="Training Profile"
)
if profile.suggestions:
output.print("\nSuggestions:")
for i, suggestion in enumerate(profile.suggestions, 1):
output.print(f" {i}. {suggestion}")
# If run_prompt provided, create agent and run
if run_prompt:
# Create or load agent
agent = None
if agent_file:
agent = _load_agent_from_file(agent_file, output)
if agent is None:
raise typer.Exit(1)
else:
try:
from praisonaiagents import Agent
agent = Agent(
instructions="You are a helpful assistant."
)
except ImportError:
output.print_error(
"praisonaiagents not installed",
remediation="pip install praisonaiagents"
)
raise typer.Exit(1)
# Apply training
success = apply_training(agent, profile=profile)
if not success:
output.print_error("Failed to apply training to agent")
raise typer.Exit(1)
output.print_success("Training applied successfully!")
output.print(f"\nRunning agent with prompt: {run_prompt}\n")
# Run the agent
try:
if hasattr(agent, 'chat'):
response = agent.chat(run_prompt)
elif hasattr(agent, 'start'):
response = agent.start(run_prompt)
else:
output.print_error("Agent has no chat or start method")
raise typer.Exit(1)
output.print_panel(str(response), title="Agent Response")
except Exception as e:
output.print_error(f"Agent execution failed: {e}")
raise typer.Exit(1)
else:
output.print_info(
"\nTo apply this training to an agent, use:\n"
f" praisonai train apply {session_id} --run 'Your prompt here'\n"
"\nOr in Python:\n"
f" from praisonai.train.agents import apply_training\n"
f" apply_training(agent, session_id='{session_id}')"
)
def _load_agent_from_file(file_path: str, output) -> Optional[object]:
+1 -1
View File
@@ -57,7 +57,7 @@ class CloudDeployer:
file.write("FROM python:3.11-slim\n")
file.write("WORKDIR /app\n")
file.write("COPY . .\n")
file.write("RUN pip install flask praisonai==4.3.0 gunicorn markdown\n")
file.write("RUN pip install flask praisonai==4.4.2 gunicorn markdown\n")
file.write("EXPOSE 8080\n")
file.write('CMD ["gunicorn", "-b", "0.0.0.0:8080", "api:app"]\n')
@@ -35,8 +35,13 @@ __all__ = [
"TrainingScenario",
"TrainingIteration",
"TrainingReport",
"TrainingProfile",
"TrainingGrader",
"TrainingStorage",
"TrainingHook",
"apply_training",
"remove_training",
"get_training_profile",
]
@@ -58,6 +63,10 @@ def __getattr__(name: str):
from .models import TrainingReport
return TrainingReport
if name == "TrainingProfile":
from .models import TrainingProfile
return TrainingProfile
if name == "TrainingGrader":
from .grader import TrainingGrader
return TrainingGrader
@@ -66,4 +75,20 @@ def __getattr__(name: str):
from .storage import TrainingStorage
return TrainingStorage
if name == "TrainingHook":
from .hook import TrainingHook
return TrainingHook
if name == "apply_training":
from .hook import apply_training
return apply_training
if name == "remove_training":
from .hook import remove_training
return remove_training
if name == "get_training_profile":
from .hook import get_training_profile
return get_training_profile
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
@@ -0,0 +1,320 @@
"""
Training Hook for Agent Runtime.
Provides hook-based injection of training suggestions into agent prompts.
This allows training data to be applied at runtime without modifying the Agent class.
Usage:
from praisonai.train.agents import apply_training, remove_training
from praisonaiagents import Agent
agent = Agent(name="assistant", instructions="Be helpful")
# Apply training from a session (uses best iteration by default)
apply_training(agent, session_id="train-abc123")
# Or apply training from a specific iteration
apply_training(agent, session_id="train-abc123", iteration=2)
# Or apply with a pre-loaded profile
apply_training(agent, profile=my_profile)
# Run the agent - training suggestions are automatically injected
response = agent.start("Hello!")
# Remove training when done
remove_training(agent)
"""
import logging
from typing import Optional, TYPE_CHECKING
if TYPE_CHECKING:
from praisonaiagents import Agent
from .models import TrainingProfile
from .storage import TrainingStorage
logger = logging.getLogger(__name__)
# Attribute name for storing hook ID on agent
_TRAINING_HOOK_ATTR = "_training_hook_id"
class TrainingHook:
"""
Hook that injects training suggestions into agent prompts.
This hook is registered on the BEFORE_AGENT event and modifies
the prompt to include training guidance.
Example:
profile = TrainingProfile(
agent_name="assistant",
suggestions=["Be concise", "Use examples"],
quality_score=8.5,
summary="Focus on clarity",
iteration_num=2,
session_id="train-abc"
)
hook = TrainingHook(profile)
# Hook is called automatically by the agent's hook system
"""
def __init__(self, profile: TrainingProfile):
"""
Initialize the training hook.
Args:
profile: The training profile with suggestions to inject
"""
self.profile = profile
def __call__(self, event):
"""
Process the BEFORE_AGENT event.
Modifies the prompt to include training guidance.
Args:
event: BeforeAgentInput event data
Returns:
HookResult with modified prompt
"""
# Lazy import to avoid circular dependencies
from praisonaiagents.hooks import HookResult
# Build training guidance section
guidance = self._build_guidance()
# Get original prompt
original_prompt = getattr(event, 'prompt', '') or ''
# Append guidance to prompt
modified_prompt = original_prompt + guidance
return HookResult(
decision="allow",
modified_input={"prompt": modified_prompt},
reason=f"Training guidance applied from session {self.profile.session_id}"
)
def _build_guidance(self) -> str:
"""
Build the training guidance text to inject.
Returns:
Formatted guidance string
"""
if not self.profile.suggestions:
return ""
lines = [
"\n\n[TRAINING GUIDANCE]",
f"Based on training session {self.profile.session_id} (iteration {self.profile.iteration_num}, score: {self.profile.quality_score}/10):",
]
for suggestion in self.profile.suggestions:
lines.append(f"- {suggestion}")
if self.profile.summary:
lines.append(f"\nSummary: {self.profile.summary}")
lines.append("\nPlease follow these guidelines in your response.")
return "\n".join(lines)
def apply_training(
agent: "Agent",
session_id: Optional[str] = None,
iteration: Optional[int] = None,
profile: Optional[TrainingProfile] = None,
) -> bool:
"""
Apply training to an agent via BEFORE_AGENT hook.
This function registers a hook that injects training suggestions
into the agent's prompts at runtime.
Args:
agent: The agent to apply training to
session_id: Training session ID to load from
iteration: Specific iteration number to use (default: best score)
profile: Pre-loaded TrainingProfile (alternative to session_id)
Returns:
True if training was applied successfully, False otherwise
Raises:
ValueError: If neither session_id nor profile is provided
Example:
# Apply best iteration from a session
apply_training(agent, session_id="train-abc123")
# Apply specific iteration
apply_training(agent, session_id="train-abc123", iteration=2)
# Apply with pre-loaded profile
apply_training(agent, profile=my_profile)
"""
if profile is None and session_id is None:
raise ValueError("Either session_id or profile must be provided")
# Load profile from session if not provided
if profile is None:
profile = _load_profile_from_session(
session_id=session_id,
iteration=iteration,
agent_name=getattr(agent, 'name', 'agent'),
)
if profile is None:
logger.warning(f"No training data found for session {session_id}")
return False
# Create the hook
hook = TrainingHook(profile)
# Get the hook registry from the agent
try:
# Lazy import
from praisonaiagents.hooks import HookEvent
registry = agent._hook_runner.registry
# Register the hook
hook_id = registry.register_function(
event=HookEvent.BEFORE_AGENT,
func=hook,
name=f"training_hook_{profile.session_id}",
description=f"Training guidance from session {profile.session_id}",
)
# Store hook ID on agent for later removal
setattr(agent, _TRAINING_HOOK_ATTR, hook_id)
logger.info(
f"Applied training to agent '{agent.name}' from session "
f"{profile.session_id} (iteration {profile.iteration_num}, "
f"score: {profile.quality_score}/10)"
)
return True
except Exception as e:
logger.error(f"Failed to apply training: {e}")
return False
def remove_training(agent: "Agent") -> bool:
"""
Remove training hook from an agent.
Args:
agent: The agent to remove training from
Returns:
True if training was removed, False if no training was applied
Example:
remove_training(agent)
"""
hook_id = getattr(agent, _TRAINING_HOOK_ATTR, None)
if hook_id is None:
logger.debug("No training hook found on agent")
return False
try:
registry = agent._hook_runner.registry
registry.unregister(hook_id)
delattr(agent, _TRAINING_HOOK_ATTR)
logger.info(f"Removed training from agent '{agent.name}'")
return True
except Exception as e:
logger.error(f"Failed to remove training: {e}")
return False
def _load_profile_from_session(
session_id: str,
iteration: Optional[int],
agent_name: str,
) -> Optional[TrainingProfile]:
"""
Load a TrainingProfile from a training session.
Args:
session_id: The training session ID
iteration: Specific iteration number, or None for best
agent_name: Name of the agent
Returns:
TrainingProfile or None if not found
"""
try:
storage = TrainingStorage(session_id=session_id)
if not storage.storage_path.exists():
logger.warning(f"Training session not found: {session_id}")
return None
report = storage.load_report()
if report is None:
logger.warning(f"No report found for session: {session_id}")
return None
# Get the iteration
if iteration is not None:
it = report.get_iteration(iteration)
if it is None:
logger.warning(f"Iteration {iteration} not found in session {session_id}")
return None
else:
# Use best iteration
it = report.get_best_iteration()
if it is None:
logger.warning(f"No iterations found in session {session_id}")
return None
# Create profile from iteration
return TrainingProfile.from_iteration(
iteration=it,
agent_name=agent_name,
session_id=session_id,
)
except Exception as e:
logger.error(f"Failed to load training profile: {e}")
return None
def get_training_profile(
session_id: str,
iteration: Optional[int] = None,
agent_name: str = "agent",
) -> Optional[TrainingProfile]:
"""
Get a training profile from a session without applying it.
Useful for inspecting training data before applying.
Args:
session_id: The training session ID
iteration: Specific iteration number, or None for best
agent_name: Name to use for the profile
Returns:
TrainingProfile or None if not found
Example:
profile = get_training_profile("train-abc123")
print(f"Score: {profile.quality_score}")
print(f"Suggestions: {profile.suggestions}")
"""
return _load_profile_from_session(session_id, iteration, agent_name)
@@ -11,6 +11,86 @@ from typing import Any, Dict, List, Optional
import statistics
@dataclass
class TrainingProfile:
"""
Consolidated training profile for runtime application.
Contains the actionable suggestions from a training session that can be
injected into an agent's prompt at runtime via hooks.
Attributes:
agent_name: Name of the agent this profile is for
suggestions: List of improvement suggestions
quality_score: Score from the training iteration
summary: Summary of the training feedback
iteration_num: Which iteration this came from
session_id: The training session ID
created_at: When this profile was created
Example:
profile = TrainingProfile(
agent_name="assistant",
suggestions=["Be concise", "Use examples"],
quality_score=8.5,
summary="Focus on clarity and examples",
iteration_num=2,
session_id="train-abc123"
)
"""
agent_name: str
suggestions: List[str]
quality_score: float
summary: str
iteration_num: int
session_id: str
created_at: str = field(default_factory=lambda: datetime.utcnow().isoformat())
def to_dict(self) -> Dict[str, Any]:
"""Convert to dictionary."""
return asdict(self)
@classmethod
def from_dict(cls, data: Dict[str, Any]) -> "TrainingProfile":
"""Create from dictionary."""
return cls(
agent_name=data["agent_name"],
suggestions=data.get("suggestions", []),
quality_score=data.get("quality_score", 0.0),
summary=data.get("summary", ""),
iteration_num=data.get("iteration_num", 0),
session_id=data.get("session_id", ""),
created_at=data.get("created_at", datetime.utcnow().isoformat()),
)
@classmethod
def from_iteration(
cls,
iteration: "TrainingIteration",
agent_name: str,
session_id: str,
) -> "TrainingProfile":
"""
Create a TrainingProfile from a TrainingIteration.
Args:
iteration: The training iteration to use
agent_name: Name of the agent
session_id: The training session ID
Returns:
TrainingProfile with data from the iteration
"""
return cls(
agent_name=agent_name,
suggestions=iteration.suggestions,
quality_score=iteration.score,
summary=iteration.feedback,
iteration_num=iteration.iteration_num,
session_id=session_id,
)
@dataclass
class TrainingScenario:
"""
@@ -190,6 +270,32 @@ class TrainingReport:
metadata=data.get("metadata", {})
)
def get_best_iteration(self) -> Optional[TrainingIteration]:
"""
Get the iteration with the highest score.
Returns:
TrainingIteration with highest score, or None if no iterations
"""
if not self.iterations:
return None
return max(self.iterations, key=lambda it: it.score)
def get_iteration(self, iteration_num: int) -> Optional[TrainingIteration]:
"""
Get a specific iteration by number.
Args:
iteration_num: The iteration number (1-based)
Returns:
TrainingIteration or None if not found
"""
for it in self.iterations:
if it.iteration_num == iteration_num:
return it
return None
def print_summary(self) -> None:
"""Print a summary of the training results."""
try:
+1 -1
View File
@@ -1 +1 @@
__version__ = "4.3.0"
__version__ = "4.4.2"
+1 -1
View File
@@ -12,7 +12,7 @@ dependencies = [
"rich>=13.7",
"markdown>=3.5",
"pyparsing>=3.0.0",
"praisonaiagents>=1.3.0",
"praisonaiagents>=1.4.2",
"python-dotenv>=0.19.0",
"litellm>=1.81.0",
"PyYAML>=6.0",
+357
View File
@@ -0,0 +1,357 @@
"""
Tests for Training Hook System.
TDD: Tests written first for:
- TrainingProfile dataclass
- get_best_iteration() method
- apply_training() function
- remove_training() function
- TrainingHook class
"""
from unittest.mock import Mock, patch
class TestTrainingProfile:
"""Tests for TrainingProfile dataclass."""
def test_training_profile_creation(self):
"""Test creating a TrainingProfile."""
from praisonai.train.agents.models import TrainingProfile
profile = TrainingProfile(
agent_name="assistant",
suggestions=["Be concise", "Use examples"],
quality_score=8.5,
summary="Focus on clarity",
iteration_num=2,
session_id="train-abc123",
)
assert profile.agent_name == "assistant"
assert profile.suggestions == ["Be concise", "Use examples"]
assert profile.quality_score == 8.5
assert profile.summary == "Focus on clarity"
assert profile.iteration_num == 2
assert profile.session_id == "train-abc123"
def test_training_profile_to_dict(self):
"""Test converting TrainingProfile to dict."""
from praisonai.train.agents.models import TrainingProfile
profile = TrainingProfile(
agent_name="assistant",
suggestions=["Be concise"],
quality_score=8.0,
summary="Test summary",
iteration_num=1,
session_id="train-123",
)
d = profile.to_dict()
assert d["agent_name"] == "assistant"
assert d["suggestions"] == ["Be concise"]
assert d["quality_score"] == 8.0
def test_training_profile_from_dict(self):
"""Test creating TrainingProfile from dict."""
from praisonai.train.agents.models import TrainingProfile
data = {
"agent_name": "researcher",
"suggestions": ["Search first", "Cite sources"],
"quality_score": 9.0,
"summary": "Research focus",
"iteration_num": 3,
"session_id": "train-xyz",
}
profile = TrainingProfile.from_dict(data)
assert profile.agent_name == "researcher"
assert profile.quality_score == 9.0
assert profile.iteration_num == 3
class TestGetBestIteration:
"""Tests for get_best_iteration() method on TrainingReport."""
def test_get_best_iteration_returns_highest_score(self):
"""Test that get_best_iteration returns iteration with highest score."""
from praisonai.train.agents.models import TrainingReport, TrainingIteration
iterations = [
TrainingIteration(
iteration_num=1, scenario_id="s1", input_text="test",
output="out1", score=6.0, feedback="ok"
),
TrainingIteration(
iteration_num=2, scenario_id="s1", input_text="test",
output="out2", score=9.0, feedback="great" # Best
),
TrainingIteration(
iteration_num=3, scenario_id="s1", input_text="test",
output="out3", score=7.5, feedback="good"
),
]
report = TrainingReport(
session_id="train-test",
iterations=iterations,
total_iterations=3,
)
best = report.get_best_iteration()
assert best is not None
assert best.iteration_num == 2
assert best.score == 9.0
def test_get_best_iteration_empty_returns_none(self):
"""Test that get_best_iteration returns None for empty iterations."""
from praisonai.train.agents.models import TrainingReport
report = TrainingReport(
session_id="train-empty",
iterations=[],
total_iterations=0,
)
best = report.get_best_iteration()
assert best is None
def test_get_iteration_by_number(self):
"""Test getting a specific iteration by number."""
from praisonai.train.agents.models import TrainingReport, TrainingIteration
iterations = [
TrainingIteration(
iteration_num=1, scenario_id="s1", input_text="test",
output="out1", score=6.0, feedback="ok"
),
TrainingIteration(
iteration_num=2, scenario_id="s1", input_text="test",
output="out2", score=8.0, feedback="good"
),
]
report = TrainingReport(
session_id="train-test",
iterations=iterations,
total_iterations=2,
)
it = report.get_iteration(2)
assert it is not None
assert it.iteration_num == 2
assert it.score == 8.0
def test_get_iteration_not_found(self):
"""Test getting non-existent iteration returns None."""
from praisonai.train.agents.models import TrainingReport, TrainingIteration
iterations = [
TrainingIteration(
iteration_num=1, scenario_id="s1", input_text="test",
output="out1", score=6.0, feedback="ok"
),
]
report = TrainingReport(
session_id="train-test",
iterations=iterations,
total_iterations=1,
)
it = report.get_iteration(5)
assert it is None
class TestApplyTraining:
"""Tests for apply_training() function."""
def test_apply_training_registers_hook(self):
"""Test that apply_training registers a BEFORE_AGENT hook."""
from praisonai.train.agents.hook import apply_training
from praisonai.train.agents.models import TrainingProfile
# Create mock agent
mock_agent = Mock()
mock_agent.name = "assistant"
mock_registry = Mock()
mock_hook_runner = Mock()
mock_hook_runner.registry = mock_registry
mock_agent._hook_runner = mock_hook_runner
# Create profile
profile = TrainingProfile(
agent_name="assistant",
suggestions=["Be concise"],
quality_score=8.0,
summary="Test",
iteration_num=1,
session_id="train-123",
)
# Apply training
result = apply_training(mock_agent, profile=profile)
assert result is True
mock_registry.register_function.assert_called_once()
def test_apply_training_from_session(self):
"""Test apply_training loads from session ID."""
from praisonai.train.agents.hook import apply_training
mock_agent = Mock()
mock_agent.name = "assistant"
mock_registry = Mock()
mock_hook_runner = Mock()
mock_hook_runner.registry = mock_registry
mock_agent._hook_runner = mock_hook_runner
# Mock storage
with patch('praisonai.train.agents.hook.TrainingStorage') as MockStorage:
mock_storage = Mock()
mock_report = Mock()
mock_iteration = Mock()
mock_iteration.suggestions = ["Be helpful"]
mock_iteration.score = 8.5
mock_iteration.feedback = "Good"
mock_iteration.iteration_num = 2
mock_report.get_best_iteration.return_value = mock_iteration
mock_report.session_id = "train-abc"
mock_storage.load_report.return_value = mock_report
MockStorage.return_value = mock_storage
result = apply_training(mock_agent, session_id="train-abc")
assert result is True
class TestRemoveTraining:
"""Tests for remove_training() function."""
def test_remove_training_unregisters_hook(self):
"""Test that remove_training removes the training hook."""
from praisonai.train.agents.hook import apply_training, remove_training
from praisonai.train.agents.models import TrainingProfile
mock_agent = Mock()
mock_agent.name = "assistant"
mock_registry = Mock()
mock_hook_runner = Mock()
mock_hook_runner.registry = mock_registry
mock_agent._hook_runner = mock_hook_runner
mock_agent._training_hook_id = None
profile = TrainingProfile(
agent_name="assistant",
suggestions=["Be concise"],
quality_score=8.0,
summary="Test",
iteration_num=1,
session_id="train-123",
)
# Apply then remove
apply_training(mock_agent, profile=profile)
result = remove_training(mock_agent)
# Should attempt to unregister
assert result is True or result is False # Depends on implementation
class TestTrainingHook:
"""Tests for TrainingHook class."""
def test_training_hook_modifies_prompt(self):
"""Test that TrainingHook injects suggestions into prompt."""
from praisonai.train.agents.hook import TrainingHook
from praisonai.train.agents.models import TrainingProfile
from praisonaiagents.hooks import BeforeAgentInput
profile = TrainingProfile(
agent_name="assistant",
suggestions=["Be concise", "Use examples"],
quality_score=8.5,
summary="Focus on clarity",
iteration_num=2,
session_id="train-abc",
)
hook = TrainingHook(profile)
# Create input event
event = BeforeAgentInput(
session_id="session-1",
cwd="/tmp",
event_name="before_agent",
timestamp="2024-01-01T00:00:00",
agent_name="assistant",
prompt="Hello, how are you?",
)
result = hook(event)
# Should allow and modify prompt
assert result.decision == "allow"
assert result.modified_input is not None
assert "[TRAINING GUIDANCE]" in result.modified_input.get("prompt", "")
assert "Be concise" in result.modified_input.get("prompt", "")
assert "Use examples" in result.modified_input.get("prompt", "")
class TestCLIApplyCommand:
"""Tests for CLI apply command."""
def test_cli_apply_command_exists(self):
"""Test that apply command is registered."""
from praisonai.cli.commands.train import app
# Check command exists
command_names = [cmd.name for cmd in app.registered_commands]
assert "apply" in command_names
def test_cli_apply_requires_session_id(self):
"""Test that apply command requires session_id argument."""
from typer.testing import CliRunner
from praisonai.cli.commands.train import app
runner = CliRunner()
result = runner.invoke(app, ["apply"])
# Should fail without session_id
assert result.exit_code != 0
class TestCLIShowEnhanced:
"""Tests for enhanced CLI show command."""
def test_cli_show_displays_iterations(self):
"""Test that show command displays iteration details."""
from typer.testing import CliRunner
from praisonai.cli.commands.train import app
runner = CliRunner()
# Mock storage
with patch('praisonai.cli.commands.train.TrainingStorage') as MockStorage:
mock_storage = Mock()
mock_storage.storage_path.exists.return_value = True
# Create mock iterations
mock_iterations = [
Mock(iteration_num=1, score=6.0, feedback="ok", suggestions=["a"]),
Mock(iteration_num=2, score=9.0, feedback="great", suggestions=["b", "c"]),
]
mock_storage.load_iterations.return_value = mock_iterations
mock_storage.load_report.return_value = Mock(
session_id="train-test",
avg_score=7.5,
to_dict=lambda: {"session_id": "train-test"}
)
MockStorage.return_value = mock_storage
result = runner.invoke(app, ["show", "train-test", "--iterations"])
# Should show iteration details
assert "Iteration" in result.output or result.exit_code == 0
+4 -4
View File
@@ -4054,7 +4054,7 @@ requires-dist = [
{ name = "praisonai-tools", marker = "extra == 'autogen'", specifier = ">=0.1.0" },
{ name = "praisonai-tools", marker = "extra == 'autogen-v4'", specifier = ">=0.1.0" },
{ name = "praisonai-tools", marker = "extra == 'crewai'", specifier = ">=0.1.0" },
{ name = "praisonaiagents", specifier = ">=1.3.0" },
{ name = "praisonaiagents", specifier = ">=1.4.2" },
{ name = "pyautogen", marker = "extra == 'autogen'", specifier = "==0.2.29" },
{ name = "pyjwt", marker = "extra == 'os'", specifier = ">=2.8.0" },
{ name = "pyngrok", marker = "extra == 'call'", specifier = ">=1.4.0" },
@@ -4110,7 +4110,7 @@ wheels = [
[[package]]
name = "praisonaiagents"
version = "1.3.0"
version = "1.4.2"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "aiohttp" },
@@ -4119,9 +4119,9 @@ dependencies = [
{ name = "pydantic" },
{ name = "rich" },
]
sdist = { url = "https://files.pythonhosted.org/packages/3b/32/01d10458a43e1a1bcef34fabdd21da110b865b50ec0ae8519bb7581231a1/praisonaiagents-1.3.0.tar.gz", hash = "sha256:c0f9d19da5996918ee5e04e5faedffbf434603b2779d7bc5c9fd3ae7b6f68a56", size = 1170867, upload-time = "2026-02-02T10:01:28.817Z" }
sdist = { url = "https://files.pythonhosted.org/packages/f8/54/ec17e7e4331cd191511d073f236e797856ab5a6998761117b6e51ee6d2f8/praisonaiagents-1.4.2.tar.gz", hash = "sha256:53760024b034d3ff166b6ee499e74fc29f41a301f6054608f68d13be310fa14f", size = 1184243, upload-time = "2026-02-03T08:42:10.738Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/c7/fa/671163138e78d76af6f6e4c13f4ac3e6c422af250e917055761d85d91c53/praisonaiagents-1.3.0-py3-none-any.whl", hash = "sha256:86bfa0a24f3e69ea297fc8146291cddf1171a649e4dd83b7f8c52315701c0d91", size = 1164709, upload-time = "2026-02-02T10:01:21.644Z" },
{ url = "https://files.pythonhosted.org/packages/47/a1/de39d2bd2a25387c513c3bc87a87fb578bea15620bd11e6133f89561a820/praisonaiagents-1.4.2-py3-none-any.whl", hash = "sha256:3d5b47c83f7b0f013b9eda85d2d870f5ef8a7e03a1326ea57d0ab3576d65f201", size = 1179296, upload-time = "2026-02-03T08:42:08.118Z" },
]
[[package]]