mirror of
https://github.com/MervinPraison/PraisonAI.git
synced 2026-11-09 03:39:37 -10:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
a970e849b4 | ||
|
|
8e68f4e71c | ||
|
|
308758cdc2 | ||
|
|
f8fa61b082 | ||
|
|
955d236924 | ||
|
|
d9d63c3a13 | ||
|
|
5bee6eb470 | ||
|
|
7cb1676b7c | ||
|
|
e2e9c3f9ee | ||
|
|
cbe8beeca6 | ||
|
|
f7b66a52b7 | ||
|
|
6dbf194312 | ||
|
|
335e8d7e47 | ||
|
|
d65840e092 | ||
|
|
5a13571fdd | ||
|
|
e3b48c2526 | ||
|
|
8d32238b4c | ||
|
|
c17b1298bb | ||
|
|
ec6c150315 | ||
|
|
3d26cedd5d | ||
|
|
8c735a60b2 | ||
|
|
2f5f2f1c13 | ||
|
|
0253073c68 | ||
|
|
7f0e334e8c | ||
|
|
cacf1097ef | ||
|
|
09f01919d2 | ||
|
|
92397615f8 | ||
|
|
4b21943a53 | ||
|
|
af56a753bd | ||
|
|
15c2559a3c | ||
|
|
3215bdc0a6 | ||
|
|
0ff0d83744 | ||
|
|
8f956f91dd | ||
|
|
954f6ac33d | ||
|
|
264d11d8d9 | ||
|
|
88da2bdd7b | ||
|
|
b84e6c2460 | ||
|
|
0434b23ada | ||
|
|
a9951e0694 | ||
|
|
073062cd96 | ||
|
|
936b422876 | ||
|
|
99ab421ce2 | ||
|
|
c1f334905e | ||
|
|
6d3ba7cdf8 | ||
|
|
96616498d2 | ||
|
|
1cad345415 | ||
|
|
802e7e52f9 | ||
|
|
53704f234c | ||
|
|
b575994d91 | ||
|
|
7ed8596261 | ||
|
|
deef65c4bb | ||
|
|
d51327effd | ||
|
|
30ce8857e8 | ||
|
|
397874ccfc | ||
|
|
2016902e9a | ||
|
|
d2d19511da | ||
|
|
bc9579b497 | ||
|
|
3161389341 | ||
|
|
b01eb2b8fa | ||
|
|
e609fb7425 | ||
|
|
168ef98473 | ||
|
|
ce2c2926f9 | ||
|
|
0a569e2b46 | ||
|
|
054aea8f66 | ||
|
|
34f1293cfc | ||
|
|
7c1e32afc0 | ||
|
|
1125c835ec |
No files matched your search
+4
-1
@@ -77,4 +77,7 @@ agents/praisonaiagents/praisonaiagents.egg-info
|
||||
.qodo
|
||||
CopilotKit*
|
||||
.gemini
|
||||
issue_details.json
|
||||
issue_details.json
|
||||
|
||||
# MCP Registry tokens
|
||||
.mcpregistry_*
|
||||
@@ -6,10 +6,13 @@
|
||||
</picture>
|
||||
</p>
|
||||
|
||||
<!-- mcp-name: io.github.MervinPraison/praisonai -->
|
||||
|
||||
<p align="center">
|
||||
<a href="https://github.com/MervinPraison/PraisonAI"><img src="https://static.pepy.tech/badge/PraisonAI" alt="Total Downloads" /></a>
|
||||
<a href="https://github.com/MervinPraison/PraisonAI"><img src="https://img.shields.io/github/v/release/MervinPraison/PraisonAI" alt="Latest Stable Version" /></a>
|
||||
<a href="https://github.com/MervinPraison/PraisonAI"><img src="https://img.shields.io/badge/License-MIT-yellow.svg" alt="License" /></a>
|
||||
<a href="https://registry.modelcontextprotocol.io/servers/io.github.MervinPraison/praisonai"><img src="https://img.shields.io/badge/MCP-Registry-blue" alt="MCP Registry" /></a>
|
||||
</p>
|
||||
|
||||
<div align="center">
|
||||
@@ -168,12 +171,13 @@ npm install praisonai
|
||||
|
||||
| Feature | Code | Docs |
|
||||
|---------|:----:|:----:|
|
||||
| 🚀 Single Agent | [Example](examples/python/agents/single-agent.py) | [📖](https://docs.praison.ai/agents/single) |
|
||||
| 🤝 Multi Agents | [Example](examples/python/general/mini_agents_example.py) | [📖](https://docs.praison.ai/concepts/agents) |
|
||||
| 🤖 Auto Agents | [Example](examples/python/general/auto_agents_example.py) | [📖](https://docs.praison.ai/features/autoagents) |
|
||||
| 🔄 Self Reflection AI Agents | [Example](examples/python/concepts/self-reflection-details.py) | [📖](https://docs.praison.ai/features/selfreflection) |
|
||||
| 🧠 Reasoning AI Agents | [Example](examples/python/concepts/reasoning-extraction.py) | [📖](https://docs.praison.ai/features/reasoning) |
|
||||
| 👁️ Multi Modal AI Agents | [Example](examples/python/general/multimodal.py) | [📖](https://docs.praison.ai/features/multimodal) |
|
||||
| **🤖 Core Agents** | | |
|
||||
| ↳ Single Agent | [Example](examples/python/agents/single-agent.py) | [📖](https://docs.praison.ai/agents/single) |
|
||||
| ↳ Multi Agents | [Example](examples/python/general/mini_agents_example.py) | [📖](https://docs.praison.ai/concepts/agents) |
|
||||
| ↳ Auto Agents | [Example](examples/python/general/auto_agents_example.py) | [📖](https://docs.praison.ai/features/autoagents) |
|
||||
| ↳ Self Reflection AI Agents | [Example](examples/python/concepts/self-reflection-details.py) | [📖](https://docs.praison.ai/features/selfreflection) |
|
||||
| ↳ Reasoning AI Agents | [Example](examples/python/concepts/reasoning-extraction.py) | [📖](https://docs.praison.ai/features/reasoning) |
|
||||
| ↳ Multi Modal AI Agents | [Example](examples/python/general/multimodal.py) | [📖](https://docs.praison.ai/features/multimodal) |
|
||||
| **🔄 Workflows** | | |
|
||||
| ↳ Simple Workflow | [Example](examples/python/workflows/simple_workflow.py) | [📖](https://docs.praison.ai/features/workflows) |
|
||||
| ↳ Workflow with Agents | [Example](examples/python/workflows/workflow_with_agents.py) | [📖](https://docs.praison.ai/features/workflows) |
|
||||
@@ -185,69 +189,111 @@ npm install praisonai
|
||||
| ↳ Workflow Branching | [Example](examples/python/workflows/workflow_branching.py) | [📖](https://docs.praison.ai/features/workflows) |
|
||||
| ↳ Workflow Early Stop | [Example](examples/python/workflows/workflow_early_stop.py) | [📖](https://docs.praison.ai/features/workflows) |
|
||||
| ↳ Workflow Checkpoints | [Example](examples/python/workflows/workflow_checkpoints.py) | [📖](https://docs.praison.ai/features/workflows) |
|
||||
| 📚 Add Custom Knowledge | [Example](examples/python/concepts/knowledge-agents.py) | [📖](https://docs.praison.ai/features/knowledge) |
|
||||
| 🧠 Memory (Short & Long Term) | [Example](examples/python/general/memory_example.py) | [📖](https://docs.praison.ai/concepts/memory) |
|
||||
| 📄 Chat with PDF Agents | [Example](examples/python/concepts/chat-with-pdf.py) | [📖](https://docs.praison.ai/features/chat-with-pdf) |
|
||||
| 💻 Code Interpreter Agents | [Example](examples/python/agents/code-agent.py) | [📖](https://docs.praison.ai/features/codeagent) |
|
||||
| ✏️ AI Code Editing Tools | [Example](#ai-code-editing) | [📖](https://docs.praison.ai/features/code) |
|
||||
| 📚 RAG Agents | [Example](examples/python/concepts/rag-agents.py) | [📖](https://docs.praison.ai/features/rag) |
|
||||
| 🤔 Async & Parallel Processing | [Example](examples/python/general/async_example.py) | [📖](https://docs.praison.ai/features/async) |
|
||||
| 🔢 Math Agents | [Example](examples/python/agents/math-agent.py) | [📖](https://docs.praison.ai/features/mathagent) |
|
||||
| 🎯 Structured Output Agents | [Example](examples/python/general/structured_agents_example.py) | [📖](https://docs.praison.ai/features/structured) |
|
||||
| 🔗 LangChain Integrated Agents | [Example](examples/python/general/langchain_example.py) | [📖](https://docs.praison.ai/features/langchain) |
|
||||
| 📞 Callback Agents | [Example](examples/python/general/advanced-callback-systems.py) | [📖](https://docs.praison.ai/features/callbacks) |
|
||||
| 🛠️ 100+ Custom Tools | [Example](examples/python/general/tools_example.py) | [📖](https://docs.praison.ai/tools/tools) |
|
||||
| 📄 YAML Configuration | [Example](examples/cookbooks/yaml/secondary_market_research_agents.yaml) | [📖](https://docs.praison.ai/developers/agents-playbook) |
|
||||
| 💯 100+ LLM Support | [Example](examples/python/providers/openai/openai_gpt4_example.py) | [📖](https://docs.praison.ai/models) |
|
||||
| 🔬 Deep Research Agents | [Example](examples/python/agents/research-agent.py) | [📖](https://docs.praison.ai/agents/deep-research) |
|
||||
| 🔄 Query Rewriter Agent | [Example](#5-query-rewriter-agent) | [📖](https://docs.praison.ai/agents/query-rewriter) |
|
||||
| 🌐 Native Web Search | [Example](examples/python/agents/websearch-agent.py) | [📖](https://docs.praison.ai/agents/websearch) |
|
||||
| 📥 Web Fetch (Anthropic) | [Example](#web-search-web-fetch--prompt-caching) | [📖](https://docs.praison.ai/features/model-capabilities) |
|
||||
| 💾 Prompt Caching | [Example](#web-search-web-fetch--prompt-caching) | [📖](https://docs.praison.ai/features/model-capabilities) |
|
||||
| 🧠 Claude Memory Tool | [Example](#claude-memory-tool-cli) | [📖](https://docs.praison.ai/features/claude-memory-tool) |
|
||||
| 💾 File-Based Memory | [Example](examples/python/general/memory_example.py) | [📖](https://docs.praison.ai/concepts/memory) |
|
||||
| 🔍 Built-in Search Tools | [Example](examples/python/agents/websearch-agent.py) | [📖](https://docs.praison.ai/tools/tavily) |
|
||||
| 📋 Planning Mode | [Example](examples/python/agents/planning-agent.py) | [📖](https://docs.praison.ai/features/planning-mode) |
|
||||
| 🔧 Planning Tools | [Example](#3-agent-with-planning-mode) | [📖](https://docs.praison.ai/features/planning-mode) |
|
||||
| 🧠 Planning Reasoning | [Example](#3-agent-with-planning-mode) | [📖](https://docs.praison.ai/features/planning-mode) |
|
||||
| 🔌 MCP Transports | [Example](examples/python/mcp/mcp-transports-overview.py) | [📖](https://docs.praison.ai/mcp/transports) |
|
||||
| 🌐 WebSocket MCP | [Example](examples/python/mcp/websocket-mcp.py) | [📖](https://docs.praison.ai/mcp/sse-transport) |
|
||||
| 🔐 MCP Security | [Example](examples/python/mcp/mcp-security.py) | [📖](https://docs.praison.ai/mcp/transports) |
|
||||
| 🔄 MCP Resumability | [Example](examples/python/mcp/mcp-resumability.py) | [📖](https://docs.praison.ai/mcp/sse-transport) |
|
||||
| ⚡ Fast Context | [Example](examples/context/00_agent_fast_context_basic.py) | [📖](https://docs.praison.ai/features/fast-context) |
|
||||
| 🖼️ Image Generation Agent | [Example](examples/python/image/image-agent.py) | [📖](https://docs.praison.ai/features/image-generation) |
|
||||
| 📷 Image to Text Agent | [Example](examples/python/agents/image-to-text-agent.py) | [📖](https://docs.praison.ai/agents/image-to-text) |
|
||||
| 🎬 Video Agent | [Example](examples/python/agents/video-agent.py) | [📖](https://docs.praison.ai/agents/video) |
|
||||
| 📊 Data Analyst Agent | [Example](examples/python/agents/data-analyst-agent.py) | [📖](https://docs.praison.ai/agents/data-analyst) |
|
||||
| 💰 Finance Agent | [Example](examples/python/agents/finance-agent.py) | [📖](https://docs.praison.ai/agents/finance) |
|
||||
| 🛒 Shopping Agent | [Example](examples/python/agents/shopping-agent.py) | [📖](https://docs.praison.ai/agents/shopping) |
|
||||
| ⭐ Recommendation Agent | [Example](examples/python/agents/recommendation-agent.py) | [📖](https://docs.praison.ai/agents/recommendation) |
|
||||
| 📖 Wikipedia Agent | [Example](examples/python/agents/wikipedia-agent.py) | [📖](https://docs.praison.ai/agents/wikipedia) |
|
||||
| 💻 Programming Agent | [Example](examples/python/agents/programming-agent.py) | [📖](https://docs.praison.ai/agents/programming) |
|
||||
| 📝 Markdown Agent | [Example](examples/python/agents/markdown-agent.py) | [📖](https://docs.praison.ai/agents/markdown) |
|
||||
| 📝 Prompt Expander Agent | [Example](#prompt-expansion) | [📖](https://docs.praison.ai/agents/prompt-expander) |
|
||||
| 🔀 Router Agent | [Example](examples/python/agents/router-agent-cost-optimization.py) | [📖](https://docs.praison.ai/features/routing) |
|
||||
| ⛓️ Prompt Chaining | [Example](examples/python/general/prompt_chaining.py) | [📖](https://docs.praison.ai/features/promptchaining) |
|
||||
| 🔍 Evaluator Optimiser | [Example](examples/python/general/evaluator-optimiser.py) | [📖](https://docs.praison.ai/features/evaluator-optimiser) |
|
||||
| 👷 Orchestrator Workers | [Example](examples/python/general/orchestrator-workers.py) | [📖](https://docs.praison.ai/features/orchestrator-worker) |
|
||||
| ⚡ Parallelisation | [Example](examples/python/general/parallelisation.py) | [📖](https://docs.praison.ai/features/parallelisation) |
|
||||
| 🔁 Repetitive Agents | [Example](examples/python/concepts/repetitive-agents.py) | [📖](https://docs.praison.ai/features/repetitive) |
|
||||
| 🤝 Agent Handoffs | [Example](examples/python/handoff/handoff_basic.py) | [📖](https://docs.praison.ai/features/handoffs) |
|
||||
| 🛡️ Guardrails | [Example](examples/python/guardrails/comprehensive-guardrails-example.py) | [📖](https://docs.praison.ai/features/guardrails) |
|
||||
| 💬 Sessions Management | [Example](examples/python/sessions/comprehensive-session-management.py) | [📖](https://docs.praison.ai/features/sessions) |
|
||||
| ✅ Human Approval | [Example](examples/python/general/human_approval_example.py) | [📖](https://docs.praison.ai/features/approval) |
|
||||
| 🔄 Stateful Agents | [Example](examples/python/stateful/workflow-state-example.py) | [📖](https://docs.praison.ai/features/stateful-agents) |
|
||||
| 🤖 Autonomous Workflow | [Example](examples/python/general/autonomous-agent.py) | [📖](https://docs.praison.ai/features/autonomous-workflow) |
|
||||
| 📜 Rules & Instructions | [Example](#6-rules--instructions) | [📖](https://docs.praison.ai/features/rules) |
|
||||
| 🪝 Hooks | [Example](#9-hooks) | [📖](https://docs.praison.ai/features/hooks) |
|
||||
| 📈 Telemetry | [Example](examples/python/telemetry/production-telemetry-example.py) | [📖](https://docs.praison.ai/features/telemetry) |
|
||||
| 📹 Camera Integration | [Example](examples/python/camera/) | [📖](https://docs.praison.ai/features/camera-integration) |
|
||||
| 📄 Project Docs (.praison/docs/) | [Example](#docs-cli) | [📖](https://docs.praison.ai/cli/docs) |
|
||||
| 🔌 MCP Config Management | [Example](#mcp-config-cli) | [📖](https://docs.praison.ai/cli/mcp) |
|
||||
| 💬 AI Commit Messages | [Example](#ai-commit-cli) | [📖](https://docs.praison.ai/cli/commit) |
|
||||
| @ @Mentions in Prompts | [Example](#mentions-in-prompts) | [📖](https://docs.praison.ai/cli/mentions) |
|
||||
| 💾 Auto-Save Sessions | [Example](#session-management-python) | [📖](https://docs.praison.ai/cli/session) |
|
||||
| 📜 History in Context | [Example](#session-management-python) | [📖](https://docs.praison.ai/cli/session) |
|
||||
| **💻 Code & Development** | | |
|
||||
| ↳ Code Interpreter Agents | [Example](examples/python/agents/code-agent.py) | [📖](https://docs.praison.ai/features/codeagent) |
|
||||
| ↳ AI Code Editing Tools | [Example](examples/python/code/code_editing_example.py) | [📖](https://docs.praison.ai/code/editing) |
|
||||
| ↳ External Agents (All) | [Example](examples/python/code/external_agents_example.py) | [📖](https://docs.praison.ai/code/external-agents) |
|
||||
| ↳ Claude Code CLI | [Example](examples/python/code/claude_code_example.py) | [📖](https://docs.praison.ai/code/claude-code) |
|
||||
| ↳ Gemini CLI | [Example](examples/python/code/gemini_cli_example.py) | [📖](https://docs.praison.ai/code/gemini-cli) |
|
||||
| ↳ Codex CLI | [Example](examples/python/code/codex_cli_example.py) | [📖](https://docs.praison.ai/code/codex-cli) |
|
||||
| ↳ Cursor CLI | [Example](examples/python/code/cursor_cli_example.py) | [📖](https://docs.praison.ai/code/cursor-cli) |
|
||||
| **🧠 Memory & Knowledge** | | |
|
||||
| ↳ Memory (Short & Long Term) | [Example](examples/python/general/memory_example.py) | [📖](https://docs.praison.ai/concepts/memory) |
|
||||
| ↳ File-Based Memory | [Example](examples/python/general/memory_example.py) | [📖](https://docs.praison.ai/concepts/memory) |
|
||||
| ↳ Claude Memory Tool | [Example](#claude-memory-tool-cli) | [📖](https://docs.praison.ai/features/claude-memory-tool) |
|
||||
| ↳ Add Custom Knowledge | [Example](examples/python/concepts/knowledge-agents.py) | [📖](https://docs.praison.ai/features/knowledge) |
|
||||
| ↳ RAG Agents | [Example](examples/python/concepts/rag-agents.py) | [📖](https://docs.praison.ai/features/rag) |
|
||||
| ↳ Chat with PDF Agents | [Example](examples/python/concepts/chat-with-pdf.py) | [📖](https://docs.praison.ai/features/chat-with-pdf) |
|
||||
| **🔬 Research & Intelligence** | | |
|
||||
| ↳ Deep Research Agents | [Example](examples/python/agents/research-agent.py) | [📖](https://docs.praison.ai/agents/deep-research) |
|
||||
| ↳ Query Rewriter Agent | [Example](#5-query-rewriter-agent) | [📖](https://docs.praison.ai/agents/query-rewriter) |
|
||||
| ↳ Native Web Search | [Example](examples/python/agents/websearch-agent.py) | [📖](https://docs.praison.ai/agents/websearch) |
|
||||
| ↳ Built-in Search Tools | [Example](examples/python/agents/websearch-agent.py) | [📖](https://docs.praison.ai/tools/tavily) |
|
||||
| ↳ Unified Web Search | [Example](src/praisonai-agents/examples/web_search_example.py) | [📖](https://docs.praison.ai/tools/web-search) |
|
||||
| ↳ Web Fetch (Anthropic) | [Example](#web-search-web-fetch--prompt-caching) | [📖](https://docs.praison.ai/features/model-capabilities) |
|
||||
| **📋 Planning & Execution** | | |
|
||||
| ↳ Planning Mode | [Example](examples/python/agents/planning-agent.py) | [📖](https://docs.praison.ai/features/planning-mode) |
|
||||
| ↳ Planning Tools | [Example](#3-agent-with-planning-mode) | [📖](https://docs.praison.ai/features/planning-mode) |
|
||||
| ↳ Planning Reasoning | [Example](#3-agent-with-planning-mode) | [📖](https://docs.praison.ai/features/planning-mode) |
|
||||
| ↳ Prompt Chaining | [Example](examples/python/general/prompt_chaining.py) | [📖](https://docs.praison.ai/features/promptchaining) |
|
||||
| ↳ Evaluator Optimiser | [Example](examples/python/general/evaluator-optimiser.py) | [📖](https://docs.praison.ai/features/evaluator-optimiser) |
|
||||
| ↳ Orchestrator Workers | [Example](examples/python/general/orchestrator-workers.py) | [📖](https://docs.praison.ai/features/orchestrator-worker) |
|
||||
| **👥 Specialized Agents** | | |
|
||||
| ↳ Data Analyst Agent | [Example](examples/python/agents/data-analyst-agent.py) | [📖](https://docs.praison.ai/agents/data-analyst) |
|
||||
| ↳ Finance Agent | [Example](examples/python/agents/finance-agent.py) | [📖](https://docs.praison.ai/agents/finance) |
|
||||
| ↳ Shopping Agent | [Example](examples/python/agents/shopping-agent.py) | [📖](https://docs.praison.ai/agents/shopping) |
|
||||
| ↳ Recommendation Agent | [Example](examples/python/agents/recommendation-agent.py) | [📖](https://docs.praison.ai/agents/recommendation) |
|
||||
| ↳ Wikipedia Agent | [Example](examples/python/agents/wikipedia-agent.py) | [📖](https://docs.praison.ai/agents/wikipedia) |
|
||||
| ↳ Programming Agent | [Example](examples/python/agents/programming-agent.py) | [📖](https://docs.praison.ai/agents/programming) |
|
||||
| ↳ Math Agents | [Example](examples/python/agents/math-agent.py) | [📖](https://docs.praison.ai/features/mathagent) |
|
||||
| ↳ Markdown Agent | [Example](examples/python/agents/markdown-agent.py) | [📖](https://docs.praison.ai/agents/markdown) |
|
||||
| ↳ Prompt Expander Agent | [Example](#prompt-expansion) | [📖](https://docs.praison.ai/agents/prompt-expander) |
|
||||
| **🎨 Media & Multimodal** | | |
|
||||
| ↳ Image Generation Agent | [Example](examples/python/image/image-agent.py) | [📖](https://docs.praison.ai/features/image-generation) |
|
||||
| ↳ Image to Text Agent | [Example](examples/python/agents/image-to-text-agent.py) | [📖](https://docs.praison.ai/agents/image-to-text) |
|
||||
| ↳ Video Agent | [Example](examples/python/agents/video-agent.py) | [📖](https://docs.praison.ai/agents/video) |
|
||||
| ↳ Camera Integration | [Example](examples/python/camera/) | [📖](https://docs.praison.ai/features/camera-integration) |
|
||||
| **🔌 Protocols & Integration** | | |
|
||||
| ↳ MCP Transports | [Example](examples/python/mcp/mcp-transports-overview.py) | [📖](https://docs.praison.ai/mcp/transports) |
|
||||
| ↳ WebSocket MCP | [Example](examples/python/mcp/websocket-mcp.py) | [📖](https://docs.praison.ai/mcp/sse-transport) |
|
||||
| ↳ MCP Security | [Example](examples/python/mcp/mcp-security.py) | [📖](https://docs.praison.ai/mcp/transports) |
|
||||
| ↳ MCP Resumability | [Example](examples/python/mcp/mcp-resumability.py) | [📖](https://docs.praison.ai/mcp/sse-transport) |
|
||||
| ↳ MCP Config Management | [Example](#mcp-config-cli) | [📖](https://docs.praison.ai/docs/cli/mcp) |
|
||||
| ↳ LangChain Integrated Agents | [Example](examples/python/general/langchain_example.py) | [📖](https://docs.praison.ai/features/langchain) |
|
||||
| **🛡️ Safety & Control** | | |
|
||||
| ↳ Guardrails | [Example](examples/python/guardrails/comprehensive-guardrails-example.py) | [📖](https://docs.praison.ai/features/guardrails) |
|
||||
| ↳ Human Approval | [Example](examples/python/general/human_approval_example.py) | [📖](https://docs.praison.ai/features/approval) |
|
||||
| ↳ Rules & Instructions | [Example](#6-rules--instructions) | [📖](https://docs.praison.ai/features/rules) |
|
||||
| **⚙️ Advanced Features** | | |
|
||||
| ↳ Async & Parallel Processing | [Example](examples/python/general/async_example.py) | [📖](https://docs.praison.ai/features/async) |
|
||||
| ↳ Parallelisation | [Example](examples/python/general/parallelisation.py) | [📖](https://docs.praison.ai/features/parallelisation) |
|
||||
| ↳ Repetitive Agents | [Example](examples/python/concepts/repetitive-agents.py) | [📖](https://docs.praison.ai/features/repetitive) |
|
||||
| ↳ Agent Handoffs | [Example](examples/python/handoff/handoff_basic.py) | [📖](https://docs.praison.ai/features/handoffs) |
|
||||
| ↳ Stateful Agents | [Example](examples/python/stateful/workflow-state-example.py) | [📖](https://docs.praison.ai/features/stateful-agents) |
|
||||
| ↳ Autonomous Workflow | [Example](examples/python/general/autonomous-agent.py) | [📖](https://docs.praison.ai/features/autonomous-workflow) |
|
||||
| ↳ Structured Output Agents | [Example](examples/python/general/structured_agents_example.py) | [📖](https://docs.praison.ai/features/structured) |
|
||||
| ↳ Model Router | [Example](examples/python/agents/router-agent-cost-optimization.py) | [📖](https://docs.praison.ai/features/model-router) |
|
||||
| ↳ Prompt Caching | [Example](#web-search-web-fetch--prompt-caching) | [📖](https://docs.praison.ai/features/model-capabilities) |
|
||||
| ↳ Fast Context | [Example](examples/context/00_agent_fast_context_basic.py) | [📖](https://docs.praison.ai/features/fast-context) |
|
||||
| **🛠️ Tools & Configuration** | | |
|
||||
| ↳ 100+ Custom Tools | [Example](examples/python/general/tools_example.py) | [📖](https://docs.praison.ai/tools/tools) |
|
||||
| ↳ YAML Configuration | [Example](examples/cookbooks/yaml/secondary_market_research_agents.yaml) | [📖](https://docs.praison.ai/developers/agents-playbook) |
|
||||
| ↳ 100+ LLM Support | [Example](examples/python/providers/openai/openai_gpt4_example.py) | [📖](https://docs.praison.ai/models) |
|
||||
| ↳ Callback Agents | [Example](examples/python/general/advanced-callback-systems.py) | [📖](https://docs.praison.ai/features/callbacks) |
|
||||
| ↳ Hooks | [Example](#9-hooks) | [📖](https://docs.praison.ai/features/hooks) |
|
||||
| **📊 Monitoring & Management** | | |
|
||||
| ↳ Sessions Management | [Example](examples/python/sessions/comprehensive-session-management.py) | [📖](https://docs.praison.ai/features/sessions) |
|
||||
| ↳ Auto-Save Sessions | [Example](#session-management-python) | [📖](https://docs.praison.ai/docs/cli/session) |
|
||||
| ↳ History in Context | [Example](#session-management-python) | [📖](https://docs.praison.ai/docs/cli/session) |
|
||||
| ↳ Telemetry | [Example](examples/python/telemetry/production-telemetry-example.py) | [📖](https://docs.praison.ai/features/telemetry) |
|
||||
| ↳ Project Docs (.praison/docs/) | [Example](#docs-cli) | [📖](https://docs.praison.ai/docs/cli/docs) |
|
||||
| ↳ AI Commit Messages | [Example](#ai-commit-cli) | [📖](https://docs.praison.ai/docs/cli/commit) |
|
||||
| ↳ @Mentions in Prompts | [Example](#mentions-in-prompts) | [📖](https://docs.praison.ai/docs/cli/mentions) |
|
||||
| **🖥️ CLI Features** | | |
|
||||
| ↳ Slash Commands | [Example](examples/python/cli/slash_commands_example.py) | [📖](https://docs.praison.ai/docs/cli/slash-commands) |
|
||||
| ↳ Autonomy Modes | [Example](examples/python/cli/autonomy_modes_example.py) | [📖](https://docs.praison.ai/docs/cli/autonomy-modes) |
|
||||
| ↳ Cost Tracking | [Example](examples/python/cli/cost_tracking_example.py) | [📖](https://docs.praison.ai/docs/cli/cost-tracking) |
|
||||
| ↳ Repository Map | [Example](examples/python/cli/repo_map_example.py) | [📖](https://docs.praison.ai/docs/cli/repo-map) |
|
||||
| ↳ Interactive TUI | [Example](examples/python/cli/interactive_tui_example.py) | [📖](https://docs.praison.ai/docs/cli/interactive-tui) |
|
||||
| ↳ Git Integration | [Example](examples/python/cli/git_integration_example.py) | [📖](https://docs.praison.ai/docs/cli/git-integration) |
|
||||
| ↳ Sandbox Execution | [Example](examples/python/cli/sandbox_execution_example.py) | [📖](https://docs.praison.ai/docs/cli/sandbox-execution) |
|
||||
| ↳ CLI Compare | [Example](examples/compare/cli_compare_basic.py) | [📖](https://docs.praison.ai/docs/cli/compare) |
|
||||
| ↳ Profile/Benchmark | [Example](#profile-benchmark) | [📖](https://docs.praison.ai/docs/cli/profile) |
|
||||
| ↳ Auto Mode | [Example](#auto-mode) | [📖](https://docs.praison.ai/docs/cli/auto) |
|
||||
| ↳ Init | [Example](#init) | [📖](https://docs.praison.ai/docs/cli/init) |
|
||||
| ↳ File Input | [Example](#file-input) | [📖](https://docs.praison.ai/docs/cli/file-input) |
|
||||
| ↳ Final Agent | [Example](#final-agent) | [📖](https://docs.praison.ai/docs/cli/final-agent) |
|
||||
| ↳ Max Tokens | [Example](#max-tokens) | [📖](https://docs.praison.ai/docs/cli/max-tokens) |
|
||||
| **🧪 Evaluation** | | |
|
||||
| ↳ Accuracy Evaluation | [Example](examples/eval/accuracy_example.py) | [📖](https://docs.praison.ai/docs/cli/eval) |
|
||||
| ↳ Performance Evaluation | [Example](examples/eval/performance_example.py) | [📖](https://docs.praison.ai/docs/cli/eval) |
|
||||
| ↳ Reliability Evaluation | [Example](examples/eval/reliability_example.py) | [📖](https://docs.praison.ai/docs/cli/eval) |
|
||||
| ↳ Criteria Evaluation | [Example](examples/eval/criteria_example.py) | [📖](https://docs.praison.ai/docs/cli/eval) |
|
||||
| **🎯 Agent Skills** | | |
|
||||
| ↳ Skills Management | [Example](examples/skills/basic_skill_usage.py) | [📖](https://docs.praison.ai/features/skills) |
|
||||
| ↳ Custom Skills | [Example](examples/skills/custom_skill_example.py) | [📖](https://docs.praison.ai/features/skills) |
|
||||
| **⏰ 24/7 Scheduling** | | |
|
||||
| ↳ Agent Scheduler | [Example](examples/python/scheduled_agents/news_checker_live.py) | [📖](https://docs.praison.ai/docs/cli/scheduler) |
|
||||
|
||||
---
|
||||
|
||||
@@ -941,6 +987,124 @@ Run with the same simple command:
|
||||
praisonai agents.yaml
|
||||
```
|
||||
|
||||
### 12. MCP (Model Context Protocol)
|
||||
|
||||
PraisonAI supports MCP Protocol Revision 2025-11-25 with multiple transports.
|
||||
|
||||
#### MCP Client (Consume MCP Servers)
|
||||
```python
|
||||
from praisonaiagents import Agent, MCP
|
||||
|
||||
# stdio - Local NPX/Python servers
|
||||
agent = Agent(tools=MCP("npx @modelcontextprotocol/server-memory"))
|
||||
|
||||
# Streamable HTTP - Production servers
|
||||
agent = Agent(tools=MCP("https://api.example.com/mcp"))
|
||||
|
||||
# WebSocket - Real-time bidirectional
|
||||
agent = Agent(tools=MCP("wss://api.example.com/mcp", auth_token="token"))
|
||||
|
||||
# SSE (Legacy) - Backward compatibility
|
||||
agent = Agent(tools=MCP("http://localhost:8080/sse"))
|
||||
|
||||
# With environment variables
|
||||
agent = Agent(
|
||||
tools=MCP(
|
||||
command="npx",
|
||||
args=["-y", "@modelcontextprotocol/server-brave-search"],
|
||||
env={"BRAVE_API_KEY": "your-key"}
|
||||
)
|
||||
)
|
||||
|
||||
# Multiple MCP servers + regular functions
|
||||
def my_custom_tool(query: str) -> str:
|
||||
"""Custom tool function."""
|
||||
return f"Result: {query}"
|
||||
|
||||
agent = Agent(
|
||||
name="MultiToolAgent",
|
||||
instructions="Agent with multiple MCP servers",
|
||||
tools=[
|
||||
MCP("uvx mcp-server-time"), # Time tools
|
||||
MCP("npx @modelcontextprotocol/server-memory"), # Memory tools
|
||||
my_custom_tool # Regular function
|
||||
]
|
||||
)
|
||||
```
|
||||
|
||||
#### MCP Server (Expose Tools as MCP Server)
|
||||
|
||||
Expose your Python functions as MCP tools for Claude Desktop, Cursor, and other MCP clients:
|
||||
|
||||
```python
|
||||
from praisonaiagents.mcp import ToolsMCPServer
|
||||
|
||||
def search_web(query: str, max_results: int = 5) -> dict:
|
||||
"""Search the web for information."""
|
||||
return {"results": [f"Result for {query}"]}
|
||||
|
||||
def calculate(expression: str) -> dict:
|
||||
"""Evaluate a mathematical expression."""
|
||||
return {"result": eval(expression)}
|
||||
|
||||
# Create and run MCP server
|
||||
server = ToolsMCPServer(name="my-tools")
|
||||
server.register_tools([search_web, calculate])
|
||||
server.run() # stdio for Claude Desktop
|
||||
# server.run_sse(host="0.0.0.0", port=8080) # SSE for web clients
|
||||
```
|
||||
|
||||
#### MCP Features
|
||||
| Feature | Description |
|
||||
|---------|-------------|
|
||||
| Session Management | Automatic Mcp-Session-Id handling |
|
||||
| Protocol Versioning | Mcp-Protocol-Version header |
|
||||
| Resumability | SSE stream recovery via Last-Event-ID |
|
||||
| Security | Origin validation, DNS rebinding prevention |
|
||||
| WebSocket | Auto-reconnect with exponential backoff |
|
||||
|
||||
### 13. A2A (Agent2Agent Protocol)
|
||||
|
||||
PraisonAI supports the [A2A Protocol](https://a2a-protocol.org) for agent-to-agent communication, enabling your agents to be discovered and collaborate with other AI agents.
|
||||
|
||||
#### A2A Server (Expose Agent as A2A Server)
|
||||
```python
|
||||
from praisonaiagents import Agent, A2A
|
||||
from fastapi import FastAPI
|
||||
|
||||
# Create an agent with tools
|
||||
def search_web(query: str) -> str:
|
||||
"""Search the web for information."""
|
||||
return f"Results for: {query}"
|
||||
|
||||
agent = Agent(
|
||||
name="Research Assistant",
|
||||
role="Research Analyst",
|
||||
goal="Help users research topics",
|
||||
tools=[search_web]
|
||||
)
|
||||
|
||||
# Expose as A2A Server
|
||||
a2a = A2A(agent=agent, url="http://localhost:8000/a2a")
|
||||
|
||||
app = FastAPI()
|
||||
app.include_router(a2a.get_router())
|
||||
|
||||
# Run: uvicorn app:app --reload
|
||||
# Agent Card: GET /.well-known/agent.json
|
||||
# Status: GET /status
|
||||
```
|
||||
|
||||
#### A2A Features
|
||||
| Feature | Description |
|
||||
|---------|-------------|
|
||||
| Agent Card | JSON metadata for agent discovery |
|
||||
| Skills Extraction | Auto-generate skills from tools |
|
||||
| Task Management | Stateful task lifecycle |
|
||||
| Streaming | SSE streaming for real-time updates |
|
||||
|
||||
> **Documentation**: [docs.praison.ai/a2a](https://docs.praison.ai/a2a) | **Examples**: [examples/python/a2a](https://github.com/MervinPraison/PraisonAI/tree/main/examples/python/a2a)
|
||||
|
||||
---
|
||||
|
||||
## 🎯 CLI / No-Code Interface
|
||||
@@ -954,6 +1118,32 @@ export OPENAI_API_KEY=xxxxxxxxxxxxxxxxxxxxxx
|
||||
praisonai --auto create a movie script about Robots in Mars
|
||||
```
|
||||
|
||||
### Interactive Mode CLI:
|
||||
```bash
|
||||
# Start interactive terminal mode (inspired by Gemini CLI, Codex CLI, Claude Code)
|
||||
praisonai --interactive
|
||||
praisonai -i
|
||||
|
||||
# Features:
|
||||
# - Streaming responses (no boxes)
|
||||
# - Built-in tools: read_file, write_file, list_files, execute_command, internet_search
|
||||
# - Slash commands: /help, /exit, /tools, /clear
|
||||
|
||||
# Chat mode - single prompt with interactive style (for testing/scripting)
|
||||
# Use --chat (or --chat-mode for backward compatibility)
|
||||
praisonai "list files in current folder" --chat
|
||||
praisonai "search the web for AI news" --chat
|
||||
praisonai "read README.md" --chat
|
||||
```
|
||||
|
||||
### Chat UI (Web Interface):
|
||||
```bash
|
||||
# Start web-based Chainlit chat interface (requires praisonai[chat])
|
||||
pip install "praisonai[chat]"
|
||||
praisonai chat
|
||||
# Opens browser at http://localhost:8084
|
||||
```
|
||||
|
||||
### Query Rewriting (works with any command):
|
||||
```bash
|
||||
# Rewrite query for better results (uses QueryRewriterAgent)
|
||||
@@ -1014,6 +1204,20 @@ praisonai "Complex analysis task" --planning --planning-reasoning
|
||||
praisonai "Task" --planning --auto-approve-plan
|
||||
```
|
||||
|
||||
### Tool Approval CLI:
|
||||
```bash
|
||||
# Auto-approve ALL tool executions (use with caution!)
|
||||
praisonai "run ls command" --trust
|
||||
|
||||
# Auto-approve tools up to a risk level (prompt for higher)
|
||||
# Levels: low, medium, high, critical
|
||||
praisonai "write to file" --approve-level high # Prompts for critical tools only
|
||||
praisonai "task" --approve-level medium # Prompts for high and critical
|
||||
|
||||
# Default behavior (no flags): prompts for all dangerous tools
|
||||
praisonai "run shell command" # Will prompt for approval
|
||||
```
|
||||
|
||||
### Memory CLI:
|
||||
```bash
|
||||
# Enable memory for agent (persists across sessions)
|
||||
@@ -1120,6 +1324,15 @@ praisonai workflow auto "Research AI trends" --pattern parallel
|
||||
# Generate routing workflow (classifier routes to specialists)
|
||||
praisonai workflow auto "Build a chatbot" --pattern routing
|
||||
|
||||
# Generate orchestrator-workers workflow (central orchestrator delegates)
|
||||
praisonai workflow auto "Comprehensive market analysis" --pattern orchestrator-workers
|
||||
|
||||
# Generate evaluator-optimizer workflow (iterative refinement)
|
||||
praisonai workflow auto "Write and refine article" --pattern evaluator-optimizer
|
||||
|
||||
# Specify output file
|
||||
praisonai workflow auto "Build a chatbot" --pattern routing
|
||||
|
||||
# Specify output file
|
||||
praisonai workflow auto "Research AI" --pattern sequential --output my_workflow.yaml
|
||||
```
|
||||
@@ -1129,7 +1342,7 @@ praisonai workflow auto "Research AI" --pattern sequential --output my_workflow.
|
||||
|------|-------------|
|
||||
| `--workflow-var key=value` | Set workflow variable (can be repeated) |
|
||||
| `--var key=value` | Set variable for YAML workflows |
|
||||
| `--pattern <pattern>` | Pattern for auto-generation (sequential, parallel, routing, loop) |
|
||||
| `--pattern <pattern>` | Pattern for auto-generation (sequential, parallel, routing, loop, orchestrator-workers, evaluator-optimizer) |
|
||||
| `--output <file>` | Output file for auto-generation |
|
||||
| `--llm <model>` | LLM model (e.g., openai/gpt-4o-mini) |
|
||||
| `--tools <tools>` | Tools (comma-separated, e.g., tavily) |
|
||||
@@ -1175,6 +1388,21 @@ praisonai "Analyze this data" --metrics
|
||||
praisonai "Complex task" --metrics --planning
|
||||
```
|
||||
|
||||
### Scheduler CLI:
|
||||
|
||||
```bash
|
||||
praisonai schedule start <name> "task" --interval hourly
|
||||
praisonai schedule list
|
||||
praisonai schedule logs <name> [--follow]
|
||||
praisonai schedule stop <name>
|
||||
praisonai schedule restart <name>
|
||||
praisonai schedule delete <name>
|
||||
praisonai schedule describe <name>
|
||||
praisonai schedule save <name> [file.yaml]
|
||||
praisonai schedule "task" --interval hourly # foreground mode
|
||||
praisonai schedule agents.yaml # foreground mode
|
||||
```
|
||||
|
||||
### Image Processing CLI:
|
||||
```bash
|
||||
# Process images with vision-based tasks
|
||||
@@ -1361,8 +1589,13 @@ praisonai "Complex analysis" --router --router-provider anthropic
|
||||
# Router automatically selects:
|
||||
# - Simple tasks → gpt-4o-mini, claude-3-haiku
|
||||
# - Complex tasks → gpt-4-turbo, claude-3-opus
|
||||
|
||||
# Create workflow with model routing template
|
||||
praisonai workflow create --template model-routing --output my_workflow.yaml
|
||||
```
|
||||
|
||||
Custom models can be configured in `agents.yaml`. See [Model Router Docs](https://docs.praison.ai/features/model-router) for details.
|
||||
|
||||
### Flow Display CLI:
|
||||
```bash
|
||||
# Enable visual workflow tracking
|
||||
@@ -1414,13 +1647,31 @@ praisonai mcp help
|
||||
|
||||
### AI Commit CLI:
|
||||
```bash
|
||||
# Generate AI commit message for staged changes
|
||||
# Full auto mode: stage all, security check, commit, and push
|
||||
praisonai commit -a
|
||||
|
||||
# Interactive mode (requires git add first)
|
||||
praisonai commit
|
||||
|
||||
# Generate, commit, and push
|
||||
# Interactive with auto-push
|
||||
praisonai commit --push
|
||||
|
||||
# Skip security check (not recommended)
|
||||
praisonai commit -a --no-verify
|
||||
```
|
||||
|
||||
**Features:**
|
||||
- 🤖 AI-generated conventional commit messages
|
||||
- 🔒 Built-in security scanning (API keys, passwords, secrets, sensitive files)
|
||||
- 📦 Auto-staging with `-a` flag
|
||||
- 🚀 Auto-push in full auto mode
|
||||
- ✏️ Edit message before commit in interactive mode
|
||||
|
||||
**Security Detection:**
|
||||
- API keys, secrets, tokens (AWS, GitHub, GitLab, Slack)
|
||||
- Passwords and private keys
|
||||
- Sensitive files (`.env`, `id_rsa`, `.pem`, `.key`, etc.)
|
||||
|
||||
### Serve CLI (API Server):
|
||||
```bash
|
||||
# Start API server for agents defined in YAML
|
||||
@@ -1451,6 +1702,55 @@ export N8N_API_KEY="your-api-key"
|
||||
praisonai agents.yaml --n8n
|
||||
```
|
||||
|
||||
### External Agents CLI:
|
||||
|
||||
Use external AI coding CLI tools (Claude Code, Gemini CLI, Codex CLI, Cursor CLI) as agent tools:
|
||||
|
||||
```bash
|
||||
# Use Claude Code for coding tasks
|
||||
praisonai "Refactor the auth module" --external-agent claude
|
||||
|
||||
# Use Gemini CLI for code analysis
|
||||
praisonai "Analyze codebase architecture" --external-agent gemini
|
||||
|
||||
# Use OpenAI Codex CLI
|
||||
praisonai "Fix all bugs in src/" --external-agent codex
|
||||
|
||||
# Use Cursor CLI
|
||||
praisonai "Add comprehensive tests" --external-agent cursor
|
||||
```
|
||||
|
||||
**Python API:**
|
||||
```python
|
||||
from praisonai.integrations import (
|
||||
ClaudeCodeIntegration,
|
||||
GeminiCLIIntegration,
|
||||
CodexCLIIntegration,
|
||||
CursorCLIIntegration
|
||||
)
|
||||
|
||||
# Create integration
|
||||
claude = ClaudeCodeIntegration(workspace="/project")
|
||||
|
||||
# Execute a coding task
|
||||
result = await claude.execute("Refactor the auth module")
|
||||
|
||||
# Use as agent tool
|
||||
from praisonai import Agent
|
||||
tool = claude.as_tool()
|
||||
agent = Agent(tools=[tool])
|
||||
```
|
||||
|
||||
**Environment Variables:**
|
||||
```bash
|
||||
export ANTHROPIC_API_KEY=your-key # Claude Code
|
||||
export GEMINI_API_KEY=your-key # Gemini CLI
|
||||
export OPENAI_API_KEY=your-key # Codex CLI
|
||||
export CURSOR_API_KEY=your-key # Cursor CLI
|
||||
```
|
||||
|
||||
See [External Agents Documentation](https://docs.praison.ai/code/external-agents) for more details.
|
||||
|
||||
### @Mentions in Prompts:
|
||||
```bash
|
||||
# Include file content in prompt
|
||||
@@ -1554,154 +1854,36 @@ agent = Agent(
|
||||
| Web Fetch | Anthropic |
|
||||
| Prompt Caching | OpenAI (auto), Anthropic, Bedrock, Deepseek |
|
||||
|
||||
## MCP (Model Context Protocol)
|
||||
|
||||
PraisonAI supports MCP Protocol Revision 2025-11-25 with multiple transports.
|
||||
|
||||
### MCP Client (Consume MCP Servers)
|
||||
```python
|
||||
from praisonaiagents import Agent, MCP
|
||||
|
||||
# stdio - Local NPX/Python servers
|
||||
agent = Agent(tools=MCP("npx @modelcontextprotocol/server-memory"))
|
||||
|
||||
# Streamable HTTP - Production servers
|
||||
agent = Agent(tools=MCP("https://api.example.com/mcp"))
|
||||
|
||||
# WebSocket - Real-time bidirectional
|
||||
agent = Agent(tools=MCP("wss://api.example.com/mcp", auth_token="token"))
|
||||
|
||||
# SSE (Legacy) - Backward compatibility
|
||||
agent = Agent(tools=MCP("http://localhost:8080/sse"))
|
||||
|
||||
# With environment variables
|
||||
agent = Agent(
|
||||
tools=MCP(
|
||||
command="npx",
|
||||
args=["-y", "@modelcontextprotocol/server-brave-search"],
|
||||
env={"BRAVE_API_KEY": "your-key"}
|
||||
)
|
||||
)
|
||||
|
||||
# Multiple MCP servers + regular functions
|
||||
def my_custom_tool(query: str) -> str:
|
||||
"""Custom tool function."""
|
||||
return f"Result: {query}"
|
||||
|
||||
agent = Agent(
|
||||
name="MultiToolAgent",
|
||||
instructions="Agent with multiple MCP servers",
|
||||
tools=[
|
||||
MCP("uvx mcp-server-time"), # Time tools
|
||||
MCP("npx @modelcontextprotocol/server-memory"), # Memory tools
|
||||
my_custom_tool # Regular function
|
||||
]
|
||||
)
|
||||
```
|
||||
|
||||
### MCP Server (Expose Tools as MCP Server)
|
||||
|
||||
Expose your Python functions as MCP tools for Claude Desktop, Cursor, and other MCP clients:
|
||||
|
||||
```python
|
||||
from praisonaiagents.mcp import ToolsMCPServer
|
||||
|
||||
def search_web(query: str, max_results: int = 5) -> dict:
|
||||
"""Search the web for information."""
|
||||
return {"results": [f"Result for {query}"]}
|
||||
|
||||
def calculate(expression: str) -> dict:
|
||||
"""Evaluate a mathematical expression."""
|
||||
return {"result": eval(expression)}
|
||||
|
||||
# Create and run MCP server
|
||||
server = ToolsMCPServer(name="my-tools")
|
||||
server.register_tools([search_web, calculate])
|
||||
server.run() # stdio for Claude Desktop
|
||||
# server.run_sse(host="0.0.0.0", port=8080) # SSE for web clients
|
||||
```
|
||||
|
||||
### MCP Features
|
||||
| Feature | Description |
|
||||
|---------|-------------|
|
||||
| Session Management | Automatic Mcp-Session-Id handling |
|
||||
| Protocol Versioning | Mcp-Protocol-Version header |
|
||||
| Resumability | SSE stream recovery via Last-Event-ID |
|
||||
| Security | Origin validation, DNS rebinding prevention |
|
||||
| WebSocket | Auto-reconnect with exponential backoff |
|
||||
|
||||
## A2A (Agent2Agent Protocol)
|
||||
|
||||
PraisonAI supports the [A2A Protocol](https://a2a-protocol.org) for agent-to-agent communication, enabling your agents to be discovered and collaborate with other AI agents.
|
||||
|
||||
### A2A Server (Expose Agent as A2A Server)
|
||||
```python
|
||||
from praisonaiagents import Agent, A2A
|
||||
from fastapi import FastAPI
|
||||
|
||||
# Create an agent with tools
|
||||
def search_web(query: str) -> str:
|
||||
"""Search the web for information."""
|
||||
return f"Results for: {query}"
|
||||
|
||||
agent = Agent(
|
||||
name="Research Assistant",
|
||||
role="Research Analyst",
|
||||
goal="Help users research topics",
|
||||
tools=[search_web]
|
||||
)
|
||||
|
||||
# Expose as A2A Server
|
||||
a2a = A2A(agent=agent, url="http://localhost:8000/a2a")
|
||||
|
||||
app = FastAPI()
|
||||
app.include_router(a2a.get_router())
|
||||
|
||||
# Run: uvicorn app:app --reload
|
||||
# Agent Card: GET /.well-known/agent.json
|
||||
# Status: GET /status
|
||||
```
|
||||
|
||||
### A2A Features
|
||||
| Feature | Description |
|
||||
|---------|-------------|
|
||||
| Agent Card | JSON metadata for agent discovery |
|
||||
| Skills Extraction | Auto-generate skills from tools |
|
||||
| Task Management | Stateful task lifecycle |
|
||||
| Streaming | SSE streaming for real-time updates |
|
||||
|
||||
> **Documentation**: [docs.praison.ai/a2a](https://docs.praison.ai/a2a) | **Examples**: [examples/python/a2a](https://github.com/MervinPraison/PraisonAI/tree/main/examples/python/a2a)
|
||||
|
||||
## CLI Features
|
||||
|
||||
| Feature | Docs |
|
||||
|---------|:----:|
|
||||
| 🔄 Query Rewrite - RAG optimization | [📖](https://docs.praison.ai/cli/query-rewrite) |
|
||||
| 🔬 Deep Research - Automated research | [📖](https://docs.praison.ai/cli/deep-research) |
|
||||
| 📋 Planning - Step-by-step execution | [📖](https://docs.praison.ai/cli/planning) |
|
||||
| 💾 Memory - Persistent agent memory | [📖](https://docs.praison.ai/cli/memory) |
|
||||
| 📜 Rules - Auto-discovered instructions | [📖](https://docs.praison.ai/cli/rules) |
|
||||
| 🔄 Workflow - Multi-step workflows | [📖](https://docs.praison.ai/cli/workflow) |
|
||||
| 🪝 Hooks - Event-driven actions | [📖](https://docs.praison.ai/cli/hooks) |
|
||||
| 🧠 Claude Memory - Anthropic memory tool | [📖](https://docs.praison.ai/cli/claude-memory) |
|
||||
| 🛡️ Guardrail - Output validation | [📖](https://docs.praison.ai/cli/guardrail) |
|
||||
| 📊 Metrics - Token usage tracking | [📖](https://docs.praison.ai/cli/metrics) |
|
||||
| 🖼️ Image - Vision processing | [📖](https://docs.praison.ai/cli/image) |
|
||||
| 📡 Telemetry - Usage monitoring | [📖](https://docs.praison.ai/cli/telemetry) |
|
||||
| 🔌 MCP - Model Context Protocol | [📖](https://docs.praison.ai/cli/mcp) |
|
||||
| ⚡ Fast Context - Codebase search | [📖](https://docs.praison.ai/cli/fast-context) |
|
||||
| 📚 Knowledge - RAG management | [📖](https://docs.praison.ai/cli/knowledge) |
|
||||
| 💬 Session - Conversation management | [📖](https://docs.praison.ai/cli/session) |
|
||||
| 🔧 Tools - Tool discovery | [📖](https://docs.praison.ai/cli/tools) |
|
||||
| 🤝 Handoff - Agent delegation | [📖](https://docs.praison.ai/cli/handoff) |
|
||||
| 🧠 Auto Memory - Memory extraction | [📖](https://docs.praison.ai/cli/auto-memory) |
|
||||
| 📋 Todo - Task management | [📖](https://docs.praison.ai/cli/todo) |
|
||||
| 🎯 Router - Smart model selection | [📖](https://docs.praison.ai/cli/router) |
|
||||
| 📈 Flow Display - Visual workflow | [📖](https://docs.praison.ai/cli/flow-display) |
|
||||
| ✨ Prompt Expansion - Detailed prompts | [📖](https://docs.praison.ai/cli/prompt-expansion) |
|
||||
| 🌐 Web Search - Real-time search | [📖](https://docs.praison.ai/cli/web-search) |
|
||||
| 📥 Web Fetch - URL content retrieval | [📖](https://docs.praison.ai/cli/web-fetch) |
|
||||
| 💾 Prompt Caching - Cost reduction | [📖](https://docs.praison.ai/cli/prompt-caching) |
|
||||
| 🔄 Query Rewrite - RAG optimization | [📖](https://docs.praison.ai/docs/cli/query-rewrite) |
|
||||
| 🔬 Deep Research - Automated research | [📖](https://docs.praison.ai/docs/cli/deep-research) |
|
||||
| 📋 Planning - Step-by-step execution | [📖](https://docs.praison.ai/docs/cli/planning) |
|
||||
| 💾 Memory - Persistent agent memory | [📖](https://docs.praison.ai/docs/cli/memory) |
|
||||
| 📜 Rules - Auto-discovered instructions | [📖](https://docs.praison.ai/docs/cli/rules) |
|
||||
| 🔄 Workflow - Multi-step workflows | [📖](https://docs.praison.ai/docs/cli/workflow) |
|
||||
| 🪝 Hooks - Event-driven actions | [📖](https://docs.praison.ai/docs/cli/hooks) |
|
||||
| 🧠 Claude Memory - Anthropic memory tool | [📖](https://docs.praison.ai/docs/cli/claude-memory) |
|
||||
| 🛡️ Guardrail - Output validation | [📖](https://docs.praison.ai/docs/cli/guardrail) |
|
||||
| 📊 Metrics - Token usage tracking | [📖](https://docs.praison.ai/docs/cli/metrics) |
|
||||
| 🖼️ Image - Vision processing | [📖](https://docs.praison.ai/docs/cli/image) |
|
||||
| 📡 Telemetry - Usage monitoring | [📖](https://docs.praison.ai/docs/cli/telemetry) |
|
||||
| 🔌 MCP - Model Context Protocol | [📖](https://docs.praison.ai/docs/cli/mcp) |
|
||||
| ⚡ Fast Context - Codebase search | [📖](https://docs.praison.ai/docs/cli/fast-context) |
|
||||
| 📚 Knowledge - RAG management | [📖](https://docs.praison.ai/docs/cli/knowledge) |
|
||||
| 💬 Session - Conversation management | [📖](https://docs.praison.ai/docs/cli/session) |
|
||||
| 🔧 Tools - Tool discovery | [📖](https://docs.praison.ai/docs/cli/tools) |
|
||||
| 🤝 Handoff - Agent delegation | [📖](https://docs.praison.ai/docs/cli/handoff) |
|
||||
| 🧠 Auto Memory - Memory extraction | [📖](https://docs.praison.ai/docs/cli/auto-memory) |
|
||||
| 📋 Todo - Task management | [📖](https://docs.praison.ai/docs/cli/todo) |
|
||||
| 🎯 Router - Smart model selection | [📖](https://docs.praison.ai/docs/cli/router) |
|
||||
| 📈 Flow Display - Visual workflow | [📖](https://docs.praison.ai/docs/cli/flow-display) |
|
||||
| ✨ Prompt Expansion - Detailed prompts | [📖](https://docs.praison.ai/docs/cli/prompt-expansion) |
|
||||
| 🌐 Web Search - Real-time search | [📖](https://docs.praison.ai/docs/cli/web-search) |
|
||||
| 📥 Web Fetch - URL content retrieval | [📖](https://docs.praison.ai/docs/cli/web-fetch) |
|
||||
| 💾 Prompt Caching - Cost reduction | [📖](https://docs.praison.ai/docs/cli/prompt-caching) |
|
||||
|
||||
---
|
||||
|
||||
@@ -2253,7 +2435,7 @@ PraisonAI provides zero-dependency persistent memory for agents. For detailed ex
|
||||
- 📖 **Wikipedia Agent** - Search and extract Wikipedia information
|
||||
- 💻 **Programming Agent** - Code development and analysis
|
||||
- 📝 **Markdown Agent** - Generate and format Markdown content
|
||||
- 🔀 **Router Agent** - Dynamic task routing with cost optimisation
|
||||
- 🔀 **Model Router** - Smart model selection based on task complexity
|
||||
|
||||
### MCP Protocol
|
||||
|
||||
@@ -2274,6 +2456,7 @@ PraisonAI provides zero-dependency persistent memory for agents. For detailed ex
|
||||
- 🤝 **Agent Handoffs** - Transfer context between specialised agents
|
||||
- 🛡️ **Guardrails** - Input/output validation and safety checks
|
||||
- ✅ **Human Approval** - Require human confirmation for critical actions
|
||||
- 🔐 **Tool Approval CLI** - `--trust` (auto-approve all) and `--approve-level` (risk-based approval)
|
||||
- 💬 **Sessions Management** - Isolated conversation contexts
|
||||
- 🔄 **Stateful Agents** - Maintain state across interactions
|
||||
|
||||
|
||||
@@ -16,7 +16,7 @@ RUN mkdir -p /root/.praison
|
||||
# Install Python packages (using latest versions)
|
||||
RUN pip install --no-cache-dir \
|
||||
praisonai_tools \
|
||||
"praisonai>=2.3.40" \
|
||||
"praisonai>=2.3.87" \
|
||||
"praisonai[chat]" \
|
||||
"embedchain[github,youtube]"
|
||||
|
||||
|
||||
@@ -20,7 +20,7 @@ RUN mkdir -p /root/.praison
|
||||
# Install Python packages (using latest versions)
|
||||
RUN pip install --no-cache-dir \
|
||||
praisonai_tools \
|
||||
"praisonai>=2.3.40" \
|
||||
"praisonai>=2.3.87" \
|
||||
"praisonai[ui]" \
|
||||
"praisonai[chat]" \
|
||||
"praisonai[realtime]" \
|
||||
|
||||
@@ -16,7 +16,7 @@ RUN mkdir -p /root/.praison
|
||||
# Install Python packages (using latest versions)
|
||||
RUN pip install --no-cache-dir \
|
||||
praisonai_tools \
|
||||
"praisonai>=2.3.40" \
|
||||
"praisonai>=2.3.87" \
|
||||
"praisonai[ui]" \
|
||||
"praisonai[crewai]"
|
||||
|
||||
|
||||
@@ -0,0 +1,49 @@
|
||||
# PraisonAI Examples
|
||||
|
||||
This folder contains examples for PraisonAI. For detailed documentation, visit [docs.praison.ai](https://docs.praison.ai).
|
||||
|
||||
## Structure
|
||||
|
||||
```
|
||||
examples/
|
||||
├── python/ # Python examples
|
||||
│ ├── agents/ # Agent examples (single, multi, router, etc.)
|
||||
│ ├── workflows/ # Workflow patterns (routing, parallel, loop)
|
||||
│ ├── tools/ # Custom tools examples
|
||||
│ ├── mcp/ # MCP protocol examples
|
||||
│ ├── memory/ # Memory and sessions
|
||||
│ ├── code/ # Code editing and external CLI tools
|
||||
│ └── ...
|
||||
├── yaml/ # YAML workflow examples
|
||||
└── cookbooks/ # Complete use-case examples
|
||||
```
|
||||
|
||||
## Quick Links
|
||||
|
||||
| Category | Examples | Docs |
|
||||
|----------|----------|------|
|
||||
| **Agents** | [python/agents/](python/agents/) | [📖](https://docs.praison.ai/concepts/agents) |
|
||||
| **Workflows** | [python/workflows/](python/workflows/) | [📖](https://docs.praison.ai/features/workflows) |
|
||||
| **Model Router** | [python/agents/router-agent-cost-optimization.py](python/agents/router-agent-cost-optimization.py) | [📖](https://docs.praison.ai/features/model-router) |
|
||||
| **MCP** | [python/mcp/](python/mcp/) | [📖](https://docs.praison.ai/mcp) |
|
||||
| **Memory** | [python/memory/](python/memory/) | [📖](https://docs.praison.ai/concepts/memory) |
|
||||
| **Tools** | [python/tools/](python/tools/) | [📖](https://docs.praison.ai/tools) |
|
||||
| **Code** | [python/code/](python/code/) | [📖](https://docs.praison.ai/code) |
|
||||
| **YAML** | [yaml/](yaml/) | [📖](https://docs.praison.ai/features/yaml-workflows) |
|
||||
|
||||
## Running Examples
|
||||
|
||||
```bash
|
||||
# Install PraisonAI
|
||||
pip install praisonai
|
||||
|
||||
# Set API key
|
||||
export OPENAI_API_KEY=your_key_here
|
||||
|
||||
# Run an example
|
||||
python examples/python/agents/single-agent.py
|
||||
```
|
||||
|
||||
## CLI Commands
|
||||
|
||||
See the main [README.md](../README.md#-cli--no-code-interface) for all CLI commands.
|
||||
@@ -0,0 +1,85 @@
|
||||
"""
|
||||
CLI Compare Example - Advanced Usage
|
||||
|
||||
This example demonstrates advanced comparison features including
|
||||
custom mode configurations and result analysis.
|
||||
"""
|
||||
|
||||
from praisonai.cli.features.compare import (
|
||||
CompareHandler,
|
||||
CompareResult,
|
||||
ModeResult,
|
||||
get_mode_config,
|
||||
save_compare_result,
|
||||
format_comparison_table,
|
||||
)
|
||||
|
||||
# Create custom mode configurations
|
||||
custom_modes = {
|
||||
"fast": {"llm": "gpt-4o-mini"},
|
||||
"accurate": {"llm": "gpt-4o", "planning": True},
|
||||
"research": {"research": True, "web_search": True},
|
||||
}
|
||||
|
||||
# Create handler with verbose output
|
||||
handler = CompareHandler(verbose=True)
|
||||
|
||||
# Example: Manually create comparison results for analysis
|
||||
# (useful for testing without API calls)
|
||||
mock_result = CompareResult(
|
||||
query="What are the latest AI trends?",
|
||||
comparisons=[
|
||||
ModeResult(
|
||||
mode="basic",
|
||||
output="AI trends include LLMs, multimodal models, and agents.",
|
||||
execution_time_ms=1234.5,
|
||||
model_used="gpt-4o-mini",
|
||||
tools_used=[]
|
||||
),
|
||||
ModeResult(
|
||||
mode="tools",
|
||||
output="Based on recent search results, AI trends in 2024 include...",
|
||||
execution_time_ms=2567.8,
|
||||
model_used="gpt-4o-mini",
|
||||
tools_used=["internet_search"]
|
||||
),
|
||||
ModeResult(
|
||||
mode="research",
|
||||
output="Comprehensive research shows that AI trends...",
|
||||
execution_time_ms=5432.1,
|
||||
model_used="gpt-4o-mini",
|
||||
tools_used=["internet_search", "web_scraper"]
|
||||
),
|
||||
]
|
||||
)
|
||||
|
||||
# Get summary statistics
|
||||
summary = mock_result.get_summary()
|
||||
print("Comparison Summary:")
|
||||
print(f" Fastest mode: {summary['fastest']} ({summary['fastest_time_ms']:.1f}ms)")
|
||||
print(f" Slowest mode: {summary['slowest']} ({summary['slowest_time_ms']:.1f}ms)")
|
||||
|
||||
# Format as table
|
||||
print("\n" + format_comparison_table(mock_result))
|
||||
|
||||
# Convert to JSON for storage
|
||||
print("\nJSON Output:")
|
||||
print(mock_result.to_json())
|
||||
|
||||
# Save to file
|
||||
# save_compare_result(mock_result, "advanced_comparison.json")
|
||||
|
||||
print("\n--- CLI Usage Examples ---")
|
||||
print("""
|
||||
# Basic comparison
|
||||
praisonai "What is AI?" --compare "basic,tools"
|
||||
|
||||
# Compare with specific model
|
||||
praisonai "Explain quantum computing" --compare "basic,planning,research" --model gpt-4o
|
||||
|
||||
# Save results to file
|
||||
praisonai "Latest tech news" --compare "basic,tools,web_search" --compare-output results.json
|
||||
|
||||
# Verbose output
|
||||
praisonai "Write a poem" --compare "basic,planning" --verbose
|
||||
""")
|
||||
@@ -0,0 +1,45 @@
|
||||
"""
|
||||
CLI Compare Example - Basic Usage
|
||||
|
||||
This example demonstrates how to compare different CLI modes programmatically.
|
||||
"""
|
||||
|
||||
from praisonai.cli.features.compare import (
|
||||
CompareHandler,
|
||||
get_mode_config,
|
||||
list_available_modes,
|
||||
parse_modes,
|
||||
)
|
||||
|
||||
# List all available comparison modes
|
||||
print("Available modes:")
|
||||
for mode in list_available_modes():
|
||||
config = get_mode_config(mode)
|
||||
print(f" - {mode}: {config}")
|
||||
|
||||
# Parse modes from comma-separated string
|
||||
modes = parse_modes("basic,tools,planning")
|
||||
print(f"\nParsed modes: {modes}")
|
||||
|
||||
# Create compare handler
|
||||
handler = CompareHandler(verbose=True)
|
||||
|
||||
# Compare modes (this will run actual agent calls)
|
||||
# Uncomment to run:
|
||||
# result = handler.compare(
|
||||
# query="What is artificial intelligence?",
|
||||
# modes=["basic", "tools"],
|
||||
# model="gpt-4o-mini"
|
||||
# )
|
||||
# handler.print_result(result)
|
||||
|
||||
# Save results to file
|
||||
# result = handler.execute(
|
||||
# query="Explain machine learning",
|
||||
# modes_str="basic,planning",
|
||||
# output_path="comparison_results.json"
|
||||
# )
|
||||
|
||||
print("\nTo run comparison via CLI:")
|
||||
print(' praisonai "What is AI?" --compare "basic,tools,planning"')
|
||||
print(' praisonai "What is AI?" --compare "basic,research" --compare-output results.json')
|
||||
@@ -0,0 +1,23 @@
|
||||
# Example: Data Collection Agent with 30-minute Schedule
|
||||
# Collects and analyzes data every 30 minutes
|
||||
|
||||
framework: praisonai
|
||||
|
||||
agents:
|
||||
- name: "Data Collector"
|
||||
role: "Data Analysis Specialist"
|
||||
goal: "Collect and analyze market data at regular intervals"
|
||||
instructions: |
|
||||
You are a data analysis specialist focused on collecting and analyzing
|
||||
market trends and metrics. Provide actionable insights from the data.
|
||||
tools:
|
||||
- search_tool
|
||||
verbose: false
|
||||
|
||||
task: "Collect latest market data and identify key trends"
|
||||
|
||||
# Schedule: Every 30 minutes with immediate execution
|
||||
schedule:
|
||||
interval: "*/30m"
|
||||
max_retries: 5
|
||||
run_immediately: true
|
||||
@@ -0,0 +1,53 @@
|
||||
# Example: 24/7 AI News Monitor with Scheduler
|
||||
# This agent runs continuously at regular intervals to monitor AI news
|
||||
|
||||
framework: praisonai
|
||||
|
||||
agents:
|
||||
- name: "AI News Monitor"
|
||||
role: "Technology News Analyst"
|
||||
goal: "Monitor and summarize the latest AI developments"
|
||||
instructions: |
|
||||
You are an expert technology news analyst specializing in AI and machine learning.
|
||||
Your job is to search for the latest AI news, identify the most important developments,
|
||||
and provide clear, concise summaries with sources.
|
||||
|
||||
Focus on:
|
||||
- Major AI product launches
|
||||
- Research breakthroughs
|
||||
- Industry trends
|
||||
- Policy and regulation updates
|
||||
tools:
|
||||
- search_tool
|
||||
verbose: true
|
||||
|
||||
# Task to execute on each scheduled run
|
||||
task: |
|
||||
Search for the latest AI news from the past 24 hours.
|
||||
Provide the top 3 most important stories with:
|
||||
1. Headline
|
||||
2. Brief summary (2-3 sentences)
|
||||
3. Source URL
|
||||
4. Why it's significant
|
||||
|
||||
# Schedule configuration
|
||||
schedule:
|
||||
# Interval options:
|
||||
# - "hourly" - Every hour
|
||||
# - "daily" - Every 24 hours
|
||||
# - "*/30m" - Every 30 minutes
|
||||
# - "*/6h" - Every 6 hours
|
||||
# - "3600" - Custom seconds
|
||||
interval: "hourly"
|
||||
|
||||
# Maximum retry attempts if execution fails
|
||||
max_retries: 3
|
||||
|
||||
# Run immediately when scheduler starts (before first scheduled interval)
|
||||
run_immediately: true
|
||||
|
||||
# Optional: Variables for customization
|
||||
variables:
|
||||
news_category: "AI and Machine Learning"
|
||||
region: "Global"
|
||||
time_range: "24 hours"
|
||||
@@ -0,0 +1,69 @@
|
||||
"""
|
||||
Accuracy Evaluation Example
|
||||
|
||||
This example demonstrates how to evaluate agent output accuracy
|
||||
by comparing against expected outputs using LLM-as-judge.
|
||||
"""
|
||||
|
||||
import os
|
||||
from praisonaiagents import Agent
|
||||
from praisonaiagents.eval import AccuracyEvaluator
|
||||
|
||||
# Check if we have an API key
|
||||
has_api_key = os.getenv("OPENAI_API_KEY") is not None
|
||||
|
||||
if has_api_key:
|
||||
# Create a simple math tutor agent
|
||||
agent = Agent(
|
||||
instructions="You are a math tutor. Answer math questions concisely with just the number."
|
||||
)
|
||||
|
||||
# Create accuracy evaluator
|
||||
evaluator = AccuracyEvaluator(
|
||||
agent=agent,
|
||||
input_text="What is 2 + 2?",
|
||||
expected_output="4",
|
||||
num_iterations=3, # Run 3 times for statistical significance
|
||||
verbose=True
|
||||
)
|
||||
|
||||
# Run evaluation
|
||||
result = evaluator.run(print_summary=True)
|
||||
|
||||
# Check results
|
||||
print(f"\nAverage Score: {result.avg_score}/10")
|
||||
print(f"Passed: {result.passed}")
|
||||
else:
|
||||
print("⚠️ No OPENAI_API_KEY found. Using mock evaluation...")
|
||||
|
||||
# Create evaluator without agent for mock testing
|
||||
evaluator = AccuracyEvaluator(
|
||||
func=lambda x: "4", # Mock function
|
||||
input_text="What is 2 + 2?",
|
||||
expected_output="4",
|
||||
num_iterations=1,
|
||||
verbose=True
|
||||
)
|
||||
|
||||
# Test with mock output
|
||||
try:
|
||||
result = evaluator.run(print_summary=True)
|
||||
print(f"\nMock Average Score: {result.avg_score}/10")
|
||||
print(f"Mock Passed: {result.passed}")
|
||||
except Exception as e:
|
||||
print(f"Mock evaluation failed: {e}")
|
||||
print("This is expected without an API key")
|
||||
|
||||
# You can also evaluate pre-generated outputs
|
||||
print("\n--- Testing pre-generated output evaluation ---")
|
||||
output = "The answer is 4"
|
||||
|
||||
if has_api_key:
|
||||
try:
|
||||
result2 = evaluator.evaluate_output(output, print_summary=True)
|
||||
print(f"Pre-generated output score: {result2.avg_score}/10")
|
||||
except Exception as e:
|
||||
print(f"Pre-generated evaluation failed: {e}")
|
||||
else:
|
||||
print("⚠️ Skipping pre-generated evaluation (no API key)")
|
||||
print("To run full evaluation, set OPENAI_API_KEY environment variable")
|
||||
@@ -0,0 +1,105 @@
|
||||
"""
|
||||
Batch Evaluation Example
|
||||
|
||||
This example demonstrates how to run batch evaluations
|
||||
from a JSON test file.
|
||||
"""
|
||||
|
||||
import os
|
||||
import json
|
||||
import tempfile
|
||||
from praisonaiagents.eval import AccuracyEvaluator
|
||||
|
||||
# Check if we have an API key
|
||||
has_api_key = os.getenv("OPENAI_API_KEY") is not None
|
||||
|
||||
# Create a test file with multiple test cases
|
||||
test_cases = [
|
||||
{
|
||||
"input": "What is 2 + 2?",
|
||||
"expected": "4"
|
||||
},
|
||||
{
|
||||
"input": "What is the capital of France?",
|
||||
"expected": "Paris"
|
||||
},
|
||||
{
|
||||
"input": "What color is the sky?",
|
||||
"expected": "Blue"
|
||||
}
|
||||
]
|
||||
|
||||
# Save test cases to a temporary file
|
||||
with tempfile.NamedTemporaryFile(mode='w', suffix='.json', delete=False) as f:
|
||||
json.dump(test_cases, f)
|
||||
test_file = f.name
|
||||
|
||||
print(f"Test file created: {test_file}")
|
||||
print(f"Running {len(test_cases)} test cases...")
|
||||
|
||||
if has_api_key:
|
||||
# You can use the CLI for batch evaluation:
|
||||
# praisonai eval batch --agent agents.yaml --test-file tests.json --batch-type accuracy
|
||||
|
||||
# Or programmatically:
|
||||
from praisonaiagents import Agent
|
||||
|
||||
agent = Agent(
|
||||
instructions="You are a helpful assistant. Answer questions concisely."
|
||||
)
|
||||
|
||||
results = []
|
||||
for i, test_case in enumerate(test_cases):
|
||||
print(f"\nTest {i + 1}: {test_case['input']}")
|
||||
|
||||
evaluator = AccuracyEvaluator(
|
||||
agent=agent,
|
||||
input_text=test_case["input"],
|
||||
expected_output=test_case["expected"],
|
||||
num_iterations=1
|
||||
)
|
||||
|
||||
result = evaluator.run()
|
||||
results.append({
|
||||
"input": test_case["input"],
|
||||
"expected": test_case["expected"],
|
||||
"score": result.avg_score,
|
||||
"passed": result.passed
|
||||
})
|
||||
|
||||
print(f" Score: {result.avg_score}/10")
|
||||
print(f" Passed: {result.passed}")
|
||||
|
||||
# Summary
|
||||
passed = sum(1 for r in results if r["passed"])
|
||||
print(f"\n{'='*50}")
|
||||
print(f"Summary: {passed}/{len(results)} tests passed")
|
||||
print(f"Pass Rate: {passed/len(results):.1%}")
|
||||
else:
|
||||
print("⚠️ No OPENAI_API_KEY found. Running mock batch evaluation...")
|
||||
|
||||
# Mock evaluation without API key
|
||||
results = []
|
||||
for i, test_case in enumerate(test_cases):
|
||||
print(f"\nTest {i + 1}: {test_case['input']}")
|
||||
|
||||
# Mock result (all would fail without API key)
|
||||
mock_score = 0.0
|
||||
mock_passed = False
|
||||
|
||||
results.append({
|
||||
"input": test_case["input"],
|
||||
"expected": test_case["expected"],
|
||||
"score": mock_score,
|
||||
"passed": mock_passed
|
||||
})
|
||||
|
||||
print(f" Score: {mock_score}/10 (mock)")
|
||||
print(f" Passed: {mock_passed} (expected without API key)")
|
||||
|
||||
# Summary
|
||||
passed = sum(1 for r in results if r["passed"])
|
||||
print(f"\n{'='*50}")
|
||||
print(f"Summary: {passed}/{len(results)} tests passed (mock evaluation)")
|
||||
print(f"Pass Rate: {passed/len(results):.1%}")
|
||||
print("\nTo run real batch evaluation, set OPENAI_API_KEY environment variable")
|
||||
@@ -0,0 +1,82 @@
|
||||
"""
|
||||
Criteria Evaluation Example
|
||||
|
||||
This example demonstrates how to evaluate agent outputs
|
||||
against custom criteria using LLM-as-judge.
|
||||
"""
|
||||
|
||||
import os
|
||||
from praisonaiagents import Agent
|
||||
from praisonaiagents.eval import CriteriaEvaluator
|
||||
|
||||
# Check if we have an API key
|
||||
has_api_key = os.getenv("OPENAI_API_KEY") is not None
|
||||
|
||||
if has_api_key:
|
||||
print("--- Testing Criteria Evaluation with Agent ---")
|
||||
# Create a customer service agent
|
||||
agent = Agent(
|
||||
instructions="You are a friendly customer service agent. Be helpful and empathetic."
|
||||
)
|
||||
|
||||
# Create criteria evaluator with numeric scoring
|
||||
evaluator = CriteriaEvaluator(
|
||||
criteria="Response is helpful, empathetic, and provides a clear solution",
|
||||
agent=agent,
|
||||
input_text="I'm frustrated because my order hasn't arrived yet.",
|
||||
scoring_type="numeric", # Score 1-10
|
||||
threshold=7.0, # Pass if score >= 7
|
||||
num_iterations=2,
|
||||
verbose=True
|
||||
)
|
||||
|
||||
# Run evaluation
|
||||
result = evaluator.run(print_summary=True)
|
||||
|
||||
print("\nNumeric Scoring Results:")
|
||||
print(f" Average Score: {result.avg_score}/10")
|
||||
print(f" Pass Rate: {result.pass_rate:.1%}")
|
||||
print(f" Passed: {result.passed}")
|
||||
|
||||
# Binary scoring example (pass/fail)
|
||||
print("\n--- Testing Binary Scoring ---")
|
||||
binary_evaluator = CriteriaEvaluator(
|
||||
criteria="Response does not contain any offensive language",
|
||||
agent=agent,
|
||||
input_text="Tell me a joke",
|
||||
scoring_type="binary", # Pass or Fail
|
||||
verbose=True
|
||||
)
|
||||
|
||||
binary_result = binary_evaluator.run(print_summary=True)
|
||||
|
||||
# With on_fail callback
|
||||
print("\n--- Testing Failure Callback ---")
|
||||
def handle_failure(score):
|
||||
print(f"ALERT: Evaluation failed with score {score.score}")
|
||||
print(f"Reasoning: {score.reasoning}")
|
||||
|
||||
callback_evaluator = CriteriaEvaluator(
|
||||
criteria="Response is professional and helpful",
|
||||
agent=agent,
|
||||
input_text="Help me",
|
||||
on_fail=handle_failure,
|
||||
threshold=8.0
|
||||
)
|
||||
|
||||
callback_evaluator.run()
|
||||
else:
|
||||
print("⚠️ No OPENAI_API_KEY found. Skipping agent-based criteria evaluation...")
|
||||
|
||||
# Test pre-generated output evaluation (doesn't need agent)
|
||||
print("\n--- Testing Pre-generated Output Evaluation ---")
|
||||
if has_api_key:
|
||||
output = "I understand your frustration. Let me check on your order right away."
|
||||
try:
|
||||
result2 = evaluator.evaluate_output(output, print_summary=True)
|
||||
print(f"Pre-generated output score: {result2.avg_score}/10")
|
||||
except Exception as e:
|
||||
print(f"Pre-generated evaluation failed: {e}")
|
||||
else:
|
||||
print("⚠️ Skipping pre-generated evaluation (no API key)")
|
||||
print("To run full evaluation, set OPENAI_API_KEY environment variable")
|
||||
@@ -0,0 +1,59 @@
|
||||
"""
|
||||
Performance Evaluation Example
|
||||
|
||||
This example demonstrates how to benchmark agent performance
|
||||
by measuring runtime and memory usage across multiple iterations.
|
||||
"""
|
||||
|
||||
import os
|
||||
from praisonaiagents import Agent
|
||||
from praisonaiagents.eval import PerformanceEvaluator
|
||||
|
||||
# Check if we have an API key
|
||||
has_api_key = os.getenv("OPENAI_API_KEY") is not None
|
||||
|
||||
if has_api_key:
|
||||
print("--- Testing Agent Performance ---")
|
||||
# Create a simple agent
|
||||
agent = Agent(
|
||||
instructions="You are a helpful assistant. Answer questions briefly."
|
||||
)
|
||||
|
||||
# Create performance evaluator
|
||||
evaluator = PerformanceEvaluator(
|
||||
agent=agent,
|
||||
input_text="What is the capital of France?",
|
||||
num_iterations=10, # Run 10 benchmark iterations
|
||||
warmup_runs=2, # 2 warmup runs before measurement
|
||||
track_memory=True, # Track memory usage
|
||||
verbose=True
|
||||
)
|
||||
|
||||
# Run evaluation
|
||||
result = evaluator.run(print_summary=True)
|
||||
|
||||
# Access detailed metrics
|
||||
print("\nAgent Performance Results:")
|
||||
print(f" Average Time: {result.avg_run_time:.4f}s")
|
||||
print(f" Min Time: {result.min_run_time:.4f}s")
|
||||
print(f" Max Time: {result.max_run_time:.4f}s")
|
||||
print(f" Median Time: {result.median_run_time:.4f}s")
|
||||
print(f" P95 Time: {result.p95_run_time:.4f}s")
|
||||
print(f" Avg Memory: {result.avg_memory:.2f} MB")
|
||||
else:
|
||||
print("⚠️ No OPENAI_API_KEY found. Skipping agent performance test...")
|
||||
|
||||
# You can also benchmark any function
|
||||
def my_function():
|
||||
import time
|
||||
time.sleep(0.1)
|
||||
return "done"
|
||||
|
||||
func_evaluator = PerformanceEvaluator(
|
||||
func=my_function,
|
||||
num_iterations=5,
|
||||
warmup_runs=1,
|
||||
track_memory=False
|
||||
)
|
||||
|
||||
func_result = func_evaluator.run(print_summary=True)
|
||||
@@ -0,0 +1,72 @@
|
||||
"""
|
||||
Reliability Evaluation Example
|
||||
|
||||
This example demonstrates how to verify that agents call
|
||||
the expected tools during execution.
|
||||
"""
|
||||
|
||||
import os
|
||||
from praisonaiagents import Agent
|
||||
from praisonaiagents.eval import ReliabilityEvaluator
|
||||
|
||||
|
||||
def search_web(query: str) -> str:
|
||||
"""Search the web for information."""
|
||||
return f"Search results for: {query}"
|
||||
|
||||
|
||||
def calculate(expression: str) -> str:
|
||||
"""Calculate a math expression."""
|
||||
return str(eval(expression))
|
||||
|
||||
|
||||
# Check if we have an API key
|
||||
has_api_key = os.getenv("OPENAI_API_KEY") is not None
|
||||
|
||||
if has_api_key:
|
||||
print("--- Testing Agent Tool Call Reliability ---")
|
||||
# Create an agent with tools
|
||||
agent = Agent(
|
||||
instructions="You are a helpful assistant with access to search and calculator tools.",
|
||||
tools=[search_web, calculate]
|
||||
)
|
||||
|
||||
# Create reliability evaluator
|
||||
evaluator = ReliabilityEvaluator(
|
||||
agent=agent,
|
||||
input_text="Search for the weather in Paris and calculate 25 * 4",
|
||||
expected_tools=["search_web", "calculate"], # Tools that should be called
|
||||
forbidden_tools=["delete_file"], # Tools that should NOT be called
|
||||
verbose=True
|
||||
)
|
||||
|
||||
# Run evaluation
|
||||
result = evaluator.run(print_summary=True)
|
||||
|
||||
# Check results
|
||||
print("\nAgent Reliability Results:")
|
||||
print(f" Passed: {result.passed}")
|
||||
print(f" Pass Rate: {result.pass_rate:.1%}")
|
||||
else:
|
||||
print("⚠️ No OPENAI_API_KEY found. Skipping agent reliability test...")
|
||||
print("Agent would fail to call tools without API key (expected behavior)")
|
||||
|
||||
# You can also evaluate pre-recorded tool calls (doesn't need API key)
|
||||
print("\n--- Testing Pre-recorded Tool Call Evaluation ---")
|
||||
# Create a mock agent for tool call evaluation
|
||||
mock_agent = Agent(instructions="Mock agent")
|
||||
evaluator = ReliabilityEvaluator(
|
||||
agent=mock_agent,
|
||||
expected_tools=["search_web", "calculate"], # Tools that should be called
|
||||
forbidden_tools=["delete_file"], # Tools that should NOT be called
|
||||
verbose=True
|
||||
)
|
||||
|
||||
result2 = evaluator.evaluate_tool_calls(
|
||||
actual_tools=["search_web", "calculate"],
|
||||
print_summary=True
|
||||
)
|
||||
|
||||
print("\nPre-recorded Tool Call Results:")
|
||||
print(f" Passed: {result2.passed}")
|
||||
print(f" Pass Rate: {result2.pass_rate:.1%}")
|
||||
@@ -0,0 +1,132 @@
|
||||
"""
|
||||
AutoAgents Workflow Patterns Example
|
||||
|
||||
This example demonstrates the new workflow auto-generation features:
|
||||
- Dynamic agent count based on task complexity
|
||||
- Multiple workflow patterns (sequential, parallel, routing, orchestrator-workers, evaluator-optimizer)
|
||||
- Pattern recommendation based on task keywords
|
||||
|
||||
Documentation: https://docs.praison.ai/features/autoagents
|
||||
CLI Reference: https://docs.praison.ai/nocode/auto
|
||||
"""
|
||||
|
||||
from praisonai.auto import AutoGenerator, WorkflowAutoGenerator
|
||||
import os
|
||||
import tempfile
|
||||
|
||||
# Ensure API key is set
|
||||
if not os.environ.get("OPENAI_API_KEY"):
|
||||
print("Please set OPENAI_API_KEY environment variable")
|
||||
exit(1)
|
||||
|
||||
print("=" * 60)
|
||||
print("AutoAgents Workflow Patterns Demo")
|
||||
print("=" * 60)
|
||||
|
||||
# =============================================================================
|
||||
# Example 1: Dynamic Agent Count (Simple vs Complex Tasks)
|
||||
# =============================================================================
|
||||
print("\n📊 Example 1: Dynamic Agent Count")
|
||||
print("-" * 40)
|
||||
|
||||
# Simple task - should create 1-2 agents
|
||||
with tempfile.NamedTemporaryFile(mode='w', suffix='.yaml', delete=False) as f:
|
||||
simple_file = f.name
|
||||
|
||||
simple_generator = AutoGenerator(
|
||||
topic="Write a haiku about spring",
|
||||
agent_file=simple_file,
|
||||
framework="praisonai"
|
||||
)
|
||||
|
||||
print("Simple task: 'Write a haiku about spring'")
|
||||
result = simple_generator.generate()
|
||||
print(f"Generated: {result}")
|
||||
|
||||
# Complex task - should create 3-4 agents
|
||||
with tempfile.NamedTemporaryFile(mode='w', suffix='.yaml', delete=False) as f:
|
||||
complex_file = f.name
|
||||
|
||||
complex_generator = AutoGenerator(
|
||||
topic="Research AI trends, analyze market data, write a comprehensive report with executive summary",
|
||||
agent_file=complex_file,
|
||||
framework="praisonai"
|
||||
)
|
||||
|
||||
print("\nComplex task: 'Research AI trends, analyze market data...'")
|
||||
result = complex_generator.generate()
|
||||
print(f"Generated: {result}")
|
||||
|
||||
# =============================================================================
|
||||
# Example 2: Pattern Recommendation
|
||||
# =============================================================================
|
||||
print("\n🎯 Example 2: Pattern Recommendation")
|
||||
print("-" * 40)
|
||||
|
||||
test_tasks = [
|
||||
("Write a blog post", "sequential"),
|
||||
("Research from multiple sources simultaneously", "parallel"),
|
||||
("Classify and route customer requests", "routing"),
|
||||
("Comprehensive analysis and break down the problem", "orchestrator-workers"),
|
||||
("Refine and improve content quality", "evaluator-optimizer"),
|
||||
]
|
||||
|
||||
for task, expected in test_tasks:
|
||||
generator = WorkflowAutoGenerator(topic=task)
|
||||
recommended = generator.recommend_pattern()
|
||||
status = "✅" if recommended == expected else "⚠️"
|
||||
print(f"{status} Task: '{task[:40]}...'")
|
||||
print(f" Recommended: {recommended}")
|
||||
|
||||
# =============================================================================
|
||||
# Example 3: Generate Workflow with Specific Pattern
|
||||
# =============================================================================
|
||||
print("\n🔄 Example 3: Generate Workflows with Patterns")
|
||||
print("-" * 40)
|
||||
|
||||
patterns = ["sequential", "parallel", "orchestrator-workers", "evaluator-optimizer"]
|
||||
|
||||
for pattern in patterns:
|
||||
with tempfile.NamedTemporaryFile(mode='w', suffix='.yaml', delete=False) as f:
|
||||
workflow_file = f.name
|
||||
|
||||
generator = WorkflowAutoGenerator(
|
||||
topic="Create a market analysis report",
|
||||
workflow_file=workflow_file
|
||||
)
|
||||
|
||||
print(f"\n📋 Pattern: {pattern}")
|
||||
result = generator.generate(pattern=pattern)
|
||||
print(f" Generated: {result}")
|
||||
|
||||
# Read and show structure
|
||||
import yaml
|
||||
with open(result, 'r') as f:
|
||||
data = yaml.safe_load(f)
|
||||
print(f" Agents: {list(data.get('agents', {}).keys())}")
|
||||
print(f" Steps: {len(data.get('steps', []))}")
|
||||
|
||||
# =============================================================================
|
||||
# Example 4: CLI Usage (for reference)
|
||||
# =============================================================================
|
||||
print("\n💻 Example 4: CLI Commands (for reference)")
|
||||
print("-" * 40)
|
||||
print("""
|
||||
# Auto-generate agents (dynamic count)
|
||||
praisonai --auto "Write a haiku about spring"
|
||||
|
||||
# Auto-generate workflow with pattern
|
||||
praisonai workflow auto "Research and write a report" --pattern sequential
|
||||
praisonai workflow auto "Research from multiple sources" --pattern parallel
|
||||
praisonai workflow auto "Comprehensive market analysis" --pattern orchestrator-workers
|
||||
praisonai workflow auto "Refine content quality" --pattern evaluator-optimizer
|
||||
|
||||
# With output file
|
||||
praisonai workflow auto "Customer support routing" --pattern routing --output support.yaml
|
||||
""")
|
||||
|
||||
print("\n" + "=" * 60)
|
||||
print("✅ AutoAgents Workflow Patterns Demo Complete!")
|
||||
print("=" * 60)
|
||||
print("\nDocumentation: https://docs.praison.ai/features/autoagents")
|
||||
print("CLI Reference: https://docs.praison.ai/nocode/auto")
|
||||
@@ -0,0 +1,38 @@
|
||||
"""
|
||||
Autonomy Modes Example for PraisonAI CLI.
|
||||
|
||||
Control AI autonomy: suggest, auto_edit, full_auto.
|
||||
Docs: https://docs.praison.ai/cli/autonomy-modes
|
||||
"""
|
||||
|
||||
from praisonai.cli.features import AutonomyModeHandler
|
||||
from praisonai.cli.features.autonomy_mode import ActionRequest, ActionType
|
||||
|
||||
# Initialize with suggest mode (safest)
|
||||
handler = AutonomyModeHandler()
|
||||
manager = handler.initialize(mode="suggest")
|
||||
|
||||
print(f"Current mode: {handler.get_mode()}")
|
||||
print("Available modes: suggest, auto_edit, full_auto")
|
||||
|
||||
# Create an action request
|
||||
action = ActionRequest(
|
||||
action_type=ActionType.FILE_WRITE,
|
||||
description="Edit src/main.py to add logging",
|
||||
details={"file": "src/main.py"}
|
||||
)
|
||||
|
||||
# In suggest mode, this would require approval
|
||||
# For demo, we'll just show the action
|
||||
print(f"\nAction: {action.description}")
|
||||
print(f"Type: {action.action_type.value}")
|
||||
print("Would require approval in 'suggest' mode")
|
||||
|
||||
# Change to auto_edit mode
|
||||
handler.set_mode("auto_edit")
|
||||
print(f"\nChanged to: {handler.get_mode()}")
|
||||
print("File edits now auto-approved, shell commands still require approval")
|
||||
|
||||
# Get statistics
|
||||
stats = handler.get_stats()
|
||||
print(f"\nStats: {stats}")
|
||||
@@ -0,0 +1,34 @@
|
||||
"""
|
||||
Cost Tracking Example for PraisonAI CLI.
|
||||
|
||||
Real-time token usage and cost monitoring.
|
||||
Docs: https://docs.praison.ai/cli/cost-tracking
|
||||
"""
|
||||
|
||||
from praisonai.cli.features import CostTrackerHandler
|
||||
|
||||
# Initialize tracker
|
||||
handler = CostTrackerHandler()
|
||||
tracker = handler.initialize(session_id="demo-session")
|
||||
|
||||
# Track some requests
|
||||
handler.track_request("gpt-4o", input_tokens=1000, output_tokens=500)
|
||||
handler.track_request("gpt-4o-mini", input_tokens=2000, output_tokens=800)
|
||||
handler.track_request("claude-3-5-sonnet", input_tokens=1500, output_tokens=600)
|
||||
|
||||
# Get totals
|
||||
print(f"Total tokens: {handler.get_tokens():,}")
|
||||
print(f"Total cost: ${handler.get_cost():.4f}")
|
||||
|
||||
# Get detailed summary
|
||||
summary = handler.get_summary()
|
||||
print("\n=== Session Summary ===")
|
||||
print(f"Session ID: {summary['session_id']}")
|
||||
print(f"Total requests: {summary['total_requests']}")
|
||||
print(f"Input tokens: {summary['total_input_tokens']:,}")
|
||||
print(f"Output tokens: {summary['total_output_tokens']:,}")
|
||||
print(f"Total cost: ${summary['total_cost']:.4f}")
|
||||
print(f"Avg cost/request: ${summary['avg_cost_per_request']:.4f}")
|
||||
|
||||
# Display formatted (uses Rich if available)
|
||||
handler.display_summary()
|
||||
@@ -0,0 +1,45 @@
|
||||
"""
|
||||
Git Integration Example for PraisonAI CLI.
|
||||
|
||||
Auto-commit with AI messages, diff viewing, undo.
|
||||
Docs: https://docs.praison.ai/cli/git-integration
|
||||
"""
|
||||
|
||||
from praisonai.cli.features import GitIntegrationHandler
|
||||
|
||||
# Initialize
|
||||
handler = GitIntegrationHandler(verbose=True)
|
||||
git = handler.initialize(repo_path=".")
|
||||
|
||||
# Check if it's a git repo
|
||||
if not git.is_repo:
|
||||
print("Not a git repository!")
|
||||
exit(1)
|
||||
|
||||
# Show status
|
||||
print("=== Git Status ===")
|
||||
status = handler.show_status()
|
||||
print(f"Branch: {status.branch}")
|
||||
print(f"Staged: {len(status.staged_files)}")
|
||||
print(f"Modified: {len(status.modified_files)}")
|
||||
print(f"Untracked: {len(status.untracked_files)}")
|
||||
|
||||
# Show diff (if any changes)
|
||||
if status.has_changes:
|
||||
print("\n=== Diff ===")
|
||||
diff = handler.show_diff()
|
||||
if diff:
|
||||
print(diff[:300] + "..." if len(diff) > 300 else diff)
|
||||
|
||||
# Show recent commits
|
||||
print("\n=== Recent Commits ===")
|
||||
commits = handler.show_log(count=5)
|
||||
for c in commits:
|
||||
print(f" {c.short_hash} {c.message[:50]}")
|
||||
|
||||
# To commit with AI message:
|
||||
# commit = handler.commit() # Auto-generates message
|
||||
# commit = handler.commit(message="Custom message")
|
||||
|
||||
# To undo last commit:
|
||||
# handler.undo(soft=True) # Keeps changes staged
|
||||
@@ -0,0 +1,49 @@
|
||||
"""
|
||||
Interactive TUI Example for PraisonAI CLI.
|
||||
|
||||
Rich terminal interface with completions and history.
|
||||
Docs: https://docs.praison.ai/cli/interactive-tui
|
||||
"""
|
||||
|
||||
from praisonai.cli.features import InteractiveTUIHandler
|
||||
from praisonai.cli.features.interactive_tui import InteractiveConfig
|
||||
|
||||
# Custom configuration
|
||||
config = InteractiveConfig(
|
||||
prompt="ai> ",
|
||||
multiline=True,
|
||||
enable_completions=True,
|
||||
show_status_bar=True
|
||||
)
|
||||
|
||||
# Define callbacks
|
||||
def on_input(text):
|
||||
"""Handle regular input."""
|
||||
return f"You said: {text}"
|
||||
|
||||
def on_command(cmd):
|
||||
"""Handle slash commands."""
|
||||
if cmd == "/exit":
|
||||
return {"type": "exit"}
|
||||
return {"type": "command", "message": f"Executed: {cmd}"}
|
||||
|
||||
# Initialize
|
||||
handler = InteractiveTUIHandler()
|
||||
session = handler.initialize(
|
||||
config=config,
|
||||
on_input=on_input,
|
||||
on_command=on_command
|
||||
)
|
||||
|
||||
# Add commands for completion
|
||||
session.add_commands(["help", "exit", "cost", "model", "plan"])
|
||||
|
||||
# Add symbols from codebase
|
||||
session.add_symbols(["MyClass", "my_function"])
|
||||
|
||||
print("Interactive TUI initialized!")
|
||||
print("Commands available: /help, /exit, /cost, /model, /plan")
|
||||
print("Run handler.run() to start interactive session")
|
||||
|
||||
# Uncomment to run interactive session:
|
||||
# handler.run()
|
||||
@@ -0,0 +1,24 @@
|
||||
"""
|
||||
Repository Map Example for PraisonAI CLI.
|
||||
|
||||
Intelligent codebase mapping with symbol extraction.
|
||||
Docs: https://docs.praison.ai/cli/repo-map
|
||||
"""
|
||||
|
||||
from praisonai.cli.features import RepoMapHandler
|
||||
|
||||
# Initialize with current directory
|
||||
handler = RepoMapHandler(verbose=True)
|
||||
repo_map = handler.initialize(root=".")
|
||||
|
||||
# Get the repository map
|
||||
print("=== Repository Map ===")
|
||||
map_str = handler.get_map()
|
||||
print(map_str[:500] if len(map_str) > 500 else map_str)
|
||||
print("..." if len(map_str) > 500 else "")
|
||||
|
||||
# Refresh after file changes
|
||||
handler.refresh()
|
||||
|
||||
# Get symbols for a specific file (if exists)
|
||||
# symbols = handler.get_context("MyClass")
|
||||
@@ -0,0 +1,42 @@
|
||||
"""
|
||||
Sandbox Execution Example for PraisonAI CLI.
|
||||
|
||||
Secure isolated command execution (--sandbox flag).
|
||||
Docs: https://docs.praison.ai/cli/sandbox-execution
|
||||
"""
|
||||
|
||||
from praisonai.cli.features import SandboxExecutorHandler
|
||||
from praisonai.cli.features.sandbox_executor import SandboxMode
|
||||
|
||||
# Initialize with basic sandbox
|
||||
handler = SandboxExecutorHandler(verbose=True)
|
||||
sandbox = handler.initialize(mode="basic")
|
||||
|
||||
print(f"Sandbox mode: {handler.get_mode()}")
|
||||
print(f"Sandbox enabled: {handler.is_enabled}")
|
||||
|
||||
# Execute a safe command
|
||||
print("\n=== Safe Command ===")
|
||||
result = handler.execute("echo 'Hello from sandbox!'")
|
||||
print(f"Success: {result.success}")
|
||||
print(f"Output: {result.stdout.strip()}")
|
||||
print(f"Sandboxed: {result.was_sandboxed}")
|
||||
|
||||
# Try a blocked command
|
||||
print("\n=== Blocked Command ===")
|
||||
result = handler.execute("rm -rf /")
|
||||
print(f"Success: {result.success}")
|
||||
print(f"Violations: {result.policy_violations}")
|
||||
|
||||
# Validate command before execution
|
||||
print("\n=== Command Validation ===")
|
||||
violations = handler.validate_command("sudo apt install something")
|
||||
if violations:
|
||||
print(f"Command blocked: {violations}")
|
||||
else:
|
||||
print("Command allowed")
|
||||
|
||||
# Available modes
|
||||
print("\n=== Available Modes ===")
|
||||
for mode in SandboxMode:
|
||||
print(f" {mode.value}")
|
||||
@@ -0,0 +1,51 @@
|
||||
"""
|
||||
Slash Commands Example for PraisonAI CLI.
|
||||
|
||||
Demonstrates interactive slash commands like /help, /cost, /model.
|
||||
Docs: https://docs.praison.ai/cli/slash-commands
|
||||
"""
|
||||
|
||||
from praisonai.cli.features import SlashCommandHandler
|
||||
from praisonai.cli.features.slash_commands import SlashCommand, CommandKind
|
||||
|
||||
# Create handler
|
||||
handler = SlashCommandHandler()
|
||||
|
||||
# Execute commands
|
||||
print("=== /help ===")
|
||||
result = handler.execute("/help")
|
||||
print(result.get("message", ""))
|
||||
|
||||
print("\n=== /cost ===")
|
||||
result = handler.execute("/cost")
|
||||
print(result.get("message", ""))
|
||||
|
||||
print("\n=== /model ===")
|
||||
result = handler.execute("/model")
|
||||
print(result.get("message", ""))
|
||||
|
||||
# Check if input is a command
|
||||
print("\n=== Command Detection ===")
|
||||
print(f"/help is command: {handler.is_command('/help')}")
|
||||
print(f"hello is command: {handler.is_command('hello')}")
|
||||
|
||||
# Get completions for auto-complete
|
||||
print("\n=== Completions for '/he' ===")
|
||||
completions = handler.get_completions("/he")
|
||||
print(completions)
|
||||
|
||||
# Register custom command
|
||||
def my_custom_handler(args, context):
|
||||
return {"type": "custom", "message": f"Custom command with args: {args}"}
|
||||
|
||||
custom_cmd = SlashCommand(
|
||||
name="mycommand",
|
||||
description="My custom command",
|
||||
handler=my_custom_handler,
|
||||
kind=CommandKind.ACTION,
|
||||
aliases=["mc"]
|
||||
)
|
||||
|
||||
handler.register(custom_cmd)
|
||||
result = handler.execute("/mycommand arg1 arg2")
|
||||
print(f"\n=== Custom Command ===\n{result.get('message', '')}")
|
||||
@@ -0,0 +1,34 @@
|
||||
"""
|
||||
Claude Code CLI Integration Example
|
||||
|
||||
Demonstrates how to use Claude Code CLI as an agent tool in PraisonAI.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
from praisonai.integrations import ClaudeCodeIntegration
|
||||
|
||||
async def main():
|
||||
# Create Claude Code integration
|
||||
claude = ClaudeCodeIntegration(
|
||||
workspace=".",
|
||||
output_format="json",
|
||||
skip_permissions=True,
|
||||
system_prompt="Be concise and follow best practices",
|
||||
allowed_tools=["Read", "Write", "Bash"]
|
||||
)
|
||||
|
||||
# Check availability
|
||||
print(f"Claude CLI available: {claude.is_available}")
|
||||
print(f"Claude SDK available: {claude.sdk_available}")
|
||||
|
||||
# Execute a coding task
|
||||
result = await claude.execute("List all Python files in the current directory")
|
||||
print(f"Result: {result}")
|
||||
|
||||
# Stream output
|
||||
print("\nStreaming output:")
|
||||
async for event in claude.stream("Explain the project structure"):
|
||||
print(event)
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,60 @@
|
||||
"""
|
||||
Code Editing Tools Example
|
||||
|
||||
Demonstrates how to use PraisonAI's code editing tools with SEARCH/REPLACE diffs.
|
||||
"""
|
||||
|
||||
from praisonai.code import (
|
||||
set_workspace,
|
||||
code_read_file,
|
||||
code_write_file,
|
||||
code_apply_diff,
|
||||
code_list_files,
|
||||
code_execute_command,
|
||||
CODE_TOOLS
|
||||
)
|
||||
|
||||
# Set workspace
|
||||
set_workspace(".")
|
||||
|
||||
# List files
|
||||
print("Files in current directory:")
|
||||
files = code_list_files(".", recursive=False)
|
||||
for f in files:
|
||||
print(f" {f}")
|
||||
|
||||
# Read a file
|
||||
print("\nReading example file:")
|
||||
try:
|
||||
content = code_read_file("README.md", start_line=1, end_line=10)
|
||||
print(content)
|
||||
except Exception as e:
|
||||
print(f" File not found: {e}")
|
||||
|
||||
# Write a file
|
||||
print("\nWriting test file:")
|
||||
code_write_file("test_output.txt", "Hello from PraisonAI Code Tools!")
|
||||
print(" Created test_output.txt")
|
||||
|
||||
# Apply a diff
|
||||
print("\nApplying diff:")
|
||||
diff = """
|
||||
<<<<<<< SEARCH
|
||||
Hello from PraisonAI Code Tools!
|
||||
=======
|
||||
Hello from PraisonAI Code Tools!
|
||||
This line was added by a diff.
|
||||
>>>>>>> REPLACE
|
||||
"""
|
||||
result = code_apply_diff("test_output.txt", diff)
|
||||
print(f" {result}")
|
||||
|
||||
# Execute command
|
||||
print("\nExecuting command:")
|
||||
output = code_execute_command("cat test_output.txt")
|
||||
print(f" {output}")
|
||||
|
||||
# Cleanup
|
||||
import os
|
||||
os.remove("test_output.txt")
|
||||
print("\nCleanup complete.")
|
||||
@@ -0,0 +1,36 @@
|
||||
"""
|
||||
Codex CLI Integration Example
|
||||
|
||||
Demonstrates how to use OpenAI Codex CLI as an agent tool in PraisonAI.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
from praisonai.integrations import CodexCLIIntegration
|
||||
|
||||
async def main():
|
||||
# Create Codex CLI integration
|
||||
codex = CodexCLIIntegration(
|
||||
workspace=".",
|
||||
full_auto=True,
|
||||
sandbox="danger-full-access",
|
||||
json_output=True
|
||||
)
|
||||
|
||||
# Check availability
|
||||
print(f"Codex CLI available: {codex.is_available}")
|
||||
print(f"Full Auto: {codex.full_auto}")
|
||||
print(f"Sandbox: {codex.sandbox}")
|
||||
|
||||
# Execute a coding task
|
||||
result = await codex.execute("List files in the current directory")
|
||||
print(f"Result: {result}")
|
||||
|
||||
# Stream JSON events
|
||||
print("\nStreaming events:")
|
||||
async for event in codex.stream("Explain the project"):
|
||||
event_type = event.get("type", "")
|
||||
if event_type == "item.completed":
|
||||
print(f" {event}")
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,37 @@
|
||||
"""
|
||||
Cursor CLI Integration Example
|
||||
|
||||
Demonstrates how to use Cursor CLI as an agent tool in PraisonAI.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
from praisonai.integrations import CursorCLIIntegration
|
||||
|
||||
async def main():
|
||||
# Create Cursor CLI integration
|
||||
cursor = CursorCLIIntegration(
|
||||
workspace=".",
|
||||
output_format="json",
|
||||
force=True,
|
||||
model="gpt-5",
|
||||
stream_partial=True
|
||||
)
|
||||
|
||||
# Check availability
|
||||
print(f"Cursor CLI available: {cursor.is_available}")
|
||||
print(f"Force mode: {cursor.force}")
|
||||
print(f"Model: {cursor.model}")
|
||||
|
||||
# Execute a coding task
|
||||
result = await cursor.execute("List files in the current directory")
|
||||
print(f"Result: {result}")
|
||||
|
||||
# Stream output
|
||||
print("\nStreaming output:")
|
||||
async for event in cursor.stream("Explain the project"):
|
||||
content = event.get("content", "")
|
||||
if content:
|
||||
print(content, end="", flush=True)
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,58 @@
|
||||
"""
|
||||
External Agents Integration Example
|
||||
|
||||
Demonstrates how to use all external CLI integrations as agent tools.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
from praisonai.integrations import (
|
||||
ClaudeCodeIntegration,
|
||||
GeminiCLIIntegration,
|
||||
CodexCLIIntegration,
|
||||
CursorCLIIntegration,
|
||||
get_available_integrations
|
||||
)
|
||||
|
||||
async def main():
|
||||
# Check availability of all integrations
|
||||
print("=== External CLI Integrations ===\n")
|
||||
|
||||
availability = get_available_integrations()
|
||||
print("Availability:")
|
||||
for name, avail in availability.items():
|
||||
status = "✅" if avail else "❌"
|
||||
print(f" {status} {name}")
|
||||
|
||||
# Create integrations
|
||||
print("\n--- Claude Code ---")
|
||||
claude = ClaudeCodeIntegration(workspace=".", skip_permissions=True)
|
||||
print(f"CLI: {claude.cli_command}")
|
||||
print(f"Available: {claude.is_available}")
|
||||
tool = claude.as_tool()
|
||||
print(f"Tool: {tool.__name__}")
|
||||
|
||||
print("\n--- Gemini CLI ---")
|
||||
gemini = GeminiCLIIntegration(workspace=".", model="gemini-2.5-flash")
|
||||
print(f"CLI: {gemini.cli_command}")
|
||||
print(f"Available: {gemini.is_available}")
|
||||
tool = gemini.as_tool()
|
||||
print(f"Tool: {tool.__name__}")
|
||||
|
||||
print("\n--- Codex CLI ---")
|
||||
codex = CodexCLIIntegration(workspace=".", full_auto=True)
|
||||
print(f"CLI: {codex.cli_command}")
|
||||
print(f"Available: {codex.is_available}")
|
||||
tool = codex.as_tool()
|
||||
print(f"Tool: {tool.__name__}")
|
||||
|
||||
print("\n--- Cursor CLI ---")
|
||||
cursor = CursorCLIIntegration(workspace=".", force=True)
|
||||
print(f"CLI: {cursor.cli_command}")
|
||||
print(f"Available: {cursor.is_available}")
|
||||
tool = cursor.as_tool()
|
||||
print(f"Tool: {tool.__name__}")
|
||||
|
||||
print("\n=== All integrations ready ===")
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,33 @@
|
||||
"""
|
||||
Gemini CLI Integration Example
|
||||
|
||||
Demonstrates how to use Gemini CLI as an agent tool in PraisonAI.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
from praisonai.integrations import GeminiCLIIntegration
|
||||
|
||||
async def main():
|
||||
# Create Gemini CLI integration
|
||||
gemini = GeminiCLIIntegration(
|
||||
workspace=".",
|
||||
model="gemini-2.5-flash",
|
||||
output_format="json",
|
||||
include_directories=["../lib", "../docs"]
|
||||
)
|
||||
|
||||
# Check availability
|
||||
print(f"Gemini CLI available: {gemini.is_available}")
|
||||
print(f"Model: {gemini.model}")
|
||||
|
||||
# Execute a coding task
|
||||
result = await gemini.execute("Analyze the code structure")
|
||||
print(f"Result: {result}")
|
||||
|
||||
# Execute with stats
|
||||
result, stats = await gemini.execute_with_stats("Explain main.py")
|
||||
print(f"Result: {result}")
|
||||
print(f"Stats: {stats}")
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -1,11 +1,48 @@
|
||||
"""
|
||||
AutoAgents Example
|
||||
|
||||
AutoAgents automatically creates and manages AI agents based on high-level instructions.
|
||||
It features dynamic agent count based on task complexity.
|
||||
|
||||
Documentation: https://docs.praison.ai/features/autoagents
|
||||
CLI Reference: https://docs.praison.ai/nocode/auto
|
||||
|
||||
New Features:
|
||||
- Dynamic agent count (1-4 based on task complexity)
|
||||
- Workflow patterns: sequential, parallel, routing, orchestrator-workers, evaluator-optimizer
|
||||
- Pattern recommendation based on task keywords
|
||||
- Tool preservation from LLM suggestions
|
||||
"""
|
||||
|
||||
from praisonaiagents import AutoAgents
|
||||
from praisonaiagents.tools import duckduckgo
|
||||
|
||||
# Basic usage - AutoAgents analyzes complexity and creates optimal agents
|
||||
agents = AutoAgents(
|
||||
instructions="Search for information about AI Agents",
|
||||
tools=[duckduckgo],
|
||||
process="sequential",
|
||||
verbose=True
|
||||
process="sequential", # or "hierarchical"
|
||||
verbose=True,
|
||||
max_agents=3 # Maximum number of agents to create
|
||||
)
|
||||
|
||||
agents.start()
|
||||
result = agents.start()
|
||||
print(result)
|
||||
|
||||
# =============================================================================
|
||||
# CLI Usage Examples (for reference)
|
||||
# =============================================================================
|
||||
#
|
||||
# # Auto-generate agents (dynamic count based on complexity)
|
||||
# praisonai --auto "Write a haiku about spring"
|
||||
# praisonai --auto "Research AI trends, analyze data, write report"
|
||||
#
|
||||
# # Auto-generate workflow with specific pattern
|
||||
# praisonai workflow auto "Research and write" --pattern sequential
|
||||
# praisonai workflow auto "Research from multiple sources" --pattern parallel
|
||||
# praisonai workflow auto "Comprehensive analysis" --pattern orchestrator-workers
|
||||
# praisonai workflow auto "Refine content quality" --pattern evaluator-optimizer
|
||||
#
|
||||
# # With framework selection
|
||||
# praisonai --framework crewai --auto "Create a movie script"
|
||||
# praisonai --framework autogen --auto "Create a movie script"
|
||||
@@ -0,0 +1,115 @@
|
||||
"""
|
||||
24/7 News Checker Agent Example
|
||||
|
||||
This example demonstrates how to create a continuously running agent
|
||||
that checks for AI news every hour and prints the results.
|
||||
|
||||
Usage:
|
||||
python news_checker_agent.py
|
||||
|
||||
The agent will:
|
||||
1. Run immediately on start
|
||||
2. Check for latest AI news every 1 hour
|
||||
3. Print results to console
|
||||
4. Continue running until stopped (Ctrl+C)
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
from datetime import datetime
|
||||
|
||||
# Add parent directory to path for imports
|
||||
sys.path.insert(0, os.path.join(os.path.dirname(__file__), '../../..'))
|
||||
|
||||
from praisonaiagents import Agent
|
||||
from praisonai.agent_scheduler import AgentScheduler
|
||||
from tools import search_tool
|
||||
|
||||
|
||||
def on_success(result):
|
||||
"""Callback function called when agent execution succeeds."""
|
||||
print("\n" + "="*80)
|
||||
print(f"✅ SUCCESS - {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}")
|
||||
print("="*80)
|
||||
print(result)
|
||||
print("="*80 + "\n")
|
||||
|
||||
|
||||
def on_failure(error):
|
||||
"""Callback function called when agent execution fails."""
|
||||
print("\n" + "="*80)
|
||||
print(f"❌ FAILURE - {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}")
|
||||
print("="*80)
|
||||
print(f"Error: {error}")
|
||||
print("="*80 + "\n")
|
||||
|
||||
|
||||
def main():
|
||||
"""Main function to set up and run the scheduled news checker agent."""
|
||||
|
||||
print("\n🤖 Starting 24/7 News Checker Agent")
|
||||
print("="*80)
|
||||
|
||||
# Create the agent with web search capability
|
||||
agent = Agent(
|
||||
name="NewsChecker",
|
||||
role="AI News Monitor",
|
||||
goal="Monitor and summarize the latest AI and technology news",
|
||||
backstory="""You are an expert AI news analyst who stays up-to-date
|
||||
with the latest developments in artificial intelligence, machine learning,
|
||||
and technology. You provide concise, accurate summaries of important news.""",
|
||||
tools=[search_tool],
|
||||
verbose=True
|
||||
)
|
||||
|
||||
# Define the task
|
||||
task = """Search for the latest AI news from today.
|
||||
Focus on major developments, breakthroughs, or announcements.
|
||||
Provide a summary of the top 3-5 most important news items.
|
||||
Include the source and key details for each item."""
|
||||
|
||||
# Create scheduler with callbacks
|
||||
scheduler = AgentScheduler(
|
||||
agent=agent,
|
||||
task=task,
|
||||
on_success=on_success,
|
||||
on_failure=on_failure
|
||||
)
|
||||
|
||||
print(f"Agent: {agent.name}")
|
||||
print(f"Task: {task}")
|
||||
print(f"Schedule: Every 1 hour")
|
||||
print("="*80)
|
||||
print("\n⏰ Starting scheduler... (Press Ctrl+C to stop)\n")
|
||||
|
||||
# Start the scheduler
|
||||
# - "hourly" or "*/1h" = every 1 hour
|
||||
# - run_immediately=True = run once before starting the schedule
|
||||
scheduler.start(
|
||||
schedule_expr="hourly", # Run every hour
|
||||
max_retries=3, # Retry up to 3 times on failure
|
||||
run_immediately=True # Run immediately on start
|
||||
)
|
||||
|
||||
try:
|
||||
# Keep the main thread alive
|
||||
while scheduler.is_running:
|
||||
import time
|
||||
time.sleep(1)
|
||||
|
||||
except KeyboardInterrupt:
|
||||
print("\n\n🛑 Stopping scheduler...")
|
||||
scheduler.stop()
|
||||
|
||||
# Print final statistics
|
||||
stats = scheduler.get_stats()
|
||||
print("\n📊 Final Statistics:")
|
||||
print(f" Total Executions: {stats['total_executions']}")
|
||||
print(f" Successful: {stats['successful_executions']}")
|
||||
print(f" Failed: {stats['failed_executions']}")
|
||||
print(f" Success Rate: {stats['success_rate']:.1f}%")
|
||||
print("\n✅ Agent stopped successfully\n")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,143 @@
|
||||
"""
|
||||
Live News Checker Agent - 24/7 Scheduled Agent
|
||||
|
||||
This example demonstrates a real working agent that checks AI news
|
||||
every hour using the AgentScheduler with actual API calls.
|
||||
|
||||
Requirements:
|
||||
- OPENAI_API_KEY environment variable set
|
||||
- praisonaiagents package installed
|
||||
|
||||
Usage:
|
||||
python news_checker_live.py
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
from datetime import datetime
|
||||
|
||||
# Add parent directory to path
|
||||
sys.path.insert(0, os.path.join(os.path.dirname(__file__), '../../..'))
|
||||
sys.path.insert(0, os.path.join(os.path.dirname(__file__), '../../../..'))
|
||||
|
||||
from praisonaiagents import Agent
|
||||
from praisonai.scheduler import AgentScheduler
|
||||
|
||||
# Import search_tool from root tools.py
|
||||
try:
|
||||
from tools import search_tool
|
||||
except ImportError:
|
||||
# Fallback: create inline search tool
|
||||
from duckduckgo_search import DDGS
|
||||
|
||||
def search_tool(query: str) -> list:
|
||||
"""Search the web using DuckDuckGo."""
|
||||
try:
|
||||
results = []
|
||||
ddgs = DDGS()
|
||||
for result in ddgs.text(keywords=query, max_results=5):
|
||||
results.append({
|
||||
"title": result.get("title", ""),
|
||||
"url": result.get("href", ""),
|
||||
"snippet": result.get("body", ""),
|
||||
})
|
||||
return results
|
||||
except Exception as e:
|
||||
print(f"Search error: {e}")
|
||||
return []
|
||||
|
||||
|
||||
def on_success(result):
|
||||
"""Callback when agent execution succeeds."""
|
||||
print("\n" + "="*80)
|
||||
print(f"✅ SUCCESS - {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}")
|
||||
print("="*80)
|
||||
print(result)
|
||||
print("="*80 + "\n")
|
||||
|
||||
|
||||
def on_failure(error):
|
||||
"""Callback when agent execution fails."""
|
||||
print("\n" + "="*80)
|
||||
print(f"❌ FAILURE - {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}")
|
||||
print("="*80)
|
||||
print(f"Error: {error}")
|
||||
print("="*80 + "\n")
|
||||
|
||||
|
||||
def main():
|
||||
"""Main function to run the scheduled news checker."""
|
||||
|
||||
# Check for API key
|
||||
if not os.getenv("OPENAI_API_KEY"):
|
||||
print("❌ Error: OPENAI_API_KEY environment variable not set")
|
||||
print("Please set it with: export OPENAI_API_KEY=your_key_here")
|
||||
sys.exit(1)
|
||||
|
||||
print("\n🤖 Starting 24/7 AI News Checker Agent")
|
||||
print("="*80)
|
||||
|
||||
# Create the agent with web search capability
|
||||
agent = Agent(
|
||||
name="AI News Monitor",
|
||||
role="Technology News Analyst",
|
||||
goal="Monitor and summarize the latest AI and technology news",
|
||||
backstory="""You are an expert AI news analyst who stays up-to-date
|
||||
with the latest developments in artificial intelligence, machine learning,
|
||||
and technology. You provide concise, accurate summaries of important news.""",
|
||||
tools=[search_tool],
|
||||
verbose=True,
|
||||
self_reflect=False # Disable reflection for faster execution
|
||||
)
|
||||
|
||||
# Define the task
|
||||
task = """Search for the latest AI news from today.
|
||||
Focus on major developments, breakthroughs, or announcements.
|
||||
Provide a summary of the top 3 most important news items.
|
||||
Include the source and key details for each item."""
|
||||
|
||||
# Create scheduler with callbacks
|
||||
scheduler = AgentScheduler(
|
||||
agent=agent,
|
||||
task=task,
|
||||
on_success=on_success,
|
||||
on_failure=on_failure
|
||||
)
|
||||
|
||||
print(f"Agent: {agent.name}")
|
||||
print(f"Task: {task}")
|
||||
print(f"Schedule: Every 1 hour")
|
||||
print("="*80)
|
||||
print("\n⏰ Starting scheduler... (Press Ctrl+C to stop)\n")
|
||||
|
||||
# Start the scheduler
|
||||
# For testing: use "*/2m" (every 2 minutes) instead of "hourly"
|
||||
# For production: use "hourly"
|
||||
scheduler.start(
|
||||
schedule_expr="*/2m", # Every 2 minutes for testing
|
||||
max_retries=3,
|
||||
run_immediately=True # Run once immediately
|
||||
)
|
||||
|
||||
try:
|
||||
# Keep the main thread alive
|
||||
while scheduler.is_running:
|
||||
import time
|
||||
time.sleep(1)
|
||||
|
||||
except KeyboardInterrupt:
|
||||
print("\n\n🛑 Stopping scheduler...")
|
||||
scheduler.stop()
|
||||
|
||||
# Print final statistics
|
||||
stats = scheduler.get_stats()
|
||||
print("\n📊 Final Statistics:")
|
||||
print(f" Total Executions: {stats['total_executions']}")
|
||||
print(f" Successful: {stats['successful_executions']}")
|
||||
print(f" Failed: {stats['failed_executions']}")
|
||||
print(f" Success Rate: {stats['success_rate']:.1f}%")
|
||||
print("\n✅ Agent stopped successfully\n")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,61 @@
|
||||
"""
|
||||
Simple test script for 24/7 Agent Scheduler
|
||||
|
||||
This script demonstrates the basic usage of AgentScheduler
|
||||
for running agents periodically.
|
||||
|
||||
Usage:
|
||||
python simple_test.py
|
||||
"""
|
||||
|
||||
import sys
|
||||
import os
|
||||
sys.path.insert(0, os.path.join(os.path.dirname(__file__), '../../..'))
|
||||
|
||||
from unittest.mock import Mock
|
||||
from praisonai.scheduler import AgentScheduler
|
||||
|
||||
# Create a mock agent for testing
|
||||
mock_agent = Mock()
|
||||
mock_agent.name = "TestAgent"
|
||||
mock_agent.start = Mock(return_value="Test execution successful!")
|
||||
|
||||
# Create scheduler
|
||||
scheduler = AgentScheduler(
|
||||
agent=mock_agent,
|
||||
task="Test task execution"
|
||||
)
|
||||
|
||||
print("🤖 Testing AgentScheduler")
|
||||
print("=" * 60)
|
||||
|
||||
# Test 1: Execute once
|
||||
print("\n✅ Test 1: Execute once")
|
||||
result = scheduler.execute_once()
|
||||
print(f"Result: {result}")
|
||||
|
||||
# Test 2: Get stats
|
||||
print("\n✅ Test 2: Get statistics")
|
||||
stats = scheduler.get_stats()
|
||||
print(f"Stats: {stats}")
|
||||
|
||||
# Test 3: Start scheduler (will run every 5 seconds)
|
||||
print("\n✅ Test 3: Start scheduler (*/5s)")
|
||||
scheduler.start("*/5s", run_immediately=True)
|
||||
|
||||
import time
|
||||
print("Running for 12 seconds...")
|
||||
time.sleep(12)
|
||||
|
||||
# Stop scheduler
|
||||
print("\n✅ Test 4: Stop scheduler")
|
||||
scheduler.stop()
|
||||
|
||||
# Final stats
|
||||
print("\n📊 Final Statistics:")
|
||||
final_stats = scheduler.get_stats()
|
||||
for key, value in final_stats.items():
|
||||
print(f" {key}: {value}")
|
||||
|
||||
print("\n✅ All tests passed!")
|
||||
print("=" * 60)
|
||||
@@ -0,0 +1,14 @@
|
||||
framework: praisonai
|
||||
|
||||
agents:
|
||||
- name: "5 Second Timer"
|
||||
role: "Timer Agent"
|
||||
instructions: "Display the current time"
|
||||
verbose: true
|
||||
|
||||
task: "Show current time"
|
||||
|
||||
schedule:
|
||||
interval: "*/5s"
|
||||
max_retries: 3
|
||||
run_immediately: true
|
||||
@@ -0,0 +1,14 @@
|
||||
framework: praisonai
|
||||
|
||||
agents:
|
||||
- name: "Counter Agent"
|
||||
role: "Counter"
|
||||
instructions: "Count from 1 to 5, incrementing by 1 each time. Just output the numbers clearly."
|
||||
verbose: true
|
||||
|
||||
task: "Print numbers 1, 2, 3, 4, 5 (one number per line)"
|
||||
|
||||
schedule:
|
||||
interval: "*/5s"
|
||||
max_retries: 3
|
||||
run_immediately: true
|
||||
@@ -0,0 +1,16 @@
|
||||
framework: praisonai
|
||||
|
||||
agents:
|
||||
- name: "Counter Agent with Limits"
|
||||
role: "Counter"
|
||||
instructions: "Count from 1 to 5, incrementing by 1 each time. Just output the numbers clearly."
|
||||
verbose: true
|
||||
|
||||
task: "Print numbers 1, 2, 3, 4, 5 (one number per line)"
|
||||
|
||||
schedule:
|
||||
interval: "*/5s" # Run every 5 seconds
|
||||
max_retries: 3 # Retry up to 3 times on failure
|
||||
run_immediately: true # Run once immediately on start
|
||||
timeout: 30 # Max 30 seconds per execution
|
||||
max_cost: 0.01 # Stop after spending $0.01 USD
|
||||
@@ -0,0 +1,15 @@
|
||||
framework: praisonai
|
||||
|
||||
agents:
|
||||
- name: "Budget Test Agent"
|
||||
role: "Tester"
|
||||
instructions: "Count from 1 to 3"
|
||||
verbose: true
|
||||
|
||||
task: "Print numbers 1, 2, 3"
|
||||
|
||||
schedule:
|
||||
interval: "*/3s"
|
||||
max_retries: 2
|
||||
run_immediately: true
|
||||
# No max_cost specified - should default to $1.00
|
||||
@@ -0,0 +1,73 @@
|
||||
"""
|
||||
Basic Agent Skills Usage Example
|
||||
|
||||
This example demonstrates how to use Agent Skills with PraisonAI Agents.
|
||||
Skills are modular capabilities that can be loaded and used by agents.
|
||||
"""
|
||||
|
||||
from praisonaiagents import Agent, SkillManager
|
||||
|
||||
# Example 1: Using SkillManager directly
|
||||
print("=" * 50)
|
||||
print("Example 1: Using SkillManager directly")
|
||||
print("=" * 50)
|
||||
|
||||
# Create a skill manager
|
||||
manager = SkillManager()
|
||||
|
||||
# Discover skills from a directory
|
||||
# This scans the directory for subdirectories containing SKILL.md files
|
||||
skill_count = manager.discover(["./pdf-processing"], include_defaults=False)
|
||||
print(f"Discovered {skill_count} skill(s)")
|
||||
|
||||
# List available skills
|
||||
for skill in manager:
|
||||
print(f" - {skill.properties.name}: {skill.properties.description[:50]}...")
|
||||
|
||||
# Generate prompt XML for system prompt injection
|
||||
prompt_xml = manager.to_prompt()
|
||||
print("\nGenerated prompt XML:")
|
||||
print(prompt_xml)
|
||||
|
||||
# Example 2: Using skills with an Agent
|
||||
print("\n" + "=" * 50)
|
||||
print("Example 2: Using skills with an Agent")
|
||||
print("=" * 50)
|
||||
|
||||
# Create an agent with skills
|
||||
# Skills are lazy-loaded only when accessed
|
||||
agent = Agent(
|
||||
name="PDF Assistant",
|
||||
instructions="You are a helpful assistant that can process PDF documents.",
|
||||
skills=["./pdf-processing"], # Direct skill paths
|
||||
)
|
||||
|
||||
# Access the skill manager through the agent
|
||||
if agent.skill_manager:
|
||||
print(f"Agent has {len(agent.skill_manager)} skill(s) loaded")
|
||||
|
||||
# Get the skills prompt for system prompt injection
|
||||
skills_prompt = agent.get_skills_prompt()
|
||||
print("\nAgent skills prompt:")
|
||||
print(skills_prompt)
|
||||
|
||||
# Example 3: Discovering skills from multiple directories
|
||||
print("\n" + "=" * 50)
|
||||
print("Example 3: Using skills_dirs for discovery")
|
||||
print("=" * 50)
|
||||
|
||||
# Create an agent that discovers skills from directories
|
||||
agent_with_discovery = Agent(
|
||||
name="Multi-Skill Agent",
|
||||
instructions="You are a versatile assistant with multiple skills.",
|
||||
skills_dirs=["./"], # Scan current directory for skill subdirectories
|
||||
)
|
||||
|
||||
if agent_with_discovery.skill_manager:
|
||||
print(f"Discovered {len(agent_with_discovery.skill_manager)} skill(s)")
|
||||
for name in agent_with_discovery.skill_manager.skill_names:
|
||||
print(f" - {name}")
|
||||
|
||||
print("\n" + "=" * 50)
|
||||
print("Examples complete!")
|
||||
print("=" * 50)
|
||||
@@ -0,0 +1,94 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Example: Creating Skills with PraisonAI CLI
|
||||
|
||||
This example demonstrates how to create Agent Skills using the CLI.
|
||||
"""
|
||||
|
||||
import subprocess
|
||||
import tempfile
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def example_create_skill_with_template():
|
||||
"""Create a skill using template mode (no API key required)."""
|
||||
print("=" * 60)
|
||||
print("Example 1: Create skill with template")
|
||||
print("=" * 60)
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
result = subprocess.run([
|
||||
"praisonai", "skills", "create",
|
||||
"--name", "my-template-skill",
|
||||
"--description", "A skill for processing text files",
|
||||
"--author", "example-team",
|
||||
"--license", "MIT",
|
||||
"--template",
|
||||
"--output-dir", tmpdir
|
||||
], capture_output=True, text=True)
|
||||
|
||||
print(result.stdout)
|
||||
if result.returncode == 0:
|
||||
skill_md = Path(tmpdir) / "my-template-skill" / "SKILL.md"
|
||||
print(f"\nGenerated SKILL.md:\n{skill_md.read_text()[:500]}...")
|
||||
else:
|
||||
print(f"Error: {result.stderr}")
|
||||
|
||||
|
||||
def example_create_skill_with_ai():
|
||||
"""Create a skill using AI generation (requires API key)."""
|
||||
print("\n" + "=" * 60)
|
||||
print("Example 2: Create skill with AI generation")
|
||||
print("=" * 60)
|
||||
|
||||
if not os.environ.get("OPENAI_API_KEY"):
|
||||
print("Skipping: OPENAI_API_KEY not set")
|
||||
return
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
result = subprocess.run([
|
||||
"praisonai", "skills", "create",
|
||||
"--name", "csv-analyzer",
|
||||
"--description", "Analyze CSV files and generate statistical reports",
|
||||
"--output-dir", tmpdir
|
||||
], capture_output=True, text=True)
|
||||
|
||||
print(result.stdout)
|
||||
if result.returncode == 0:
|
||||
skill_md = Path(tmpdir) / "csv-analyzer" / "SKILL.md"
|
||||
print(f"\nGenerated SKILL.md:\n{skill_md.read_text()[:800]}...")
|
||||
|
||||
# Check if script was generated
|
||||
script_path = Path(tmpdir) / "csv-analyzer" / "scripts" / "skill.py"
|
||||
if script_path.exists():
|
||||
print(f"\nGenerated skill.py:\n{script_path.read_text()[:500]}...")
|
||||
|
||||
|
||||
def example_create_skill_with_script():
|
||||
"""Create a skill with auto-generated Python script."""
|
||||
print("\n" + "=" * 60)
|
||||
print("Example 3: Create skill with script")
|
||||
print("=" * 60)
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
result = subprocess.run([
|
||||
"praisonai", "skills", "create",
|
||||
"--name", "data-processor",
|
||||
"--description", "Process and transform data files",
|
||||
"--template",
|
||||
"--script",
|
||||
"--output-dir", tmpdir
|
||||
], capture_output=True, text=True)
|
||||
|
||||
print(result.stdout)
|
||||
if result.returncode == 0:
|
||||
script_path = Path(tmpdir) / "data-processor" / "scripts" / "skill.py"
|
||||
if script_path.exists():
|
||||
print(f"\nGenerated skill.py:\n{script_path.read_text()}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
example_create_skill_with_template()
|
||||
example_create_skill_with_ai()
|
||||
example_create_skill_with_script()
|
||||
@@ -0,0 +1,163 @@
|
||||
"""
|
||||
Custom Skill Creation Example
|
||||
|
||||
This example demonstrates how to programmatically create and validate
|
||||
custom Agent Skills using PraisonAI Agents.
|
||||
"""
|
||||
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
from praisonaiagents.skills import (
|
||||
SkillManager,
|
||||
SkillLoader,
|
||||
validate,
|
||||
read_properties,
|
||||
)
|
||||
|
||||
|
||||
def create_custom_skill(
|
||||
name: str,
|
||||
description: str,
|
||||
instructions: str,
|
||||
base_dir: str = None
|
||||
) -> Path:
|
||||
"""Create a custom skill programmatically.
|
||||
|
||||
Args:
|
||||
name: Skill name (kebab-case)
|
||||
description: What the skill does
|
||||
instructions: Markdown instructions for the skill
|
||||
base_dir: Directory to create skill in (default: temp dir)
|
||||
|
||||
Returns:
|
||||
Path to the created skill directory
|
||||
"""
|
||||
if base_dir is None:
|
||||
base_dir = tempfile.mkdtemp()
|
||||
|
||||
skill_dir = Path(base_dir) / name
|
||||
skill_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Create SKILL.md
|
||||
skill_md_content = f"""---
|
||||
name: {name}
|
||||
description: {description}
|
||||
license: Apache-2.0
|
||||
metadata:
|
||||
author: custom
|
||||
version: "1.0"
|
||||
---
|
||||
|
||||
{instructions}
|
||||
"""
|
||||
(skill_dir / "SKILL.md").write_text(skill_md_content)
|
||||
|
||||
# Create optional directories
|
||||
(skill_dir / "scripts").mkdir(exist_ok=True)
|
||||
(skill_dir / "references").mkdir(exist_ok=True)
|
||||
(skill_dir / "assets").mkdir(exist_ok=True)
|
||||
|
||||
return skill_dir
|
||||
|
||||
|
||||
def main():
|
||||
print("=" * 60)
|
||||
print("Custom Skill Creation Example")
|
||||
print("=" * 60)
|
||||
|
||||
# Create a custom skill
|
||||
print("\n1. Creating a custom skill...")
|
||||
|
||||
skill_path = create_custom_skill(
|
||||
name="code-review",
|
||||
description="Review code for best practices, bugs, and improvements. Use when asked to review or analyze code quality.",
|
||||
instructions="""
|
||||
# Code Review Skill
|
||||
|
||||
## Overview
|
||||
This skill enables thorough code review with focus on:
|
||||
- Code quality and readability
|
||||
- Potential bugs and edge cases
|
||||
- Performance considerations
|
||||
- Security vulnerabilities
|
||||
- Best practices adherence
|
||||
|
||||
## Instructions
|
||||
|
||||
When reviewing code:
|
||||
|
||||
1. **Read the code carefully** - Understand the purpose and flow
|
||||
2. **Check for bugs** - Look for null checks, edge cases, error handling
|
||||
3. **Evaluate readability** - Variable names, comments, structure
|
||||
4. **Consider performance** - Time/space complexity, unnecessary operations
|
||||
5. **Security review** - Input validation, injection risks, data exposure
|
||||
6. **Suggest improvements** - Provide actionable recommendations
|
||||
|
||||
## Output Format
|
||||
|
||||
Structure your review as:
|
||||
- Summary: Brief overview of the code
|
||||
- Issues: List of problems found (severity: high/medium/low)
|
||||
- Suggestions: Recommended improvements
|
||||
- Positive aspects: What's done well
|
||||
"""
|
||||
)
|
||||
|
||||
print(f" Created skill at: {skill_path}")
|
||||
|
||||
# Validate the skill
|
||||
print("\n2. Validating the skill...")
|
||||
errors = validate(skill_path)
|
||||
|
||||
if errors:
|
||||
print(" Validation errors:")
|
||||
for error in errors:
|
||||
print(f" - {error}")
|
||||
else:
|
||||
print(" ✓ Skill is valid!")
|
||||
|
||||
# Load and inspect the skill
|
||||
print("\n3. Loading skill properties...")
|
||||
props = read_properties(skill_path)
|
||||
print(f" Name: {props.name}")
|
||||
print(f" Description: {props.description[:60]}...")
|
||||
print(f" License: {props.license}")
|
||||
|
||||
# Use with SkillLoader for progressive loading
|
||||
print("\n4. Using SkillLoader for progressive disclosure...")
|
||||
loader = SkillLoader()
|
||||
|
||||
# Level 1: Load metadata only
|
||||
skill = loader.load_metadata(str(skill_path))
|
||||
print(f" Level 1 - Metadata loaded: {skill.properties.name}")
|
||||
print(f" Instructions loaded: {skill.is_activated}")
|
||||
|
||||
# Level 2: Activate (load instructions)
|
||||
loader.activate(skill)
|
||||
print(f" Level 2 - Activated: {skill.is_activated}")
|
||||
print(f" Instructions preview: {skill.instructions[:100]}...")
|
||||
|
||||
# Use with SkillManager
|
||||
print("\n5. Using with SkillManager...")
|
||||
manager = SkillManager()
|
||||
manager.add_skill(str(skill_path))
|
||||
|
||||
# Generate prompt XML
|
||||
prompt = manager.to_prompt()
|
||||
print(" Generated prompt XML:")
|
||||
print(prompt)
|
||||
|
||||
# Clean up
|
||||
print("\n6. Cleanup...")
|
||||
import shutil
|
||||
shutil.rmtree(skill_path.parent)
|
||||
print(" Temporary files removed.")
|
||||
|
||||
print("\n" + "=" * 60)
|
||||
print("Example complete!")
|
||||
print("=" * 60)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,137 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Example: Multi-Agent CLI Usage
|
||||
|
||||
This example demonstrates how to use the praisonai agents command
|
||||
to define and run multiple agents from the command line.
|
||||
"""
|
||||
|
||||
import subprocess
|
||||
import os
|
||||
|
||||
|
||||
def example_single_agent():
|
||||
"""Run a single agent with a simple task."""
|
||||
print("=" * 60)
|
||||
print("Example 1: Single Agent")
|
||||
print("=" * 60)
|
||||
|
||||
result = subprocess.run([
|
||||
"praisonai", "agents", "run",
|
||||
"--agent", "assistant:Helper",
|
||||
"--task", "What is the capital of France?"
|
||||
], capture_output=True, text=True)
|
||||
|
||||
print(result.stdout)
|
||||
if result.returncode != 0:
|
||||
print(f"Error: {result.stderr}")
|
||||
|
||||
|
||||
def example_agent_with_tools():
|
||||
"""Run an agent with tools."""
|
||||
print("\n" + "=" * 60)
|
||||
print("Example 2: Agent with Tools")
|
||||
print("=" * 60)
|
||||
|
||||
result = subprocess.run([
|
||||
"praisonai", "agents", "run",
|
||||
"--agent", "researcher:Research Analyst:internet_search",
|
||||
"--task", "Find the latest news about renewable energy"
|
||||
], capture_output=True, text=True)
|
||||
|
||||
print(result.stdout)
|
||||
|
||||
|
||||
def example_multiple_agents_sequential():
|
||||
"""Run multiple agents in sequential mode."""
|
||||
print("\n" + "=" * 60)
|
||||
print("Example 3: Multiple Agents (Sequential)")
|
||||
print("=" * 60)
|
||||
|
||||
result = subprocess.run([
|
||||
"praisonai", "agents", "run",
|
||||
"--agent", "researcher:Research Analyst:internet_search",
|
||||
"--agent", "writer:Content Writer:write_file",
|
||||
"--task", "Research AI trends and write a brief summary"
|
||||
], capture_output=True, text=True)
|
||||
|
||||
print(result.stdout)
|
||||
|
||||
|
||||
def example_multiple_agents_parallel():
|
||||
"""Run multiple agents in parallel mode."""
|
||||
print("\n" + "=" * 60)
|
||||
print("Example 4: Multiple Agents (Parallel)")
|
||||
print("=" * 60)
|
||||
|
||||
result = subprocess.run([
|
||||
"praisonai", "agents", "run",
|
||||
"--agent", "analyst1:Market Analyst",
|
||||
"--agent", "analyst2:Tech Analyst",
|
||||
"--process", "parallel",
|
||||
"--task", "Analyze the current state of AI industry"
|
||||
], capture_output=True, text=True)
|
||||
|
||||
print(result.stdout)
|
||||
|
||||
|
||||
def example_with_custom_llm():
|
||||
"""Run agents with a custom LLM model."""
|
||||
print("\n" + "=" * 60)
|
||||
print("Example 5: Custom LLM Model")
|
||||
print("=" * 60)
|
||||
|
||||
result = subprocess.run([
|
||||
"praisonai", "agents", "run",
|
||||
"--agent", "coder:Developer:execute_command",
|
||||
"--llm", "gpt-4o-mini",
|
||||
"--task", "Write a simple hello world Python script"
|
||||
], capture_output=True, text=True)
|
||||
|
||||
print(result.stdout)
|
||||
|
||||
|
||||
def example_with_instructions():
|
||||
"""Run agents with additional instructions."""
|
||||
print("\n" + "=" * 60)
|
||||
print("Example 6: With Additional Instructions")
|
||||
print("=" * 60)
|
||||
|
||||
result = subprocess.run([
|
||||
"praisonai", "agents", "run",
|
||||
"--agent", "analyst:Data Analyst",
|
||||
"--instructions", "Be concise and provide only key insights",
|
||||
"--task", "Explain the benefits of using AI in healthcare"
|
||||
], capture_output=True, text=True)
|
||||
|
||||
print(result.stdout)
|
||||
|
||||
|
||||
def example_list_tools():
|
||||
"""List available tools for agents."""
|
||||
print("\n" + "=" * 60)
|
||||
print("Example 7: List Available Tools")
|
||||
print("=" * 60)
|
||||
|
||||
result = subprocess.run([
|
||||
"praisonai", "agents", "list"
|
||||
], capture_output=True, text=True)
|
||||
|
||||
print(result.stdout)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
# Check for API key
|
||||
if not os.environ.get("OPENAI_API_KEY"):
|
||||
print("Warning: OPENAI_API_KEY not set. Some examples may fail.")
|
||||
print()
|
||||
|
||||
# Run examples
|
||||
example_list_tools()
|
||||
example_single_agent()
|
||||
# Uncomment to run more examples:
|
||||
# example_agent_with_tools()
|
||||
# example_multiple_agents_sequential()
|
||||
# example_multiple_agents_parallel()
|
||||
# example_with_custom_llm()
|
||||
# example_with_instructions()
|
||||
@@ -0,0 +1,38 @@
|
||||
---
|
||||
name: pdf-processing
|
||||
description: Process and extract information from PDF documents. Use this skill when the user asks to read, analyze, or extract data from PDF files.
|
||||
license: Apache-2.0
|
||||
compatibility: Works with PraisonAI Agents
|
||||
metadata:
|
||||
author: praisonai
|
||||
version: "1.0"
|
||||
---
|
||||
|
||||
# PDF Processing Skill
|
||||
|
||||
## Overview
|
||||
|
||||
This skill enables agents to process PDF documents, extract text content, and analyze the information within them.
|
||||
|
||||
## When to Use
|
||||
|
||||
Activate this skill when:
|
||||
- User asks to read or analyze a PDF file
|
||||
- User needs to extract text from a PDF
|
||||
- User wants to summarize a PDF document
|
||||
- User needs to search for information within a PDF
|
||||
|
||||
## Instructions
|
||||
|
||||
1. First, verify the PDF file exists at the specified path
|
||||
2. Use appropriate tools to read the PDF content
|
||||
3. Extract text while preserving structure where possible
|
||||
4. For large PDFs, process in chunks to manage context
|
||||
5. Summarize or analyze based on user's specific request
|
||||
|
||||
## Best Practices
|
||||
|
||||
- Always confirm the file path before processing
|
||||
- Handle encrypted PDFs gracefully with appropriate error messages
|
||||
- For scanned PDFs, note that OCR may be required
|
||||
- Preserve important formatting like tables and lists
|
||||
@@ -0,0 +1,90 @@
|
||||
"""
|
||||
Unified Web Search Example
|
||||
|
||||
This example demonstrates the search_web tool that automatically tries
|
||||
multiple search providers with fallback support.
|
||||
|
||||
Search Provider Priority:
|
||||
1. Tavily (requires TAVILY_API_KEY + tavily-python)
|
||||
2. Exa (requires EXA_API_KEY + exa_py)
|
||||
3. You.com (requires YDC_API_KEY + youdotcom)
|
||||
4. DuckDuckGo (requires duckduckgo_search package, no API key)
|
||||
5. SearxNG (requires requests + running SearxNG instance)
|
||||
|
||||
Installation:
|
||||
pip install praisonaiagents duckduckgo_search
|
||||
|
||||
Usage:
|
||||
python web_search_example.py
|
||||
"""
|
||||
|
||||
from praisonaiagents import Agent
|
||||
from praisonaiagents.tools import search_web, get_available_providers
|
||||
|
||||
|
||||
def main():
|
||||
print("=" * 60)
|
||||
print("Unified Web Search - Provider Status")
|
||||
print("=" * 60)
|
||||
|
||||
# Check which providers are available
|
||||
providers = get_available_providers()
|
||||
for p in providers:
|
||||
status = "✓ Available" if p["available"] else f"✗ {p['reason']}"
|
||||
print(f" {p['name']:12} {status}")
|
||||
|
||||
print("\n" + "=" * 60)
|
||||
print("Example 1: Basic Search")
|
||||
print("=" * 60)
|
||||
|
||||
# Simple search - automatically uses best available provider
|
||||
results = search_web("Python programming best practices", max_results=3)
|
||||
|
||||
if results and "error" not in results[0]:
|
||||
print(f"\nFound {len(results)} results using {results[0].get('provider', 'unknown')}:\n")
|
||||
for r in results:
|
||||
print(f" Title: {r.get('title', 'N/A')[:60]}...")
|
||||
print(f" URL: {r.get('url', 'N/A')}")
|
||||
print()
|
||||
else:
|
||||
print(f"\nSearch failed: {results}")
|
||||
|
||||
print("=" * 60)
|
||||
print("Example 2: Specify Providers")
|
||||
print("=" * 60)
|
||||
|
||||
# Only try specific providers
|
||||
results = search_web(
|
||||
"machine learning tutorials",
|
||||
max_results=3,
|
||||
providers=["duckduckgo", "tavily"] # Try these in order
|
||||
)
|
||||
|
||||
if results and "error" not in results[0]:
|
||||
print(f"\nFound {len(results)} results using {results[0].get('provider', 'unknown')}:\n")
|
||||
for r in results:
|
||||
print(f" Title: {r.get('title', 'N/A')[:60]}...")
|
||||
print(f" URL: {r.get('url', 'N/A')}")
|
||||
print()
|
||||
else:
|
||||
print(f"\nSearch failed: {results}")
|
||||
|
||||
print("=" * 60)
|
||||
print("Example 3: With PraisonAI Agent")
|
||||
print("=" * 60)
|
||||
|
||||
# Use search_web as an agent tool
|
||||
agent = Agent(
|
||||
name="SearchAgent",
|
||||
role="Web Researcher",
|
||||
goal="Find and summarize information from the web",
|
||||
instructions="Search the web and provide concise summaries of findings.",
|
||||
tools=[search_web]
|
||||
)
|
||||
|
||||
result = agent.start("What are the top 3 AI trends in 2025?")
|
||||
print(f"\nAgent Response:\n{result}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -3,7 +3,7 @@ Praison AI Agents - A package for hierarchical AI agent task execution
|
||||
"""
|
||||
|
||||
# Apply warning patch BEFORE any imports to intercept warnings at the source
|
||||
from . import _warning_patch
|
||||
from . import _warning_patch # noqa: F401
|
||||
|
||||
# Import centralized logging configuration FIRST
|
||||
from . import _logging
|
||||
@@ -78,6 +78,19 @@ def __getattr__(name):
|
||||
elif name == "LineRange":
|
||||
from praisonaiagents.context.fast import LineRange
|
||||
return LineRange
|
||||
# Agent Skills support (lazy loaded for zero performance impact)
|
||||
elif name == "SkillManager":
|
||||
from praisonaiagents.skills import SkillManager
|
||||
return SkillManager
|
||||
elif name == "SkillProperties":
|
||||
from praisonaiagents.skills import SkillProperties
|
||||
return SkillProperties
|
||||
elif name == "SkillMetadata":
|
||||
from praisonaiagents.skills import SkillMetadata
|
||||
return SkillMetadata
|
||||
elif name == "SkillLoader":
|
||||
from praisonaiagents.skills import SkillLoader
|
||||
return SkillLoader
|
||||
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
||||
|
||||
# Planning mode support
|
||||
@@ -289,6 +302,23 @@ __all__ = [
|
||||
'ExpandResult'
|
||||
]
|
||||
|
||||
# Add workflow exports
|
||||
__all__.extend([
|
||||
'Workflow',
|
||||
'WorkflowStep',
|
||||
'WorkflowContext',
|
||||
'StepResult',
|
||||
'Route',
|
||||
'Parallel',
|
||||
'Loop',
|
||||
'Repeat',
|
||||
'route',
|
||||
'parallel',
|
||||
'loop',
|
||||
'repeat',
|
||||
'Pipeline'
|
||||
])
|
||||
|
||||
# Add MCP to __all__ if available
|
||||
if _mcp_available:
|
||||
__all__.append('MCP')
|
||||
@@ -305,6 +335,14 @@ __all__.extend([
|
||||
'LineRange'
|
||||
])
|
||||
|
||||
# Add Agent Skills exports (lazy loaded)
|
||||
__all__.extend([
|
||||
'SkillManager',
|
||||
'SkillProperties',
|
||||
'SkillMetadata',
|
||||
'SkillLoader'
|
||||
])
|
||||
|
||||
# AG-UI support (optional, lazy loaded)
|
||||
try:
|
||||
from praisonaiagents.ui.agui import AGUI
|
||||
|
||||
@@ -55,10 +55,10 @@ def _get_all_noisy_loggers() -> List[str]:
|
||||
|
||||
def _configure_loggers():
|
||||
"""Configure all loggers based on LOGLEVEL environment variable."""
|
||||
loglevel = os.environ.get('LOGLEVEL', 'INFO').upper()
|
||||
loglevel = os.environ.get('LOGLEVEL', 'WARNING').upper()
|
||||
|
||||
# When DEBUG is set, allow some HTTP logging for API endpoints
|
||||
if loglevel == 'DEBUG':
|
||||
# When DEBUG or INFO is set, allow some HTTP logging for API endpoints
|
||||
if loglevel in ('DEBUG', 'INFO'):
|
||||
allowed_debug_loggers = {"httpx", "httpx._client", "openai._client"}
|
||||
|
||||
for logger_name in _get_all_noisy_loggers():
|
||||
@@ -109,10 +109,10 @@ def configure_root_logger():
|
||||
"""Configure the root logger with RichHandler."""
|
||||
from rich.logging import RichHandler
|
||||
|
||||
loglevel = os.environ.get('LOGLEVEL', 'INFO').upper()
|
||||
loglevel = os.environ.get('LOGLEVEL', 'WARNING').upper()
|
||||
|
||||
logging.basicConfig(
|
||||
level=getattr(logging, loglevel, logging.INFO),
|
||||
level=getattr(logging, loglevel, logging.WARNING),
|
||||
format="%(asctime)s %(filename)s:%(lineno)d %(levelname)s %(message)s",
|
||||
datefmt="[%X]",
|
||||
handlers=[RichHandler(rich_tracebacks=True)],
|
||||
|
||||
@@ -8,9 +8,10 @@ import functools
|
||||
import sys
|
||||
|
||||
# Apply aggressive warning filters first
|
||||
warnings.filterwarnings("ignore", message=".*There is no current event loop.*", category=DeprecationWarning)
|
||||
warnings.filterwarnings("ignore", message=".*Pydantic serializer warnings.*")
|
||||
warnings.filterwarnings("ignore", message=".*PydanticSerializationUnexpectedValue.*")
|
||||
warnings.filterwarnings("ignore", message=".*Expected 9 fields but got.*")
|
||||
warnings.filterwarnings("ignore", message=".*Expected \\d+ fields but got.*")
|
||||
warnings.filterwarnings("ignore", message=".*Expected `StreamingChoices` but got.*")
|
||||
warnings.filterwarnings("ignore", message=".*serialized value may not be as expected.*")
|
||||
warnings.filterwarnings("ignore", message=".*Use 'content=<...>' to upload raw bytes/text content.*")
|
||||
@@ -27,9 +28,12 @@ SUPPRESSED_PATTERNS = [
|
||||
"The `dict` method is deprecated; use `model_dump` instead",
|
||||
"Pydantic serializer warnings",
|
||||
"PydanticSerializationUnexpectedValue",
|
||||
"Expected 9 fields but got 5 for type `Message`",
|
||||
"Expected", # Catches all "Expected N fields but got M" patterns
|
||||
"Expected `StreamingChoices` but got `Choices`",
|
||||
"serialized value may not be as expected"
|
||||
"serialized value may not be as expected",
|
||||
"Mixing V1 models and V2 models",
|
||||
"Please upgrade `Settings` to V2",
|
||||
"There is no current event loop"
|
||||
]
|
||||
|
||||
@functools.wraps(_original_warn)
|
||||
@@ -41,7 +45,7 @@ def _patched_warn(message, category=None, stacklevel=1, source=None):
|
||||
if pattern in msg_str:
|
||||
return
|
||||
|
||||
if category == UserWarning and "pydantic" in msg_str.lower():
|
||||
if category is UserWarning and "pydantic" in msg_str.lower():
|
||||
return
|
||||
|
||||
_original_warn(message, category, stacklevel, source)
|
||||
@@ -55,7 +59,7 @@ def _patched_warn_explicit(message, category, filename, lineno, module=None, reg
|
||||
if pattern in msg_str:
|
||||
return
|
||||
|
||||
if category == UserWarning and "pydantic" in msg_str.lower():
|
||||
if category is UserWarning and "pydantic" in msg_str.lower():
|
||||
return
|
||||
|
||||
if module and "pydantic" in str(module):
|
||||
|
||||
@@ -250,7 +250,9 @@ class Agent:
|
||||
fast_context_parallelism: int = 8,
|
||||
fast_context_timeout: float = 30.0,
|
||||
history_in_context: Optional[int] = None,
|
||||
auto_save: Optional[str] = None
|
||||
auto_save: Optional[str] = None,
|
||||
skills: Optional[List[str]] = None,
|
||||
skills_dirs: Optional[List[str]] = None
|
||||
):
|
||||
"""Initialize an Agent instance.
|
||||
|
||||
@@ -607,6 +609,12 @@ Your Goal: {self.goal}
|
||||
self.fast_context_timeout = fast_context_timeout
|
||||
self._fast_context_instance = None # Lazy loaded
|
||||
|
||||
# Agent Skills configuration (lazy loaded for zero performance impact)
|
||||
self._skills = skills
|
||||
self._skills_dirs = skills_dirs
|
||||
self._skill_manager = None # Lazy loaded
|
||||
self._skills_initialized = False
|
||||
|
||||
@property
|
||||
def console(self):
|
||||
"""Lazily initialize Rich Console only when needed."""
|
||||
@@ -626,6 +634,72 @@ Your Goal: {self.goal}
|
||||
def fast_context_path(self, value):
|
||||
self._fast_context_path = value
|
||||
|
||||
@property
|
||||
def skill_manager(self):
|
||||
"""Lazily initialize SkillManager only when skills are accessed."""
|
||||
if self._skill_manager is None and (self._skills or self._skills_dirs):
|
||||
from ..skills import SkillManager
|
||||
self._skill_manager = SkillManager()
|
||||
|
||||
# Add explicit skill paths
|
||||
if self._skills:
|
||||
for skill_path in self._skills:
|
||||
self._skill_manager.add_skill(skill_path)
|
||||
|
||||
# Discover skills from directories
|
||||
if self._skills_dirs:
|
||||
self._skill_manager.discover(self._skills_dirs, include_defaults=False)
|
||||
|
||||
self._skills_initialized = True
|
||||
|
||||
# Auto-add skill execution tools if not already present
|
||||
self._add_skill_tools()
|
||||
return self._skill_manager
|
||||
|
||||
def _add_skill_tools(self):
|
||||
"""Add tools required for skill execution (read_file, run_skill_script).
|
||||
|
||||
Uses lazy imports from praisonaiagents.tools to avoid performance impact
|
||||
when skills are not used.
|
||||
"""
|
||||
# Check if tools already include required capabilities
|
||||
tool_names = set()
|
||||
for tool in self.tools:
|
||||
if callable(tool) and hasattr(tool, '__name__'):
|
||||
tool_names.add(tool.__name__)
|
||||
elif hasattr(tool, 'name'):
|
||||
tool_names.add(tool.name)
|
||||
|
||||
# Add read_file if not present
|
||||
if 'read_file' not in tool_names:
|
||||
try:
|
||||
from ..tools import read_file
|
||||
self.tools.append(read_file)
|
||||
logging.debug("Added read_file tool for skill support")
|
||||
except ImportError:
|
||||
logging.warning("Could not import read_file tool for skills")
|
||||
|
||||
# Add run_skill_script from skill_tools module
|
||||
if 'run_skill_script' not in tool_names:
|
||||
try:
|
||||
from ..tools.skill_tools import create_skill_tools
|
||||
# Create skill tools with current working directory
|
||||
skill_tools = create_skill_tools()
|
||||
self.tools.append(skill_tools.run_skill_script)
|
||||
logging.debug("Added run_skill_script tool for skill support")
|
||||
except ImportError as e:
|
||||
logging.warning(f"Could not import skill_tools: {e}")
|
||||
|
||||
def get_skills_prompt(self) -> str:
|
||||
"""Get the XML prompt for available skills.
|
||||
|
||||
Returns:
|
||||
XML string with <available_skills> block, or empty string if no skills
|
||||
"""
|
||||
if self.skill_manager is None:
|
||||
return ""
|
||||
return self.skill_manager.to_prompt()
|
||||
|
||||
@property
|
||||
def _openai_client(self):
|
||||
"""Lazily initialize OpenAI client only when needed."""
|
||||
@@ -1284,6 +1358,13 @@ Your Goal: {self.goal}"""
|
||||
if self.verbose:
|
||||
self._display_memory_info()
|
||||
|
||||
# Add skills prompt if skills are configured
|
||||
if self._skills or self._skills_dirs:
|
||||
skills_prompt = self.get_skills_prompt()
|
||||
if skills_prompt:
|
||||
system_prompt += f"\n\n## Available Skills\n{skills_prompt}"
|
||||
system_prompt += "\n\nWhen a skill is relevant to the task, read its SKILL.md file to get detailed instructions. If the skill has scripts in its scripts/ directory, you can execute them using the execute_code or run_script tool."
|
||||
|
||||
# Add tool usage instructions if tools are available
|
||||
# Use provided tools or fall back to self.tools
|
||||
tools_to_use = tools if tools is not None else self.tools
|
||||
@@ -1538,13 +1619,13 @@ Your Goal: {self.goal}"""
|
||||
logging.debug(f"{self.name} executing tool {function_name} with arguments: {arguments}")
|
||||
|
||||
# Check if approval is required for this tool
|
||||
from ..approval import is_approval_required, console_approval_callback, get_risk_level, mark_approved, ApprovalDecision
|
||||
from ..approval import is_approval_required, console_approval_callback, get_risk_level, mark_approved, ApprovalDecision, get_approval_callback
|
||||
if is_approval_required(function_name):
|
||||
risk_level = get_risk_level(function_name)
|
||||
logging.info(f"Tool {function_name} requires approval (risk level: {risk_level})")
|
||||
logging.debug(f"Tool {function_name} requires approval (risk level: {risk_level})")
|
||||
|
||||
# Use global approval callback or default console callback
|
||||
callback = approval_callback or console_approval_callback
|
||||
callback = get_approval_callback() or console_approval_callback
|
||||
|
||||
try:
|
||||
decision = callback(function_name, arguments, risk_level)
|
||||
|
||||
@@ -43,6 +43,15 @@ def set_approval_callback(callback_fn: Callable):
|
||||
global approval_callback
|
||||
approval_callback = callback_fn
|
||||
|
||||
def get_approval_callback() -> Optional[Callable]:
|
||||
"""Get the current approval callback function.
|
||||
|
||||
Returns the custom callback if set, otherwise None.
|
||||
This should be used instead of directly accessing approval_callback
|
||||
to ensure the latest callback is always used.
|
||||
"""
|
||||
return approval_callback
|
||||
|
||||
def mark_approved(tool_name: str):
|
||||
"""Mark a tool as approved in the current context."""
|
||||
approved = _approved_context.get(set())
|
||||
@@ -90,7 +99,7 @@ def require_approval(risk_level: RiskLevel = "high"):
|
||||
except Exception as e:
|
||||
# Fallback to sync approval if async fails
|
||||
logging.warning(f"Async approval failed, using sync fallback: {e}")
|
||||
callback = approval_callback or console_approval_callback
|
||||
callback = get_approval_callback() or console_approval_callback
|
||||
decision = callback(tool_name, kwargs, risk_level)
|
||||
|
||||
if not decision.approved:
|
||||
@@ -198,7 +207,7 @@ async def request_approval(function_name: str, arguments: Dict[str, Any]) -> App
|
||||
risk_level = TOOL_RISK_LEVELS.get(function_name, "medium")
|
||||
|
||||
# Use custom callback if set, otherwise use console callback
|
||||
callback = approval_callback or console_approval_callback
|
||||
callback = get_approval_callback() or console_approval_callback
|
||||
|
||||
try:
|
||||
# Handle async callbacks
|
||||
|
||||
@@ -0,0 +1,68 @@
|
||||
"""
|
||||
PraisonAI Agents Evaluation Framework.
|
||||
|
||||
Provides comprehensive evaluation capabilities for AI agents with zero performance
|
||||
impact when not in use through lazy loading.
|
||||
|
||||
Evaluator Types:
|
||||
- AccuracyEvaluator: Compare output against expected output using LLM-as-judge
|
||||
- PerformanceEvaluator: Measure runtime and memory usage
|
||||
- ReliabilityEvaluator: Verify expected tool calls are made
|
||||
- CriteriaEvaluator: Evaluate against custom criteria
|
||||
|
||||
Example:
|
||||
>>> from praisonaiagents.eval import AccuracyEvaluator
|
||||
>>> evaluator = AccuracyEvaluator(
|
||||
... agent=my_agent,
|
||||
... input_text="What is 2+2?",
|
||||
... expected_output="4"
|
||||
... )
|
||||
>>> result = evaluator.run(print_summary=True)
|
||||
"""
|
||||
|
||||
__all__ = [
|
||||
"BaseEvaluator",
|
||||
"AccuracyEvaluator",
|
||||
"PerformanceEvaluator",
|
||||
"ReliabilityEvaluator",
|
||||
"CriteriaEvaluator",
|
||||
"AccuracyResult",
|
||||
"PerformanceResult",
|
||||
"ReliabilityResult",
|
||||
"CriteriaResult",
|
||||
"EvaluationScore",
|
||||
"PerformanceMetrics",
|
||||
"ToolCallResult",
|
||||
"CriteriaScore",
|
||||
]
|
||||
|
||||
_LAZY_IMPORTS = {
|
||||
"BaseEvaluator": ("base", "BaseEvaluator"),
|
||||
"AccuracyEvaluator": ("accuracy", "AccuracyEvaluator"),
|
||||
"PerformanceEvaluator": ("performance", "PerformanceEvaluator"),
|
||||
"ReliabilityEvaluator": ("reliability", "ReliabilityEvaluator"),
|
||||
"CriteriaEvaluator": ("criteria", "CriteriaEvaluator"),
|
||||
"AccuracyResult": ("results", "AccuracyResult"),
|
||||
"PerformanceResult": ("results", "PerformanceResult"),
|
||||
"ReliabilityResult": ("results", "ReliabilityResult"),
|
||||
"CriteriaResult": ("results", "CriteriaResult"),
|
||||
"EvaluationScore": ("results", "EvaluationScore"),
|
||||
"PerformanceMetrics": ("results", "PerformanceMetrics"),
|
||||
"ToolCallResult": ("results", "ToolCallResult"),
|
||||
"CriteriaScore": ("results", "CriteriaScore"),
|
||||
}
|
||||
|
||||
|
||||
def __getattr__(name: str):
|
||||
"""Lazy import mechanism for zero-cost imports when not used."""
|
||||
if name in _LAZY_IMPORTS:
|
||||
module_name, attr_name = _LAZY_IMPORTS[name]
|
||||
import importlib
|
||||
module = importlib.import_module(f".{module_name}", __name__)
|
||||
return getattr(module, attr_name)
|
||||
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
||||
|
||||
|
||||
def __dir__():
|
||||
"""Return list of available attributes for tab completion."""
|
||||
return __all__
|
||||
@@ -0,0 +1,239 @@
|
||||
"""
|
||||
Accuracy evaluator for PraisonAI Agents.
|
||||
|
||||
Evaluates agent output accuracy by comparing against expected output using LLM-as-judge.
|
||||
"""
|
||||
|
||||
import os
|
||||
import logging
|
||||
from typing import Any, Callable, List, Optional, Union, TYPE_CHECKING
|
||||
|
||||
from .base import BaseEvaluator
|
||||
from .results import AccuracyResult, EvaluationScore
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from ..agent.agent import Agent
|
||||
from ..agents.agents import PraisonAIAgents
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class AccuracyEvaluator(BaseEvaluator):
|
||||
"""
|
||||
Evaluates the accuracy of agent outputs against expected outputs.
|
||||
|
||||
Uses an LLM as a judge to score outputs on a scale of 1-10.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
agent: Optional[Union["Agent", "PraisonAIAgents"]] = None,
|
||||
func: Optional[Callable[..., str]] = None,
|
||||
input_text: str = "",
|
||||
expected_output: str = "",
|
||||
num_iterations: int = 1,
|
||||
model: Optional[str] = None,
|
||||
name: Optional[str] = None,
|
||||
save_results_path: Optional[str] = None,
|
||||
verbose: bool = False,
|
||||
):
|
||||
"""
|
||||
Initialize the accuracy evaluator.
|
||||
|
||||
Args:
|
||||
agent: Agent or PraisonAIAgents instance to evaluate
|
||||
func: Alternative callable that returns output string
|
||||
input_text: Input to provide to the agent/function
|
||||
expected_output: Expected output to compare against
|
||||
num_iterations: Number of evaluation iterations
|
||||
model: LLM model to use for judging (defaults to gpt-4o-mini)
|
||||
name: Name for this evaluation
|
||||
save_results_path: Path to save results
|
||||
verbose: Enable verbose output
|
||||
"""
|
||||
super().__init__(name=name, save_results_path=save_results_path, verbose=verbose)
|
||||
|
||||
self.agent = agent
|
||||
self.func = func
|
||||
self.input_text = input_text
|
||||
self.expected_output = expected_output
|
||||
self.num_iterations = num_iterations
|
||||
self.model = model or os.getenv("OPENAI_MODEL_NAME", "gpt-4o-mini")
|
||||
|
||||
if agent is None and func is None:
|
||||
raise ValueError("Either 'agent' or 'func' must be provided")
|
||||
|
||||
def _get_output(self) -> str:
|
||||
"""Get output from agent or function."""
|
||||
if self.func:
|
||||
return str(self.func(self.input_text))
|
||||
|
||||
if hasattr(self.agent, 'chat'):
|
||||
return str(self.agent.chat(self.input_text))
|
||||
elif hasattr(self.agent, 'start'):
|
||||
result = self.agent.start(self.input_text)
|
||||
if hasattr(result, 'raw'):
|
||||
return str(result.raw)
|
||||
return str(result)
|
||||
else:
|
||||
raise ValueError("Agent must have 'chat' or 'start' method")
|
||||
|
||||
def _judge_output(self, output: str) -> EvaluationScore:
|
||||
"""
|
||||
Use LLM to judge the output against expected output.
|
||||
|
||||
Args:
|
||||
output: The actual output to evaluate
|
||||
|
||||
Returns:
|
||||
EvaluationScore with score and reasoning
|
||||
"""
|
||||
try:
|
||||
from openai import OpenAI
|
||||
except ImportError:
|
||||
raise ImportError("OpenAI package required for accuracy evaluation. Install with: pip install openai")
|
||||
|
||||
client = OpenAI()
|
||||
|
||||
prompt = f"""You are an expert evaluator. Compare the actual output against the expected output and provide a score from 1-10.
|
||||
|
||||
Input: {self.input_text}
|
||||
|
||||
Expected Output: {self.expected_output}
|
||||
|
||||
Actual Output: {output}
|
||||
|
||||
Scoring Guidelines:
|
||||
- 10: Perfect match in meaning and completeness
|
||||
- 8-9: Very close, minor differences that don't affect correctness
|
||||
- 6-7: Mostly correct but missing some details or has minor errors
|
||||
- 4-5: Partially correct but significant issues
|
||||
- 2-3: Mostly incorrect but shows some understanding
|
||||
- 1: Completely wrong or irrelevant
|
||||
|
||||
Respond in this exact format:
|
||||
SCORE: [number 1-10]
|
||||
REASONING: [brief explanation]"""
|
||||
|
||||
response = client.chat.completions.create(
|
||||
model=self.model,
|
||||
messages=[{"role": "user", "content": prompt}],
|
||||
temperature=0.1,
|
||||
max_tokens=200
|
||||
)
|
||||
|
||||
response_text = response.choices[0].message.content or ""
|
||||
|
||||
score = 5.0
|
||||
reasoning = "Unable to parse response"
|
||||
|
||||
lines = response_text.strip().split('\n')
|
||||
for line in lines:
|
||||
if line.startswith('SCORE:'):
|
||||
try:
|
||||
score = float(line.replace('SCORE:', '').strip())
|
||||
score = max(1.0, min(10.0, score))
|
||||
except ValueError:
|
||||
pass
|
||||
elif line.startswith('REASONING:'):
|
||||
reasoning = line.replace('REASONING:', '').strip()
|
||||
|
||||
return EvaluationScore(
|
||||
score=score,
|
||||
reasoning=reasoning,
|
||||
input_text=self.input_text,
|
||||
output_text=output,
|
||||
expected_output=self.expected_output
|
||||
)
|
||||
|
||||
def run(self, print_summary: bool = False) -> AccuracyResult:
|
||||
"""
|
||||
Execute the accuracy evaluation.
|
||||
|
||||
Args:
|
||||
print_summary: Whether to print summary after evaluation
|
||||
|
||||
Returns:
|
||||
AccuracyResult with all evaluation scores
|
||||
"""
|
||||
self.before_run()
|
||||
|
||||
result = AccuracyResult(
|
||||
eval_id=self.eval_id,
|
||||
name=self.name
|
||||
)
|
||||
|
||||
for i in range(self.num_iterations):
|
||||
if self.verbose:
|
||||
logger.info(f"Running iteration {i + 1}/{self.num_iterations}")
|
||||
|
||||
try:
|
||||
output = self._get_output()
|
||||
score = self._judge_output(output)
|
||||
result.evaluations.append(score)
|
||||
|
||||
if self.verbose:
|
||||
logger.info(f" Score: {score.score}/10 - {score.reasoning}")
|
||||
except Exception as e:
|
||||
logger.error(f"Error in iteration {i + 1}: {e}")
|
||||
result.evaluations.append(EvaluationScore(
|
||||
score=0.0,
|
||||
reasoning=f"Error: {str(e)}",
|
||||
input_text=self.input_text,
|
||||
output_text="",
|
||||
expected_output=self.expected_output
|
||||
))
|
||||
|
||||
self.after_run(result)
|
||||
|
||||
if print_summary:
|
||||
result.print_summary()
|
||||
|
||||
return result
|
||||
|
||||
async def run_async(self, print_summary: bool = False) -> AccuracyResult:
|
||||
"""
|
||||
Execute the accuracy evaluation asynchronously.
|
||||
|
||||
Args:
|
||||
print_summary: Whether to print summary after evaluation
|
||||
|
||||
Returns:
|
||||
AccuracyResult with all evaluation scores
|
||||
"""
|
||||
await self.async_before_run()
|
||||
result = self.run(print_summary=False)
|
||||
await self.async_after_run(result)
|
||||
|
||||
if print_summary:
|
||||
result.print_summary()
|
||||
|
||||
return result
|
||||
|
||||
def evaluate_output(self, output: str, print_summary: bool = False) -> AccuracyResult:
|
||||
"""
|
||||
Evaluate a pre-generated output without running the agent.
|
||||
|
||||
Args:
|
||||
output: The output to evaluate
|
||||
print_summary: Whether to print summary after evaluation
|
||||
|
||||
Returns:
|
||||
AccuracyResult with evaluation score
|
||||
"""
|
||||
self.before_run()
|
||||
|
||||
result = AccuracyResult(
|
||||
eval_id=self.eval_id,
|
||||
name=self.name
|
||||
)
|
||||
|
||||
score = self._judge_output(output)
|
||||
result.evaluations.append(score)
|
||||
|
||||
self.after_run(result)
|
||||
|
||||
if print_summary:
|
||||
result.print_summary()
|
||||
|
||||
return result
|
||||
@@ -0,0 +1,143 @@
|
||||
"""
|
||||
Base evaluator class for PraisonAI Agents evaluation framework.
|
||||
|
||||
This module provides the abstract base class that all evaluators inherit from.
|
||||
"""
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import Any, Dict, Optional, Union, TYPE_CHECKING
|
||||
import uuid
|
||||
import logging
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from .results import AccuracyResult, PerformanceResult, ReliabilityResult, CriteriaResult
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class BaseEvaluator(ABC):
|
||||
"""
|
||||
Abstract base class for all evaluators in PraisonAI Agents.
|
||||
|
||||
Provides common functionality for evaluation lifecycle management,
|
||||
result storage, and telemetry integration.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
name: Optional[str] = None,
|
||||
save_results_path: Optional[str] = None,
|
||||
verbose: bool = False,
|
||||
):
|
||||
"""
|
||||
Initialize the base evaluator.
|
||||
|
||||
Args:
|
||||
name: Optional name for this evaluation run
|
||||
save_results_path: Optional file path to save results (supports {name}, {eval_id} placeholders)
|
||||
verbose: Enable verbose output during evaluation
|
||||
"""
|
||||
self.eval_id = str(uuid.uuid4())[:8]
|
||||
self.name = name or f"eval_{self.eval_id}"
|
||||
self.save_results_path = save_results_path
|
||||
self.verbose = verbose
|
||||
self._result = None
|
||||
|
||||
def before_run(self) -> None:
|
||||
"""
|
||||
Hook called before evaluation runs.
|
||||
Override in subclasses for custom pre-evaluation logic.
|
||||
"""
|
||||
if self.verbose:
|
||||
logger.info(f"Starting evaluation: {self.name} (ID: {self.eval_id})")
|
||||
|
||||
def after_run(self, result: Any) -> None:
|
||||
"""
|
||||
Hook called after evaluation completes.
|
||||
Override in subclasses for custom post-evaluation logic.
|
||||
|
||||
Args:
|
||||
result: The evaluation result
|
||||
"""
|
||||
if self.verbose:
|
||||
logger.info(f"Completed evaluation: {self.name}")
|
||||
|
||||
if self.save_results_path:
|
||||
self._save_result(result)
|
||||
|
||||
async def async_before_run(self) -> None:
|
||||
"""
|
||||
Async hook called before evaluation runs.
|
||||
Override in subclasses for custom async pre-evaluation logic.
|
||||
"""
|
||||
self.before_run()
|
||||
|
||||
async def async_after_run(self, result: Any) -> None:
|
||||
"""
|
||||
Async hook called after evaluation completes.
|
||||
Override in subclasses for custom async post-evaluation logic.
|
||||
|
||||
Args:
|
||||
result: The evaluation result
|
||||
"""
|
||||
self.after_run(result)
|
||||
|
||||
@abstractmethod
|
||||
def run(self, **kwargs) -> Any:
|
||||
"""
|
||||
Execute the evaluation synchronously.
|
||||
|
||||
Returns:
|
||||
Evaluation result (type depends on evaluator)
|
||||
"""
|
||||
pass
|
||||
|
||||
async def run_async(self, **kwargs) -> Any:
|
||||
"""
|
||||
Execute the evaluation asynchronously.
|
||||
Default implementation calls sync run().
|
||||
Override for true async evaluation.
|
||||
|
||||
Returns:
|
||||
Evaluation result (type depends on evaluator)
|
||||
"""
|
||||
return self.run(**kwargs)
|
||||
|
||||
def _save_result(self, result: Any) -> None:
|
||||
"""
|
||||
Save evaluation result to file.
|
||||
|
||||
Args:
|
||||
result: The evaluation result to save
|
||||
"""
|
||||
if not self.save_results_path:
|
||||
return
|
||||
|
||||
try:
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
file_path = self.save_results_path.format(
|
||||
name=self.name,
|
||||
eval_id=self.eval_id
|
||||
)
|
||||
|
||||
path = Path(file_path)
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
if hasattr(result, 'to_dict'):
|
||||
data = result.to_dict()
|
||||
elif hasattr(result, '__dict__'):
|
||||
data = result.__dict__
|
||||
else:
|
||||
data = {"result": str(result)}
|
||||
|
||||
path.write_text(json.dumps(data, indent=2, default=str))
|
||||
|
||||
if self.verbose:
|
||||
logger.info(f"Saved results to: {file_path}")
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to save results: {e}")
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return f"{self.__class__.__name__}(name={self.name!r}, eval_id={self.eval_id!r})"
|
||||
@@ -0,0 +1,289 @@
|
||||
"""
|
||||
Criteria evaluator for PraisonAI Agents.
|
||||
|
||||
Evaluates agent outputs against custom criteria using LLM-as-judge.
|
||||
"""
|
||||
|
||||
import os
|
||||
import logging
|
||||
from typing import Callable, List, Literal, Optional, Union, TYPE_CHECKING
|
||||
|
||||
from .base import BaseEvaluator
|
||||
from .results import CriteriaResult, CriteriaScore
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from ..agent.agent import Agent
|
||||
from ..agents.agents import PraisonAIAgents
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class CriteriaEvaluator(BaseEvaluator):
|
||||
"""
|
||||
Evaluates agent outputs against custom criteria.
|
||||
|
||||
Uses an LLM as a judge to score outputs based on user-defined criteria.
|
||||
Supports both numeric (1-10) and binary (pass/fail) scoring.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
criteria: str,
|
||||
agent: Optional[Union["Agent", "PraisonAIAgents"]] = None,
|
||||
func: Optional[Callable[..., str]] = None,
|
||||
input_text: str = "",
|
||||
scoring_type: Literal["numeric", "binary"] = "numeric",
|
||||
threshold: float = 7.0,
|
||||
num_iterations: int = 1,
|
||||
model: Optional[str] = None,
|
||||
on_fail: Optional[Callable[[CriteriaScore], None]] = None,
|
||||
name: Optional[str] = None,
|
||||
save_results_path: Optional[str] = None,
|
||||
verbose: bool = False,
|
||||
):
|
||||
"""
|
||||
Initialize the criteria evaluator.
|
||||
|
||||
Args:
|
||||
criteria: The criteria to evaluate against (e.g., "Response is helpful and accurate")
|
||||
agent: Agent or PraisonAIAgents instance to evaluate
|
||||
func: Alternative callable that returns output string
|
||||
input_text: Input to provide to the agent/function
|
||||
scoring_type: "numeric" (1-10 scale) or "binary" (pass/fail)
|
||||
threshold: Score threshold for passing (numeric mode only)
|
||||
num_iterations: Number of evaluation iterations
|
||||
model: LLM model to use for judging (defaults to gpt-4o-mini)
|
||||
on_fail: Callback function called when evaluation fails
|
||||
name: Name for this evaluation
|
||||
save_results_path: Path to save results
|
||||
verbose: Enable verbose output
|
||||
"""
|
||||
super().__init__(name=name, save_results_path=save_results_path, verbose=verbose)
|
||||
|
||||
self.criteria = criteria
|
||||
self.agent = agent
|
||||
self.func = func
|
||||
self.input_text = input_text
|
||||
self.scoring_type = scoring_type
|
||||
self.threshold = threshold
|
||||
self.num_iterations = num_iterations
|
||||
self.model = model or os.getenv("OPENAI_MODEL_NAME", "gpt-4o-mini")
|
||||
self.on_fail = on_fail
|
||||
|
||||
def _get_output(self) -> str:
|
||||
"""Get output from agent or function."""
|
||||
if self.func:
|
||||
return str(self.func(self.input_text))
|
||||
|
||||
if self.agent is None:
|
||||
raise ValueError("Either 'agent' or 'func' must be provided")
|
||||
|
||||
if hasattr(self.agent, 'chat'):
|
||||
return str(self.agent.chat(self.input_text))
|
||||
elif hasattr(self.agent, 'start'):
|
||||
result = self.agent.start(self.input_text)
|
||||
if hasattr(result, 'raw'):
|
||||
return str(result.raw)
|
||||
return str(result)
|
||||
else:
|
||||
raise ValueError("Agent must have 'chat' or 'start' method")
|
||||
|
||||
def _judge_output(self, output: str) -> CriteriaScore:
|
||||
"""
|
||||
Use LLM to judge the output against criteria.
|
||||
|
||||
Args:
|
||||
output: The output to evaluate
|
||||
|
||||
Returns:
|
||||
CriteriaScore with score, passed status, and reasoning
|
||||
"""
|
||||
try:
|
||||
from openai import OpenAI
|
||||
except ImportError:
|
||||
raise ImportError("OpenAI package required for criteria evaluation. Install with: pip install openai")
|
||||
|
||||
client = OpenAI()
|
||||
|
||||
if self.scoring_type == "binary":
|
||||
prompt = f"""You are an expert evaluator. Evaluate the following output against the given criteria.
|
||||
|
||||
Criteria: {self.criteria}
|
||||
|
||||
Output to evaluate:
|
||||
{output}
|
||||
|
||||
Determine if the output PASSES or FAILS the criteria.
|
||||
|
||||
Respond in this exact format:
|
||||
RESULT: [PASS or FAIL]
|
||||
REASONING: [brief explanation]"""
|
||||
else:
|
||||
prompt = f"""You are an expert evaluator. Evaluate the following output against the given criteria.
|
||||
|
||||
Criteria: {self.criteria}
|
||||
|
||||
Output to evaluate:
|
||||
{output}
|
||||
|
||||
Score the output from 1-10 based on how well it meets the criteria.
|
||||
- 10: Perfectly meets all criteria
|
||||
- 8-9: Meets criteria very well with minor issues
|
||||
- 6-7: Meets most criteria but has some gaps
|
||||
- 4-5: Partially meets criteria
|
||||
- 2-3: Barely meets criteria
|
||||
- 1: Does not meet criteria at all
|
||||
|
||||
Respond in this exact format:
|
||||
SCORE: [number 1-10]
|
||||
REASONING: [brief explanation]"""
|
||||
|
||||
response = client.chat.completions.create(
|
||||
model=self.model,
|
||||
messages=[{"role": "user", "content": prompt}],
|
||||
temperature=0.1,
|
||||
max_tokens=200
|
||||
)
|
||||
|
||||
response_text = response.choices[0].message.content or ""
|
||||
|
||||
score = 5.0
|
||||
passed = False
|
||||
reasoning = "Unable to parse response"
|
||||
|
||||
lines = response_text.strip().split('\n')
|
||||
for line in lines:
|
||||
if self.scoring_type == "binary":
|
||||
if line.startswith('RESULT:'):
|
||||
result_str = line.replace('RESULT:', '').strip().upper()
|
||||
passed = result_str == 'PASS'
|
||||
score = 10.0 if passed else 0.0
|
||||
else:
|
||||
if line.startswith('SCORE:'):
|
||||
try:
|
||||
score = float(line.replace('SCORE:', '').strip())
|
||||
score = max(1.0, min(10.0, score))
|
||||
passed = score >= self.threshold
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
if line.startswith('REASONING:'):
|
||||
reasoning = line.replace('REASONING:', '').strip()
|
||||
|
||||
return CriteriaScore(
|
||||
score=score,
|
||||
passed=passed,
|
||||
reasoning=reasoning,
|
||||
output_text=output,
|
||||
criteria=self.criteria
|
||||
)
|
||||
|
||||
def run(self, print_summary: bool = False) -> CriteriaResult:
|
||||
"""
|
||||
Execute the criteria evaluation.
|
||||
|
||||
Args:
|
||||
print_summary: Whether to print summary after evaluation
|
||||
|
||||
Returns:
|
||||
CriteriaResult with all evaluation scores
|
||||
"""
|
||||
self.before_run()
|
||||
|
||||
result = CriteriaResult(
|
||||
criteria=self.criteria,
|
||||
scoring_type=self.scoring_type,
|
||||
threshold=self.threshold,
|
||||
eval_id=self.eval_id,
|
||||
name=self.name
|
||||
)
|
||||
|
||||
for i in range(self.num_iterations):
|
||||
if self.verbose:
|
||||
logger.info(f"Running iteration {i + 1}/{self.num_iterations}")
|
||||
|
||||
try:
|
||||
output = self._get_output()
|
||||
score = self._judge_output(output)
|
||||
result.evaluations.append(score)
|
||||
|
||||
if self.verbose:
|
||||
status = "PASS" if score.passed else "FAIL"
|
||||
logger.info(f" {status} (Score: {score.score}) - {score.reasoning}")
|
||||
|
||||
if not score.passed and self.on_fail:
|
||||
self.on_fail(score)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error in iteration {i + 1}: {e}")
|
||||
error_score = CriteriaScore(
|
||||
score=0.0,
|
||||
passed=False,
|
||||
reasoning=f"Error: {str(e)}",
|
||||
output_text="",
|
||||
criteria=self.criteria
|
||||
)
|
||||
result.evaluations.append(error_score)
|
||||
|
||||
if self.on_fail:
|
||||
self.on_fail(error_score)
|
||||
|
||||
self.after_run(result)
|
||||
|
||||
if print_summary:
|
||||
result.print_summary()
|
||||
|
||||
return result
|
||||
|
||||
async def run_async(self, print_summary: bool = False) -> CriteriaResult:
|
||||
"""
|
||||
Execute the criteria evaluation asynchronously.
|
||||
|
||||
Args:
|
||||
print_summary: Whether to print summary after evaluation
|
||||
|
||||
Returns:
|
||||
CriteriaResult with all evaluation scores
|
||||
"""
|
||||
await self.async_before_run()
|
||||
result = self.run(print_summary=False)
|
||||
await self.async_after_run(result)
|
||||
|
||||
if print_summary:
|
||||
result.print_summary()
|
||||
|
||||
return result
|
||||
|
||||
def evaluate_output(self, output: str, print_summary: bool = False) -> CriteriaResult:
|
||||
"""
|
||||
Evaluate a pre-generated output without running the agent.
|
||||
|
||||
Args:
|
||||
output: The output to evaluate
|
||||
print_summary: Whether to print summary after evaluation
|
||||
|
||||
Returns:
|
||||
CriteriaResult with evaluation score
|
||||
"""
|
||||
self.before_run()
|
||||
|
||||
result = CriteriaResult(
|
||||
criteria=self.criteria,
|
||||
scoring_type=self.scoring_type,
|
||||
threshold=self.threshold,
|
||||
eval_id=self.eval_id,
|
||||
name=self.name
|
||||
)
|
||||
|
||||
score = self._judge_output(output)
|
||||
result.evaluations.append(score)
|
||||
|
||||
if not score.passed and self.on_fail:
|
||||
self.on_fail(score)
|
||||
|
||||
self.after_run(result)
|
||||
|
||||
if print_summary:
|
||||
result.print_summary()
|
||||
|
||||
return result
|
||||
@@ -0,0 +1,184 @@
|
||||
"""
|
||||
Performance evaluator for PraisonAI Agents.
|
||||
|
||||
Measures runtime and memory usage of agent executions or functions.
|
||||
"""
|
||||
|
||||
import time
|
||||
import logging
|
||||
from typing import Callable, Optional, TYPE_CHECKING
|
||||
|
||||
from .base import BaseEvaluator
|
||||
from .results import PerformanceResult, PerformanceMetrics
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from ..agent.agent import Agent
|
||||
from ..agents.agents import PraisonAIAgents
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class PerformanceEvaluator(BaseEvaluator):
|
||||
"""
|
||||
Evaluates the performance of agent executions or functions.
|
||||
|
||||
Measures runtime and memory usage across multiple iterations.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
func: Optional[Callable[[], None]] = None,
|
||||
agent: Optional["Agent"] = None,
|
||||
input_text: str = "",
|
||||
num_iterations: int = 10,
|
||||
warmup_runs: int = 2,
|
||||
track_memory: bool = True,
|
||||
name: Optional[str] = None,
|
||||
save_results_path: Optional[str] = None,
|
||||
verbose: bool = False,
|
||||
):
|
||||
"""
|
||||
Initialize the performance evaluator.
|
||||
|
||||
Args:
|
||||
func: Callable to benchmark (takes no arguments)
|
||||
agent: Agent to benchmark (will call chat with input_text)
|
||||
input_text: Input text for agent benchmarking
|
||||
num_iterations: Number of benchmark iterations
|
||||
warmup_runs: Number of warmup runs before measurement
|
||||
track_memory: Whether to track memory usage
|
||||
name: Name for this evaluation
|
||||
save_results_path: Path to save results
|
||||
verbose: Enable verbose output
|
||||
"""
|
||||
super().__init__(name=name, save_results_path=save_results_path, verbose=verbose)
|
||||
|
||||
self.func = func
|
||||
self.agent = agent
|
||||
self.input_text = input_text
|
||||
self.num_iterations = num_iterations
|
||||
self.warmup_runs = warmup_runs
|
||||
self.track_memory = track_memory
|
||||
|
||||
if func is None and agent is None:
|
||||
raise ValueError("Either 'func' or 'agent' must be provided")
|
||||
|
||||
def _get_memory_usage(self) -> float:
|
||||
"""Get current memory usage in MB."""
|
||||
if not self.track_memory:
|
||||
return 0.0
|
||||
|
||||
try:
|
||||
import tracemalloc
|
||||
current, _ = tracemalloc.get_traced_memory()
|
||||
return current / (1024 * 1024)
|
||||
except Exception:
|
||||
return 0.0
|
||||
|
||||
def _execute(self) -> None:
|
||||
"""Execute the function or agent."""
|
||||
if self.func:
|
||||
self.func()
|
||||
elif self.agent:
|
||||
if hasattr(self.agent, 'chat'):
|
||||
self.agent.chat(self.input_text)
|
||||
elif hasattr(self.agent, 'start'):
|
||||
self.agent.start(self.input_text)
|
||||
else:
|
||||
raise ValueError("Agent must have 'chat' or 'start' method")
|
||||
|
||||
def run(self, print_summary: bool = False) -> PerformanceResult:
|
||||
"""
|
||||
Execute the performance evaluation.
|
||||
|
||||
Args:
|
||||
print_summary: Whether to print summary after evaluation
|
||||
|
||||
Returns:
|
||||
PerformanceResult with all performance metrics
|
||||
"""
|
||||
self.before_run()
|
||||
|
||||
result = PerformanceResult(
|
||||
warmup_runs=self.warmup_runs,
|
||||
eval_id=self.eval_id,
|
||||
name=self.name
|
||||
)
|
||||
|
||||
if self.track_memory:
|
||||
try:
|
||||
import tracemalloc
|
||||
tracemalloc.start()
|
||||
except Exception as e:
|
||||
logger.warning(f"Could not start memory tracking: {e}")
|
||||
self.track_memory = False
|
||||
|
||||
if self.verbose:
|
||||
logger.info(f"Running {self.warmup_runs} warmup iterations...")
|
||||
|
||||
for i in range(self.warmup_runs):
|
||||
try:
|
||||
self._execute()
|
||||
except Exception as e:
|
||||
logger.warning(f"Warmup iteration {i + 1} failed: {e}")
|
||||
|
||||
if self.verbose:
|
||||
logger.info(f"Running {self.num_iterations} benchmark iterations...")
|
||||
|
||||
for i in range(self.num_iterations):
|
||||
memory_before = self._get_memory_usage()
|
||||
|
||||
start_time = time.perf_counter()
|
||||
try:
|
||||
self._execute()
|
||||
success = True
|
||||
except Exception as e:
|
||||
logger.error(f"Iteration {i + 1} failed: {e}")
|
||||
success = False
|
||||
end_time = time.perf_counter()
|
||||
|
||||
memory_after = self._get_memory_usage()
|
||||
|
||||
if success:
|
||||
metrics = PerformanceMetrics(
|
||||
run_time=end_time - start_time,
|
||||
memory_usage=max(0, memory_after - memory_before),
|
||||
iteration=i + 1
|
||||
)
|
||||
result.metrics.append(metrics)
|
||||
|
||||
if self.verbose:
|
||||
logger.info(f" Iteration {i + 1}: {metrics.run_time:.4f}s, {metrics.memory_usage:.2f}MB")
|
||||
|
||||
if self.track_memory:
|
||||
try:
|
||||
import tracemalloc
|
||||
tracemalloc.stop()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
self.after_run(result)
|
||||
|
||||
if print_summary:
|
||||
result.print_summary()
|
||||
|
||||
return result
|
||||
|
||||
async def run_async(self, print_summary: bool = False) -> PerformanceResult:
|
||||
"""
|
||||
Execute the performance evaluation asynchronously.
|
||||
|
||||
Args:
|
||||
print_summary: Whether to print summary after evaluation
|
||||
|
||||
Returns:
|
||||
PerformanceResult with all performance metrics
|
||||
"""
|
||||
await self.async_before_run()
|
||||
result = self.run(print_summary=False)
|
||||
await self.async_after_run(result)
|
||||
|
||||
if print_summary:
|
||||
result.print_summary()
|
||||
|
||||
return result
|
||||
@@ -0,0 +1,225 @@
|
||||
"""
|
||||
Reliability evaluator for PraisonAI Agents.
|
||||
|
||||
Evaluates agent reliability by verifying expected tool calls are made.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from typing import Dict, List, Optional, Set, Any, TYPE_CHECKING
|
||||
|
||||
from .base import BaseEvaluator
|
||||
from .results import ReliabilityResult, ToolCallResult
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from ..agent.agent import Agent
|
||||
from ..agents.agents import PraisonAIAgents
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ReliabilityEvaluator(BaseEvaluator):
|
||||
"""
|
||||
Evaluates the reliability of agent tool usage.
|
||||
|
||||
Verifies that expected tools are called during agent execution.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
agent: Optional["Agent"] = None,
|
||||
input_text: str = "",
|
||||
expected_tools: Optional[List[str]] = None,
|
||||
forbidden_tools: Optional[List[str]] = None,
|
||||
name: Optional[str] = None,
|
||||
save_results_path: Optional[str] = None,
|
||||
verbose: bool = False,
|
||||
):
|
||||
"""
|
||||
Initialize the reliability evaluator.
|
||||
|
||||
Args:
|
||||
agent: Agent to evaluate
|
||||
input_text: Input to provide to the agent
|
||||
expected_tools: List of tool names that should be called
|
||||
forbidden_tools: List of tool names that should NOT be called
|
||||
name: Name for this evaluation
|
||||
save_results_path: Path to save results
|
||||
verbose: Enable verbose output
|
||||
"""
|
||||
super().__init__(name=name, save_results_path=save_results_path, verbose=verbose)
|
||||
|
||||
self.agent = agent
|
||||
self.input_text = input_text
|
||||
self.expected_tools = set(expected_tools or [])
|
||||
self.forbidden_tools = set(forbidden_tools or [])
|
||||
|
||||
if agent is None:
|
||||
raise ValueError("'agent' must be provided")
|
||||
|
||||
def _extract_tool_calls(self, response: Any) -> Set[str]:
|
||||
"""
|
||||
Extract tool names from agent response.
|
||||
|
||||
Args:
|
||||
response: Agent response object
|
||||
|
||||
Returns:
|
||||
Set of tool names that were called
|
||||
"""
|
||||
tool_calls = set()
|
||||
|
||||
if hasattr(response, 'tool_calls') and response.tool_calls:
|
||||
for tc in response.tool_calls:
|
||||
if hasattr(tc, 'name'):
|
||||
tool_calls.add(tc.name)
|
||||
elif hasattr(tc, 'function') and hasattr(tc.function, 'name'):
|
||||
tool_calls.add(tc.function.name)
|
||||
|
||||
if hasattr(self.agent, 'chat_history'):
|
||||
for msg in self.agent.chat_history:
|
||||
if isinstance(msg, dict):
|
||||
if msg.get('role') == 'assistant' and 'tool_calls' in msg:
|
||||
for tc in msg.get('tool_calls', []):
|
||||
if isinstance(tc, dict) and 'function' in tc:
|
||||
tool_calls.add(tc['function'].get('name', ''))
|
||||
elif msg.get('role') == 'tool':
|
||||
tool_name = msg.get('name', '')
|
||||
if tool_name:
|
||||
tool_calls.add(tool_name)
|
||||
|
||||
return tool_calls
|
||||
|
||||
def run(self, print_summary: bool = False) -> ReliabilityResult:
|
||||
"""
|
||||
Execute the reliability evaluation.
|
||||
|
||||
Args:
|
||||
print_summary: Whether to print summary after evaluation
|
||||
|
||||
Returns:
|
||||
ReliabilityResult with tool call verification results
|
||||
"""
|
||||
self.before_run()
|
||||
|
||||
result = ReliabilityResult(
|
||||
eval_id=self.eval_id,
|
||||
name=self.name
|
||||
)
|
||||
|
||||
try:
|
||||
if hasattr(self.agent, 'chat'):
|
||||
response = self.agent.chat(self.input_text)
|
||||
elif hasattr(self.agent, 'start'):
|
||||
response = self.agent.start(self.input_text)
|
||||
else:
|
||||
raise ValueError("Agent must have 'chat' or 'start' method")
|
||||
|
||||
actual_tools = self._extract_tool_calls(response)
|
||||
|
||||
if self.verbose:
|
||||
logger.info(f"Expected tools: {self.expected_tools}")
|
||||
logger.info(f"Actual tools called: {actual_tools}")
|
||||
|
||||
for tool in self.expected_tools:
|
||||
was_called = tool in actual_tools
|
||||
result.tool_results.append(ToolCallResult(
|
||||
tool_name=tool,
|
||||
expected=True,
|
||||
actual=was_called,
|
||||
passed=was_called
|
||||
))
|
||||
|
||||
for tool in self.forbidden_tools:
|
||||
was_called = tool in actual_tools
|
||||
result.tool_results.append(ToolCallResult(
|
||||
tool_name=tool,
|
||||
expected=False,
|
||||
actual=was_called,
|
||||
passed=not was_called
|
||||
))
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error during reliability evaluation: {e}")
|
||||
for tool in self.expected_tools:
|
||||
result.tool_results.append(ToolCallResult(
|
||||
tool_name=tool,
|
||||
expected=True,
|
||||
actual=False,
|
||||
passed=False,
|
||||
metadata={"error": str(e)}
|
||||
))
|
||||
|
||||
self.after_run(result)
|
||||
|
||||
if print_summary:
|
||||
result.print_summary()
|
||||
|
||||
return result
|
||||
|
||||
async def run_async(self, print_summary: bool = False) -> ReliabilityResult:
|
||||
"""
|
||||
Execute the reliability evaluation asynchronously.
|
||||
|
||||
Args:
|
||||
print_summary: Whether to print summary after evaluation
|
||||
|
||||
Returns:
|
||||
ReliabilityResult with tool call verification results
|
||||
"""
|
||||
await self.async_before_run()
|
||||
result = self.run(print_summary=False)
|
||||
await self.async_after_run(result)
|
||||
|
||||
if print_summary:
|
||||
result.print_summary()
|
||||
|
||||
return result
|
||||
|
||||
def evaluate_tool_calls(
|
||||
self,
|
||||
actual_tools: List[str],
|
||||
print_summary: bool = False
|
||||
) -> ReliabilityResult:
|
||||
"""
|
||||
Evaluate a pre-recorded list of tool calls without running the agent.
|
||||
|
||||
Args:
|
||||
actual_tools: List of tool names that were called
|
||||
print_summary: Whether to print summary after evaluation
|
||||
|
||||
Returns:
|
||||
ReliabilityResult with tool call verification results
|
||||
"""
|
||||
self.before_run()
|
||||
|
||||
result = ReliabilityResult(
|
||||
eval_id=self.eval_id,
|
||||
name=self.name
|
||||
)
|
||||
|
||||
actual_set = set(actual_tools)
|
||||
|
||||
for tool in self.expected_tools:
|
||||
was_called = tool in actual_set
|
||||
result.tool_results.append(ToolCallResult(
|
||||
tool_name=tool,
|
||||
expected=True,
|
||||
actual=was_called,
|
||||
passed=was_called
|
||||
))
|
||||
|
||||
for tool in self.forbidden_tools:
|
||||
was_called = tool in actual_set
|
||||
result.tool_results.append(ToolCallResult(
|
||||
tool_name=tool,
|
||||
expected=False,
|
||||
actual=was_called,
|
||||
passed=not was_called
|
||||
))
|
||||
|
||||
self.after_run(result)
|
||||
|
||||
if print_summary:
|
||||
result.print_summary()
|
||||
|
||||
return result
|
||||
@@ -0,0 +1,463 @@
|
||||
"""
|
||||
Result dataclasses for PraisonAI Agents evaluation framework.
|
||||
|
||||
This module provides structured result types for different evaluation methods.
|
||||
"""
|
||||
|
||||
from dataclasses import dataclass, field, asdict
|
||||
from typing import List, Optional, Dict, Any
|
||||
from datetime import datetime
|
||||
import statistics
|
||||
import json
|
||||
|
||||
|
||||
@dataclass
|
||||
class EvaluationScore:
|
||||
"""Individual evaluation score with reasoning."""
|
||||
score: float
|
||||
reasoning: str
|
||||
input_text: str
|
||||
output_text: str
|
||||
expected_output: Optional[str] = None
|
||||
timestamp: str = field(default_factory=lambda: datetime.now().isoformat())
|
||||
metadata: Dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
|
||||
@dataclass
|
||||
class AccuracyResult:
|
||||
"""Result from accuracy evaluation comparing output to expected output."""
|
||||
evaluations: List[EvaluationScore] = field(default_factory=list)
|
||||
eval_id: str = ""
|
||||
name: str = ""
|
||||
|
||||
@property
|
||||
def scores(self) -> List[float]:
|
||||
"""Get list of all scores."""
|
||||
return [e.score for e in self.evaluations]
|
||||
|
||||
@property
|
||||
def avg_score(self) -> float:
|
||||
"""Calculate average score."""
|
||||
if not self.scores:
|
||||
return 0.0
|
||||
return statistics.mean(self.scores)
|
||||
|
||||
@property
|
||||
def min_score(self) -> float:
|
||||
"""Get minimum score."""
|
||||
if not self.scores:
|
||||
return 0.0
|
||||
return min(self.scores)
|
||||
|
||||
@property
|
||||
def max_score(self) -> float:
|
||||
"""Get maximum score."""
|
||||
if not self.scores:
|
||||
return 0.0
|
||||
return max(self.scores)
|
||||
|
||||
@property
|
||||
def std_dev(self) -> float:
|
||||
"""Calculate standard deviation of scores."""
|
||||
if len(self.scores) < 2:
|
||||
return 0.0
|
||||
return statistics.stdev(self.scores)
|
||||
|
||||
@property
|
||||
def passed(self) -> bool:
|
||||
"""Check if evaluation passed (avg score >= 7)."""
|
||||
return self.avg_score >= 7.0
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
"""Convert result to dictionary."""
|
||||
return {
|
||||
"eval_id": self.eval_id,
|
||||
"name": self.name,
|
||||
"evaluations": [asdict(e) for e in self.evaluations],
|
||||
"avg_score": self.avg_score,
|
||||
"min_score": self.min_score,
|
||||
"max_score": self.max_score,
|
||||
"std_dev": self.std_dev,
|
||||
"passed": self.passed,
|
||||
"num_evaluations": len(self.evaluations)
|
||||
}
|
||||
|
||||
def to_json(self) -> str:
|
||||
"""Convert result to JSON string."""
|
||||
return json.dumps(self.to_dict(), indent=2)
|
||||
|
||||
def print_summary(self) -> None:
|
||||
"""Print a summary of the evaluation results."""
|
||||
try:
|
||||
from rich.console import Console
|
||||
from rich.table import Table
|
||||
from rich.panel import Panel # noqa: F401
|
||||
|
||||
console = Console()
|
||||
|
||||
table = Table(title="Accuracy Evaluation Summary")
|
||||
table.add_column("Metric", style="cyan")
|
||||
table.add_column("Value", style="green")
|
||||
|
||||
table.add_row("Evaluations", str(len(self.evaluations)))
|
||||
table.add_row("Average Score", f"{self.avg_score:.2f}")
|
||||
table.add_row("Min Score", f"{self.min_score:.2f}")
|
||||
table.add_row("Max Score", f"{self.max_score:.2f}")
|
||||
table.add_row("Std Dev", f"{self.std_dev:.2f}")
|
||||
table.add_row("Status", "✅ PASSED" if self.passed else "❌ FAILED")
|
||||
|
||||
console.print(table)
|
||||
except ImportError:
|
||||
print("Accuracy Evaluation Summary")
|
||||
print(f" Evaluations: {len(self.evaluations)}")
|
||||
print(f" Average Score: {self.avg_score:.2f}")
|
||||
print(f" Min Score: {self.min_score:.2f}")
|
||||
print(f" Max Score: {self.max_score:.2f}")
|
||||
print(f" Std Dev: {self.std_dev:.2f}")
|
||||
print(f" Status: {'PASSED' if self.passed else 'FAILED'}")
|
||||
|
||||
|
||||
@dataclass
|
||||
class PerformanceMetrics:
|
||||
"""Individual performance measurement."""
|
||||
run_time: float
|
||||
memory_usage: float
|
||||
iteration: int
|
||||
timestamp: str = field(default_factory=lambda: datetime.now().isoformat())
|
||||
metadata: Dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
|
||||
@dataclass
|
||||
class PerformanceResult:
|
||||
"""Result from performance evaluation measuring runtime and memory."""
|
||||
metrics: List[PerformanceMetrics] = field(default_factory=list)
|
||||
warmup_runs: int = 0
|
||||
eval_id: str = ""
|
||||
name: str = ""
|
||||
|
||||
@property
|
||||
def run_times(self) -> List[float]:
|
||||
"""Get list of all run times."""
|
||||
return [m.run_time for m in self.metrics]
|
||||
|
||||
@property
|
||||
def memory_usages(self) -> List[float]:
|
||||
"""Get list of all memory usages."""
|
||||
return [m.memory_usage for m in self.metrics]
|
||||
|
||||
@property
|
||||
def avg_run_time(self) -> float:
|
||||
"""Calculate average run time."""
|
||||
if not self.run_times:
|
||||
return 0.0
|
||||
return statistics.mean(self.run_times)
|
||||
|
||||
@property
|
||||
def min_run_time(self) -> float:
|
||||
"""Get minimum run time."""
|
||||
if not self.run_times:
|
||||
return 0.0
|
||||
return min(self.run_times)
|
||||
|
||||
@property
|
||||
def max_run_time(self) -> float:
|
||||
"""Get maximum run time."""
|
||||
if not self.run_times:
|
||||
return 0.0
|
||||
return max(self.run_times)
|
||||
|
||||
@property
|
||||
def std_dev_run_time(self) -> float:
|
||||
"""Calculate standard deviation of run times."""
|
||||
if len(self.run_times) < 2:
|
||||
return 0.0
|
||||
return statistics.stdev(self.run_times)
|
||||
|
||||
@property
|
||||
def median_run_time(self) -> float:
|
||||
"""Calculate median run time."""
|
||||
if not self.run_times:
|
||||
return 0.0
|
||||
return statistics.median(self.run_times)
|
||||
|
||||
@property
|
||||
def p95_run_time(self) -> float:
|
||||
"""Calculate 95th percentile run time."""
|
||||
if not self.run_times:
|
||||
return 0.0
|
||||
sorted_times = sorted(self.run_times)
|
||||
idx = int(len(sorted_times) * 0.95)
|
||||
return sorted_times[min(idx, len(sorted_times) - 1)]
|
||||
|
||||
@property
|
||||
def avg_memory(self) -> float:
|
||||
"""Calculate average memory usage."""
|
||||
if not self.memory_usages:
|
||||
return 0.0
|
||||
return statistics.mean(self.memory_usages)
|
||||
|
||||
@property
|
||||
def max_memory(self) -> float:
|
||||
"""Get maximum memory usage."""
|
||||
if not self.memory_usages:
|
||||
return 0.0
|
||||
return max(self.memory_usages)
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
"""Convert result to dictionary."""
|
||||
return {
|
||||
"eval_id": self.eval_id,
|
||||
"name": self.name,
|
||||
"metrics": [asdict(m) for m in self.metrics],
|
||||
"warmup_runs": self.warmup_runs,
|
||||
"avg_run_time": self.avg_run_time,
|
||||
"min_run_time": self.min_run_time,
|
||||
"max_run_time": self.max_run_time,
|
||||
"std_dev_run_time": self.std_dev_run_time,
|
||||
"median_run_time": self.median_run_time,
|
||||
"p95_run_time": self.p95_run_time,
|
||||
"avg_memory": self.avg_memory,
|
||||
"max_memory": self.max_memory,
|
||||
"num_iterations": len(self.metrics)
|
||||
}
|
||||
|
||||
def to_json(self) -> str:
|
||||
"""Convert result to JSON string."""
|
||||
return json.dumps(self.to_dict(), indent=2)
|
||||
|
||||
def print_summary(self) -> None:
|
||||
"""Print a summary of the performance results."""
|
||||
try:
|
||||
from rich.console import Console
|
||||
from rich.table import Table
|
||||
|
||||
console = Console()
|
||||
|
||||
table = Table(title="Performance Evaluation Summary")
|
||||
table.add_column("Metric", style="cyan")
|
||||
table.add_column("Value", style="green")
|
||||
|
||||
table.add_row("Iterations", str(len(self.metrics)))
|
||||
table.add_row("Warmup Runs", str(self.warmup_runs))
|
||||
table.add_row("Avg Run Time", f"{self.avg_run_time:.4f}s")
|
||||
table.add_row("Min Run Time", f"{self.min_run_time:.4f}s")
|
||||
table.add_row("Max Run Time", f"{self.max_run_time:.4f}s")
|
||||
table.add_row("Std Dev", f"{self.std_dev_run_time:.4f}s")
|
||||
table.add_row("Median", f"{self.median_run_time:.4f}s")
|
||||
table.add_row("P95", f"{self.p95_run_time:.4f}s")
|
||||
table.add_row("Avg Memory", f"{self.avg_memory:.2f} MB")
|
||||
table.add_row("Max Memory", f"{self.max_memory:.2f} MB")
|
||||
|
||||
console.print(table)
|
||||
except ImportError:
|
||||
print("Performance Evaluation Summary")
|
||||
print(f" Iterations: {len(self.metrics)}")
|
||||
print(f" Warmup Runs: {self.warmup_runs}")
|
||||
print(f" Avg Run Time: {self.avg_run_time:.4f}s")
|
||||
print(f" Min Run Time: {self.min_run_time:.4f}s")
|
||||
print(f" Max Run Time: {self.max_run_time:.4f}s")
|
||||
print(f" Std Dev: {self.std_dev_run_time:.4f}s")
|
||||
print(f" Median: {self.median_run_time:.4f}s")
|
||||
print(f" P95: {self.p95_run_time:.4f}s")
|
||||
print(f" Avg Memory: {self.avg_memory:.2f} MB")
|
||||
print(f" Max Memory: {self.max_memory:.2f} MB")
|
||||
|
||||
|
||||
@dataclass
|
||||
class ToolCallResult:
|
||||
"""Result of a single tool call verification."""
|
||||
tool_name: str
|
||||
expected: bool
|
||||
actual: bool
|
||||
passed: bool
|
||||
arguments: Optional[Dict[str, Any]] = None
|
||||
metadata: Dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
|
||||
@dataclass
|
||||
class ReliabilityResult:
|
||||
"""Result from reliability evaluation checking tool calls."""
|
||||
tool_results: List[ToolCallResult] = field(default_factory=list)
|
||||
eval_id: str = ""
|
||||
name: str = ""
|
||||
|
||||
@property
|
||||
def passed_calls(self) -> List[ToolCallResult]:
|
||||
"""Get list of passed tool calls."""
|
||||
return [t for t in self.tool_results if t.passed]
|
||||
|
||||
@property
|
||||
def failed_calls(self) -> List[ToolCallResult]:
|
||||
"""Get list of failed tool calls."""
|
||||
return [t for t in self.tool_results if not t.passed]
|
||||
|
||||
@property
|
||||
def pass_rate(self) -> float:
|
||||
"""Calculate pass rate."""
|
||||
if not self.tool_results:
|
||||
return 0.0
|
||||
return len(self.passed_calls) / len(self.tool_results)
|
||||
|
||||
@property
|
||||
def status(self) -> str:
|
||||
"""Get overall status."""
|
||||
return "PASSED" if len(self.failed_calls) == 0 else "FAILED"
|
||||
|
||||
@property
|
||||
def passed(self) -> bool:
|
||||
"""Check if all tool calls passed."""
|
||||
return len(self.failed_calls) == 0
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
"""Convert result to dictionary."""
|
||||
return {
|
||||
"eval_id": self.eval_id,
|
||||
"name": self.name,
|
||||
"tool_results": [asdict(t) for t in self.tool_results],
|
||||
"passed_count": len(self.passed_calls),
|
||||
"failed_count": len(self.failed_calls),
|
||||
"pass_rate": self.pass_rate,
|
||||
"status": self.status
|
||||
}
|
||||
|
||||
def to_json(self) -> str:
|
||||
"""Convert result to JSON string."""
|
||||
return json.dumps(self.to_dict(), indent=2)
|
||||
|
||||
def print_summary(self) -> None:
|
||||
"""Print a summary of the reliability results."""
|
||||
try:
|
||||
from rich.console import Console
|
||||
from rich.table import Table
|
||||
|
||||
console = Console()
|
||||
|
||||
table = Table(title="Reliability Evaluation Summary")
|
||||
table.add_column("Metric", style="cyan")
|
||||
table.add_column("Value", style="green")
|
||||
|
||||
table.add_row("Total Tool Calls", str(len(self.tool_results)))
|
||||
table.add_row("Passed", str(len(self.passed_calls)))
|
||||
table.add_row("Failed", str(len(self.failed_calls)))
|
||||
table.add_row("Pass Rate", f"{self.pass_rate:.1%}")
|
||||
table.add_row("Status", "✅ PASSED" if self.passed else "❌ FAILED")
|
||||
|
||||
console.print(table)
|
||||
|
||||
if self.failed_calls:
|
||||
fail_table = Table(title="Failed Tool Calls")
|
||||
fail_table.add_column("Tool", style="red")
|
||||
fail_table.add_column("Expected", style="yellow")
|
||||
fail_table.add_column("Actual", style="yellow")
|
||||
|
||||
for tc in self.failed_calls:
|
||||
fail_table.add_row(tc.tool_name, str(tc.expected), str(tc.actual))
|
||||
|
||||
console.print(fail_table)
|
||||
except ImportError:
|
||||
print("Reliability Evaluation Summary")
|
||||
print(f" Total Tool Calls: {len(self.tool_results)}")
|
||||
print(f" Passed: {len(self.passed_calls)}")
|
||||
print(f" Failed: {len(self.failed_calls)}")
|
||||
print(f" Pass Rate: {self.pass_rate:.1%}")
|
||||
print(f" Status: {self.status}")
|
||||
|
||||
|
||||
@dataclass
|
||||
class CriteriaScore:
|
||||
"""Individual criteria evaluation score."""
|
||||
score: float
|
||||
passed: bool
|
||||
reasoning: str
|
||||
output_text: str
|
||||
criteria: str
|
||||
timestamp: str = field(default_factory=lambda: datetime.now().isoformat())
|
||||
metadata: Dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
|
||||
@dataclass
|
||||
class CriteriaResult:
|
||||
"""Result from criteria-based evaluation (agent-as-judge)."""
|
||||
evaluations: List[CriteriaScore] = field(default_factory=list)
|
||||
criteria: str = ""
|
||||
scoring_type: str = "numeric" # "numeric" or "binary"
|
||||
threshold: float = 7.0
|
||||
eval_id: str = ""
|
||||
name: str = ""
|
||||
|
||||
@property
|
||||
def scores(self) -> List[float]:
|
||||
"""Get list of all scores."""
|
||||
return [e.score for e in self.evaluations]
|
||||
|
||||
@property
|
||||
def avg_score(self) -> float:
|
||||
"""Calculate average score."""
|
||||
if not self.scores:
|
||||
return 0.0
|
||||
return statistics.mean(self.scores)
|
||||
|
||||
@property
|
||||
def pass_rate(self) -> float:
|
||||
"""Calculate pass rate."""
|
||||
if not self.evaluations:
|
||||
return 0.0
|
||||
passed = sum(1 for e in self.evaluations if e.passed)
|
||||
return passed / len(self.evaluations)
|
||||
|
||||
@property
|
||||
def passed(self) -> bool:
|
||||
"""Check if evaluation passed overall."""
|
||||
if self.scoring_type == "binary":
|
||||
return self.pass_rate >= 0.5
|
||||
return self.avg_score >= self.threshold
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
"""Convert result to dictionary."""
|
||||
return {
|
||||
"eval_id": self.eval_id,
|
||||
"name": self.name,
|
||||
"criteria": self.criteria,
|
||||
"scoring_type": self.scoring_type,
|
||||
"threshold": self.threshold,
|
||||
"evaluations": [asdict(e) for e in self.evaluations],
|
||||
"avg_score": self.avg_score,
|
||||
"pass_rate": self.pass_rate,
|
||||
"passed": self.passed,
|
||||
"num_evaluations": len(self.evaluations)
|
||||
}
|
||||
|
||||
def to_json(self) -> str:
|
||||
"""Convert result to JSON string."""
|
||||
return json.dumps(self.to_dict(), indent=2)
|
||||
|
||||
def print_summary(self) -> None:
|
||||
"""Print a summary of the criteria evaluation results."""
|
||||
try:
|
||||
from rich.console import Console
|
||||
from rich.table import Table
|
||||
|
||||
console = Console()
|
||||
|
||||
table = Table(title="Criteria Evaluation Summary")
|
||||
table.add_column("Metric", style="cyan")
|
||||
table.add_column("Value", style="green")
|
||||
|
||||
table.add_row("Criteria", self.criteria[:50] + "..." if len(self.criteria) > 50 else self.criteria)
|
||||
table.add_row("Scoring Type", self.scoring_type)
|
||||
table.add_row("Threshold", str(self.threshold))
|
||||
table.add_row("Evaluations", str(len(self.evaluations)))
|
||||
table.add_row("Average Score", f"{self.avg_score:.2f}")
|
||||
table.add_row("Pass Rate", f"{self.pass_rate:.1%}")
|
||||
table.add_row("Status", "✅ PASSED" if self.passed else "❌ FAILED")
|
||||
|
||||
console.print(table)
|
||||
except ImportError:
|
||||
print("Criteria Evaluation Summary")
|
||||
print(f" Criteria: {self.criteria[:50]}...")
|
||||
print(f" Scoring Type: {self.scoring_type}")
|
||||
print(f" Threshold: {self.threshold}")
|
||||
print(f" Evaluations: {len(self.evaluations)}")
|
||||
print(f" Average Score: {self.avg_score:.2f}")
|
||||
print(f" Pass Rate: {self.pass_rate:.1%}")
|
||||
print(f" Status: {'PASSED' if self.passed else 'FAILED'}")
|
||||
@@ -0,0 +1,154 @@
|
||||
"""
|
||||
Utility functions for PraisonAI Agents evaluation framework.
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, Optional, Union
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def save_result_to_file(
|
||||
result: Any,
|
||||
file_path: str,
|
||||
eval_id: Optional[str] = None,
|
||||
name: Optional[str] = None
|
||||
) -> bool:
|
||||
"""
|
||||
Save evaluation result to a file.
|
||||
|
||||
Args:
|
||||
result: The evaluation result to save
|
||||
file_path: Path to save the result (supports {name}, {eval_id} placeholders)
|
||||
eval_id: Optional eval ID for placeholder substitution
|
||||
name: Optional name for placeholder substitution
|
||||
|
||||
Returns:
|
||||
True if save was successful, False otherwise
|
||||
"""
|
||||
try:
|
||||
formatted_path = file_path.format(name=name or "", eval_id=eval_id or "")
|
||||
path = Path(formatted_path)
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
if hasattr(result, 'to_dict'):
|
||||
data = result.to_dict()
|
||||
elif hasattr(result, '__dict__'):
|
||||
data = result.__dict__
|
||||
else:
|
||||
data = {"result": str(result)}
|
||||
|
||||
path.write_text(json.dumps(data, indent=2, default=str))
|
||||
logger.debug(f"Saved result to: {formatted_path}")
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to save result: {e}")
|
||||
return False
|
||||
|
||||
|
||||
def load_result_from_file(file_path: str) -> Optional[Dict[str, Any]]:
|
||||
"""
|
||||
Load evaluation result from a file.
|
||||
|
||||
Args:
|
||||
file_path: Path to the result file
|
||||
|
||||
Returns:
|
||||
Dictionary containing the result data, or None if load failed
|
||||
"""
|
||||
try:
|
||||
path = Path(file_path)
|
||||
if not path.exists():
|
||||
logger.warning(f"Result file not found: {file_path}")
|
||||
return None
|
||||
|
||||
data = json.loads(path.read_text())
|
||||
return data
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to load result: {e}")
|
||||
return None
|
||||
|
||||
|
||||
def format_score(score: float, max_score: float = 10.0) -> str:
|
||||
"""
|
||||
Format a score for display.
|
||||
|
||||
Args:
|
||||
score: The score value
|
||||
max_score: Maximum possible score
|
||||
|
||||
Returns:
|
||||
Formatted score string
|
||||
"""
|
||||
return f"{score:.1f}/{max_score:.0f}"
|
||||
|
||||
|
||||
def format_percentage(value: float) -> str:
|
||||
"""
|
||||
Format a value as a percentage.
|
||||
|
||||
Args:
|
||||
value: Value between 0 and 1
|
||||
|
||||
Returns:
|
||||
Formatted percentage string
|
||||
"""
|
||||
return f"{value * 100:.1f}%"
|
||||
|
||||
|
||||
def format_duration(seconds: float) -> str:
|
||||
"""
|
||||
Format a duration in seconds for display.
|
||||
|
||||
Args:
|
||||
seconds: Duration in seconds
|
||||
|
||||
Returns:
|
||||
Formatted duration string
|
||||
"""
|
||||
if seconds < 0.001:
|
||||
return f"{seconds * 1000000:.0f}μs"
|
||||
elif seconds < 1:
|
||||
return f"{seconds * 1000:.2f}ms"
|
||||
elif seconds < 60:
|
||||
return f"{seconds:.2f}s"
|
||||
else:
|
||||
minutes = int(seconds // 60)
|
||||
secs = seconds % 60
|
||||
return f"{minutes}m {secs:.1f}s"
|
||||
|
||||
|
||||
def format_memory(mb: float) -> str:
|
||||
"""
|
||||
Format memory usage for display.
|
||||
|
||||
Args:
|
||||
mb: Memory in megabytes
|
||||
|
||||
Returns:
|
||||
Formatted memory string
|
||||
"""
|
||||
if mb < 1:
|
||||
return f"{mb * 1024:.1f} KB"
|
||||
elif mb < 1024:
|
||||
return f"{mb:.2f} MB"
|
||||
else:
|
||||
return f"{mb / 1024:.2f} GB"
|
||||
|
||||
|
||||
def calculate_pass_rate(passed: int, total: int) -> float:
|
||||
"""
|
||||
Calculate pass rate.
|
||||
|
||||
Args:
|
||||
passed: Number of passed evaluations
|
||||
total: Total number of evaluations
|
||||
|
||||
Returns:
|
||||
Pass rate as a float between 0 and 1
|
||||
"""
|
||||
if total == 0:
|
||||
return 0.0
|
||||
return passed / total
|
||||
@@ -1054,7 +1054,7 @@ class LLM:
|
||||
**kwargs
|
||||
) -> str:
|
||||
"""Enhanced get_response with all OpenAI-like features"""
|
||||
logging.info(f"Getting response from {self.model}")
|
||||
logging.debug(f"Getting response from {self.model}")
|
||||
# Log all self values when in debug mode
|
||||
self._log_llm_config(
|
||||
'LLM instance',
|
||||
@@ -2472,7 +2472,7 @@ Output MUST be JSON with 'reflection' and 'satisfactory'.
|
||||
"""Async version of get_response with identical functionality."""
|
||||
try:
|
||||
import litellm
|
||||
logging.info(f"Getting async response from {self.model}")
|
||||
logging.debug(f"Getting async response from {self.model}")
|
||||
# Log all self values when in debug mode
|
||||
self._log_llm_config(
|
||||
'get_response_async',
|
||||
|
||||
@@ -19,6 +19,15 @@ from rich.console import Console
|
||||
from rich.live import Live
|
||||
import inspect
|
||||
|
||||
# Import display_tool_call for callback support (lazy import to avoid circular imports)
|
||||
_display_tool_call = None
|
||||
def _get_display_tool_call():
|
||||
global _display_tool_call
|
||||
if _display_tool_call is None:
|
||||
from ..main import display_tool_call
|
||||
_display_tool_call = display_tool_call
|
||||
return _display_tool_call
|
||||
|
||||
# Constants
|
||||
LOCAL_SERVER_API_KEY_PLACEHOLDER = "not-needed"
|
||||
|
||||
@@ -978,16 +987,19 @@ class OpenAIClient:
|
||||
function_name = tool_call.function.name
|
||||
arguments = json.loads(tool_call.function.arguments)
|
||||
|
||||
# Always trigger callback for tool call tracking (even when verbose=False)
|
||||
display_tool_call_fn = _get_display_tool_call()
|
||||
display_tool_call_fn(f"Calling function: {function_name}", console=console if verbose else None)
|
||||
|
||||
if verbose and console:
|
||||
console.print(f"[bold]Calling function:[/bold] {function_name}")
|
||||
console.print(f"[dim]Arguments:[/dim] {arguments}")
|
||||
|
||||
# Execute the tool
|
||||
tool_result = execute_tool_fn(function_name, arguments)
|
||||
results_str = json.dumps(tool_result) if tool_result else "Function returned an empty output"
|
||||
|
||||
if verbose and console:
|
||||
console.print(f"[dim]Result:[/dim] {results_str}")
|
||||
# Trigger callback with result
|
||||
display_tool_call_fn(f"Function {function_name} returned: {results_str[:200]}{'...' if len(results_str) > 200 else ''}", console=console if verbose else None)
|
||||
|
||||
messages.append({
|
||||
"role": "tool",
|
||||
@@ -1150,8 +1162,11 @@ class OpenAIClient:
|
||||
function_name = tool_call.function.name
|
||||
arguments = json.loads(tool_call.function.arguments)
|
||||
|
||||
# Always trigger callback for tool call tracking (even when verbose=False)
|
||||
display_tool_call_fn = _get_display_tool_call()
|
||||
display_tool_call_fn(f"Calling function: {function_name}", console=console if verbose else None)
|
||||
|
||||
if verbose and console:
|
||||
console.print(f"[bold]Calling function:[/bold] {function_name}")
|
||||
console.print(f"[dim]Arguments:[/dim] {arguments}")
|
||||
|
||||
# Execute the tool (async)
|
||||
@@ -1167,8 +1182,8 @@ class OpenAIClient:
|
||||
|
||||
results_str = json.dumps(tool_result) if tool_result else "Function returned an empty output"
|
||||
|
||||
if verbose and console:
|
||||
console.print(f"[dim]Result:[/dim] {results_str}")
|
||||
# Trigger callback with result
|
||||
display_tool_call_fn(f"Function {function_name} returned: {results_str[:200]}{'...' if len(results_str) > 200 else ''}", console=console if verbose else None)
|
||||
|
||||
messages.append({
|
||||
"role": "tool",
|
||||
@@ -1299,6 +1314,10 @@ class OpenAIClient:
|
||||
yield f"\n[Error parsing arguments for {function_name if 'function_name' in locals() else 'unknown function'}: {str(e)}]"
|
||||
continue
|
||||
|
||||
# Always trigger callback for tool call tracking (even when verbose=False)
|
||||
display_tool_call_fn = _get_display_tool_call()
|
||||
display_tool_call_fn(f"Calling function: {function_name}", console=None)
|
||||
|
||||
if verbose:
|
||||
yield f"\n[Calling function: {function_name}]"
|
||||
|
||||
@@ -1311,6 +1330,9 @@ class OpenAIClient:
|
||||
if verbose:
|
||||
yield f"\n[Function error: {str(e)}]"
|
||||
|
||||
# Trigger callback with result
|
||||
display_tool_call_fn(f"Function {function_name} returned: {results_str[:200]}{'...' if len(results_str) > 200 else ''}", console=None)
|
||||
|
||||
if verbose:
|
||||
yield f"\n[Function result: {results_str}]"
|
||||
|
||||
|
||||
@@ -231,15 +231,15 @@ def display_tool_call(message: str, console=None):
|
||||
if not message or not message.strip():
|
||||
logging.debug("Empty message in display_tool_call, returning early")
|
||||
return
|
||||
if console is None:
|
||||
console = Console()
|
||||
message = _clean_display_content(str(message))
|
||||
logging.debug(f"Cleaned message in display_tool_call: {repr(message)}")
|
||||
|
||||
# Execute synchronous callbacks
|
||||
# Execute synchronous callbacks (always, even when console is None)
|
||||
execute_sync_callback('tool_call', message=message)
|
||||
|
||||
console.print(Panel.fit(Text(message, style="bold cyan"), title="Tool Call", border_style="green"))
|
||||
# Only print panel if console is provided (verbose mode)
|
||||
if console is not None:
|
||||
console.print(Panel.fit(Text(message, style="bold cyan"), title="Tool Call", border_style="green"))
|
||||
|
||||
def display_error(message: str, console=None):
|
||||
if not message or not message.strip():
|
||||
|
||||
@@ -0,0 +1,76 @@
|
||||
"""Agent Skills module for PraisonAI Agents.
|
||||
|
||||
This module provides support for the open Agent Skills standard (agentskills.io),
|
||||
enabling agents to load and use modular capabilities through SKILL.md files.
|
||||
|
||||
Zero Performance Impact:
|
||||
- All imports are lazy loaded via __getattr__
|
||||
- Skills only loaded when explicitly enabled
|
||||
- No auto-discovery at import time
|
||||
|
||||
Usage:
|
||||
from praisonaiagents.skills import SkillManager, SkillProperties
|
||||
|
||||
manager = SkillManager()
|
||||
manager.discover(["./skills"])
|
||||
prompt_xml = manager.to_prompt()
|
||||
"""
|
||||
|
||||
__all__ = [
|
||||
# Models
|
||||
"SkillProperties",
|
||||
"SkillMetadata",
|
||||
"ParseError",
|
||||
"ValidationError",
|
||||
# Parser
|
||||
"parse_frontmatter",
|
||||
"find_skill_md",
|
||||
"read_properties",
|
||||
# Validator
|
||||
"validate",
|
||||
"validate_metadata",
|
||||
# Prompt
|
||||
"to_prompt",
|
||||
"generate_skills_xml",
|
||||
"format_skill_for_prompt",
|
||||
# Discovery
|
||||
"discover_skills",
|
||||
"get_default_skill_dirs",
|
||||
# Loader
|
||||
"SkillLoader",
|
||||
# Manager
|
||||
"SkillManager",
|
||||
]
|
||||
|
||||
|
||||
def __getattr__(name: str):
|
||||
"""Lazy load module components to avoid import overhead."""
|
||||
if name in ("SkillProperties", "SkillMetadata", "ParseError", "ValidationError"):
|
||||
from .models import SkillProperties, SkillMetadata, ParseError, ValidationError
|
||||
return locals()[name]
|
||||
|
||||
if name in ("parse_frontmatter", "find_skill_md", "read_properties"):
|
||||
from .parser import parse_frontmatter, find_skill_md, read_properties
|
||||
return locals()[name]
|
||||
|
||||
if name in ("validate", "validate_metadata", "_validate_name", "_validate_description", "_validate_compatibility"):
|
||||
from .validator import validate, validate_metadata, _validate_name, _validate_description, _validate_compatibility
|
||||
return locals()[name]
|
||||
|
||||
if name in ("to_prompt", "generate_skills_xml", "format_skill_for_prompt"):
|
||||
from .prompt import to_prompt, generate_skills_xml, format_skill_for_prompt
|
||||
return locals()[name]
|
||||
|
||||
if name in ("discover_skills", "get_default_skill_dirs"):
|
||||
from .discovery import discover_skills, get_default_skill_dirs
|
||||
return locals()[name]
|
||||
|
||||
if name == "SkillLoader":
|
||||
from .loader import SkillLoader
|
||||
return SkillLoader
|
||||
|
||||
if name == "SkillManager":
|
||||
from .manager import SkillManager
|
||||
return SkillManager
|
||||
|
||||
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
||||
@@ -0,0 +1,129 @@
|
||||
"""Skill discovery and directory scanning."""
|
||||
|
||||
from pathlib import Path
|
||||
from typing import List, Optional
|
||||
|
||||
from .parser import find_skill_md, read_properties
|
||||
from .models import SkillProperties
|
||||
|
||||
|
||||
def get_default_skill_dirs() -> List[Path]:
|
||||
"""Get default skill directory locations.
|
||||
|
||||
Returns directories in precedence order (high to low):
|
||||
1. Project: ./.praison/skills/ or ./.claude/skills/
|
||||
2. User: ~/.praison/skills/
|
||||
3. System: /etc/praison/skills/ (admin-managed)
|
||||
|
||||
Returns:
|
||||
List of existing skill directories
|
||||
"""
|
||||
dirs = []
|
||||
cwd = Path.cwd()
|
||||
|
||||
# Project-level directories
|
||||
project_dirs = [
|
||||
cwd / ".praison" / "skills",
|
||||
cwd / ".claude" / "skills",
|
||||
]
|
||||
for d in project_dirs:
|
||||
if d.exists() and d.is_dir():
|
||||
dirs.append(d)
|
||||
|
||||
# User-level directory
|
||||
user_dir = Path.home() / ".praison" / "skills"
|
||||
if user_dir.exists() and user_dir.is_dir():
|
||||
dirs.append(user_dir)
|
||||
|
||||
# System-level directory (Unix-like systems)
|
||||
system_dir = Path("/etc/praison/skills")
|
||||
if system_dir.exists() and system_dir.is_dir():
|
||||
dirs.append(system_dir)
|
||||
|
||||
return dirs
|
||||
|
||||
|
||||
def discover_skills(
|
||||
skill_dirs: Optional[List[str]] = None,
|
||||
include_defaults: bool = True,
|
||||
) -> List[SkillProperties]:
|
||||
"""Discover all valid skills in the given directories.
|
||||
|
||||
Args:
|
||||
skill_dirs: List of directory paths to scan for skills.
|
||||
Each directory should contain skill subdirectories.
|
||||
include_defaults: Whether to include default skill directories
|
||||
|
||||
Returns:
|
||||
List of SkillProperties for all valid skills found
|
||||
"""
|
||||
all_dirs = []
|
||||
|
||||
# Add explicit directories
|
||||
if skill_dirs:
|
||||
for d in skill_dirs:
|
||||
path = Path(d).expanduser().resolve()
|
||||
if path.exists() and path.is_dir():
|
||||
all_dirs.append(path)
|
||||
|
||||
# Add default directories
|
||||
if include_defaults:
|
||||
all_dirs.extend(get_default_skill_dirs())
|
||||
|
||||
# Remove duplicates while preserving order
|
||||
seen = set()
|
||||
unique_dirs = []
|
||||
for d in all_dirs:
|
||||
if d not in seen:
|
||||
seen.add(d)
|
||||
unique_dirs.append(d)
|
||||
|
||||
skills = []
|
||||
|
||||
for parent_dir in unique_dirs:
|
||||
# Each subdirectory in parent_dir might be a skill
|
||||
try:
|
||||
for item in parent_dir.iterdir():
|
||||
if not item.is_dir():
|
||||
continue
|
||||
|
||||
# Check if this directory contains a SKILL.md
|
||||
skill_md = find_skill_md(item)
|
||||
if skill_md is None:
|
||||
continue
|
||||
|
||||
try:
|
||||
props = read_properties(item)
|
||||
skills.append(props)
|
||||
except Exception:
|
||||
# Skip invalid skills
|
||||
continue
|
||||
except PermissionError:
|
||||
# Skip directories we can't read
|
||||
continue
|
||||
|
||||
return skills
|
||||
|
||||
|
||||
def discover_skill(skill_path: str) -> Optional[SkillProperties]:
|
||||
"""Discover a single skill from a directory path.
|
||||
|
||||
Args:
|
||||
skill_path: Path to a skill directory
|
||||
|
||||
Returns:
|
||||
SkillProperties if valid, None otherwise
|
||||
"""
|
||||
path = Path(skill_path).expanduser().resolve()
|
||||
|
||||
if not path.exists() or not path.is_dir():
|
||||
return None
|
||||
|
||||
skill_md = find_skill_md(path)
|
||||
if skill_md is None:
|
||||
return None
|
||||
|
||||
try:
|
||||
return read_properties(path)
|
||||
except Exception:
|
||||
return None
|
||||
@@ -0,0 +1,236 @@
|
||||
"""Progressive skill loading for Agent Skills."""
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
from .parser import find_skill_md, read_properties, parse_frontmatter
|
||||
from .models import SkillProperties, SkillMetadata
|
||||
|
||||
|
||||
@dataclass
|
||||
class LoadedSkill:
|
||||
"""A fully or partially loaded skill.
|
||||
|
||||
Supports progressive disclosure:
|
||||
- Level 1: metadata only (name, description, location)
|
||||
- Level 2: full instructions (SKILL.md body)
|
||||
- Level 3: resources (scripts, references, assets)
|
||||
"""
|
||||
|
||||
properties: SkillProperties
|
||||
instructions: Optional[str] = None
|
||||
resources_loaded: bool = False
|
||||
_scripts: dict = field(default_factory=dict)
|
||||
_references: dict = field(default_factory=dict)
|
||||
_assets: dict = field(default_factory=dict)
|
||||
|
||||
@property
|
||||
def metadata(self) -> SkillMetadata:
|
||||
"""Get lightweight metadata for system prompt."""
|
||||
return SkillMetadata.from_properties(self.properties)
|
||||
|
||||
@property
|
||||
def is_activated(self) -> bool:
|
||||
"""Check if skill has been activated (instructions loaded)."""
|
||||
return self.instructions is not None
|
||||
|
||||
def get_scripts(self) -> dict:
|
||||
"""Get loaded scripts."""
|
||||
return self._scripts
|
||||
|
||||
def get_references(self) -> dict:
|
||||
"""Get loaded references."""
|
||||
return self._references
|
||||
|
||||
def get_assets(self) -> dict:
|
||||
"""Get loaded assets."""
|
||||
return self._assets
|
||||
|
||||
|
||||
class SkillLoader:
|
||||
"""Loader for Agent Skills with progressive disclosure support.
|
||||
|
||||
Usage:
|
||||
loader = SkillLoader()
|
||||
|
||||
# Level 1: Load metadata only
|
||||
skill = loader.load_metadata("/path/to/skill")
|
||||
|
||||
# Level 2: Activate skill (load instructions)
|
||||
loader.activate(skill)
|
||||
|
||||
# Level 3: Load specific resources
|
||||
loader.load_scripts(skill)
|
||||
loader.load_references(skill)
|
||||
"""
|
||||
|
||||
def load_metadata(self, skill_path: str) -> Optional[LoadedSkill]:
|
||||
"""Load skill metadata only (Level 1).
|
||||
|
||||
This is the minimal load for system prompt injection.
|
||||
Only reads the YAML frontmatter, not the full SKILL.md body.
|
||||
|
||||
Args:
|
||||
skill_path: Path to skill directory
|
||||
|
||||
Returns:
|
||||
LoadedSkill with metadata only, or None if invalid
|
||||
"""
|
||||
path = Path(skill_path).expanduser().resolve()
|
||||
|
||||
if not path.exists() or not path.is_dir():
|
||||
return None
|
||||
|
||||
try:
|
||||
props = read_properties(path)
|
||||
return LoadedSkill(properties=props)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
def activate(self, skill: LoadedSkill) -> bool:
|
||||
"""Activate a skill by loading its full instructions (Level 2).
|
||||
|
||||
Args:
|
||||
skill: LoadedSkill instance to activate
|
||||
|
||||
Returns:
|
||||
True if activation succeeded
|
||||
"""
|
||||
if skill.is_activated:
|
||||
return True
|
||||
|
||||
if skill.properties.path is None:
|
||||
return False
|
||||
|
||||
skill_md = find_skill_md(skill.properties.path)
|
||||
if skill_md is None:
|
||||
return False
|
||||
|
||||
try:
|
||||
content = skill_md.read_text()
|
||||
_, body = parse_frontmatter(content)
|
||||
skill.instructions = body
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
def load_scripts(self, skill: LoadedSkill) -> dict:
|
||||
"""Load scripts from skill's scripts/ directory (Level 3).
|
||||
|
||||
Args:
|
||||
skill: LoadedSkill instance
|
||||
|
||||
Returns:
|
||||
Dict mapping script names to their contents
|
||||
"""
|
||||
if skill.properties.path is None:
|
||||
return {}
|
||||
|
||||
scripts_dir = skill.properties.path / "scripts"
|
||||
if not scripts_dir.exists() or not scripts_dir.is_dir():
|
||||
return {}
|
||||
|
||||
scripts = {}
|
||||
try:
|
||||
for script_file in scripts_dir.iterdir():
|
||||
if script_file.is_file():
|
||||
try:
|
||||
scripts[script_file.name] = script_file.read_text()
|
||||
except Exception:
|
||||
continue
|
||||
except PermissionError:
|
||||
pass
|
||||
|
||||
skill._scripts = scripts
|
||||
return scripts
|
||||
|
||||
def load_references(self, skill: LoadedSkill) -> dict:
|
||||
"""Load references from skill's references/ directory (Level 3).
|
||||
|
||||
Args:
|
||||
skill: LoadedSkill instance
|
||||
|
||||
Returns:
|
||||
Dict mapping reference names to their contents
|
||||
"""
|
||||
if skill.properties.path is None:
|
||||
return {}
|
||||
|
||||
refs_dir = skill.properties.path / "references"
|
||||
if not refs_dir.exists() or not refs_dir.is_dir():
|
||||
return {}
|
||||
|
||||
refs = {}
|
||||
try:
|
||||
for ref_file in refs_dir.iterdir():
|
||||
if ref_file.is_file():
|
||||
try:
|
||||
refs[ref_file.name] = ref_file.read_text()
|
||||
except Exception:
|
||||
continue
|
||||
except PermissionError:
|
||||
pass
|
||||
|
||||
skill._references = refs
|
||||
return refs
|
||||
|
||||
def load_assets(self, skill: LoadedSkill) -> dict:
|
||||
"""Load asset paths from skill's assets/ directory (Level 3).
|
||||
|
||||
Note: For binary assets, only paths are returned, not contents.
|
||||
|
||||
Args:
|
||||
skill: LoadedSkill instance
|
||||
|
||||
Returns:
|
||||
Dict mapping asset names to their paths
|
||||
"""
|
||||
if skill.properties.path is None:
|
||||
return {}
|
||||
|
||||
assets_dir = skill.properties.path / "assets"
|
||||
if not assets_dir.exists() or not assets_dir.is_dir():
|
||||
return {}
|
||||
|
||||
assets = {}
|
||||
try:
|
||||
for asset_file in assets_dir.iterdir():
|
||||
if asset_file.is_file():
|
||||
assets[asset_file.name] = str(asset_file)
|
||||
except PermissionError:
|
||||
pass
|
||||
|
||||
skill._assets = assets
|
||||
skill.resources_loaded = True
|
||||
return assets
|
||||
|
||||
def load_all_resources(self, skill: LoadedSkill) -> None:
|
||||
"""Load all resources for a skill (Level 3 complete).
|
||||
|
||||
Args:
|
||||
skill: LoadedSkill instance
|
||||
"""
|
||||
self.load_scripts(skill)
|
||||
self.load_references(skill)
|
||||
self.load_assets(skill)
|
||||
skill.resources_loaded = True
|
||||
|
||||
@classmethod
|
||||
def load(cls, skill_path: str, activate: bool = False) -> Optional[LoadedSkill]:
|
||||
"""Convenience method to load a skill.
|
||||
|
||||
Args:
|
||||
skill_path: Path to skill directory
|
||||
activate: Whether to also load instructions
|
||||
|
||||
Returns:
|
||||
LoadedSkill or None if invalid
|
||||
"""
|
||||
loader = cls()
|
||||
skill = loader.load_metadata(skill_path)
|
||||
|
||||
if skill and activate:
|
||||
loader.activate(skill)
|
||||
|
||||
return skill
|
||||
@@ -0,0 +1,189 @@
|
||||
"""SkillManager for Agent Skills integration."""
|
||||
|
||||
from typing import List, Optional, Dict
|
||||
|
||||
from .models import SkillMetadata
|
||||
from .discovery import discover_skills
|
||||
from .loader import SkillLoader, LoadedSkill
|
||||
from .prompt import generate_skills_xml
|
||||
|
||||
|
||||
class SkillManager:
|
||||
"""Manager for discovering, loading, and using Agent Skills.
|
||||
|
||||
This is the main entry point for Agent Skills integration.
|
||||
Supports progressive disclosure and zero performance impact
|
||||
when skills are not used.
|
||||
|
||||
Usage:
|
||||
manager = SkillManager()
|
||||
|
||||
# Discover skills from directories
|
||||
manager.discover(["./skills", "~/.praison/skills"])
|
||||
|
||||
# Get XML for system prompt
|
||||
prompt_xml = manager.to_prompt()
|
||||
|
||||
# Activate a specific skill
|
||||
skill = manager.get_skill("pdf-processing")
|
||||
manager.activate(skill)
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
"""Initialize the SkillManager."""
|
||||
self._skills: Dict[str, LoadedSkill] = {}
|
||||
self._loader = SkillLoader()
|
||||
self._discovered = False
|
||||
|
||||
@property
|
||||
def skills(self) -> List[LoadedSkill]:
|
||||
"""Get all loaded skills."""
|
||||
return list(self._skills.values())
|
||||
|
||||
@property
|
||||
def skill_names(self) -> List[str]:
|
||||
"""Get names of all loaded skills."""
|
||||
return list(self._skills.keys())
|
||||
|
||||
def discover(
|
||||
self,
|
||||
skill_dirs: Optional[List[str]] = None,
|
||||
include_defaults: bool = True,
|
||||
) -> int:
|
||||
"""Discover skills from directories.
|
||||
|
||||
Args:
|
||||
skill_dirs: List of directory paths to scan
|
||||
include_defaults: Whether to include default skill directories
|
||||
|
||||
Returns:
|
||||
Number of skills discovered
|
||||
"""
|
||||
props_list = discover_skills(skill_dirs, include_defaults)
|
||||
|
||||
for props in props_list:
|
||||
if props.name not in self._skills:
|
||||
self._skills[props.name] = LoadedSkill(properties=props)
|
||||
|
||||
self._discovered = True
|
||||
return len(props_list)
|
||||
|
||||
def add_skill(self, skill_path: str) -> Optional[LoadedSkill]:
|
||||
"""Add a single skill from a directory path.
|
||||
|
||||
Args:
|
||||
skill_path: Path to skill directory
|
||||
|
||||
Returns:
|
||||
LoadedSkill if successful, None otherwise
|
||||
"""
|
||||
skill = self._loader.load_metadata(skill_path)
|
||||
if skill:
|
||||
self._skills[skill.properties.name] = skill
|
||||
return skill
|
||||
|
||||
def get_skill(self, name: str) -> Optional[LoadedSkill]:
|
||||
"""Get a skill by name.
|
||||
|
||||
Args:
|
||||
name: Skill name
|
||||
|
||||
Returns:
|
||||
LoadedSkill or None if not found
|
||||
"""
|
||||
return self._skills.get(name)
|
||||
|
||||
def activate(self, skill: LoadedSkill) -> bool:
|
||||
"""Activate a skill (load its instructions).
|
||||
|
||||
Args:
|
||||
skill: LoadedSkill to activate
|
||||
|
||||
Returns:
|
||||
True if activation succeeded
|
||||
"""
|
||||
return self._loader.activate(skill)
|
||||
|
||||
def activate_by_name(self, name: str) -> bool:
|
||||
"""Activate a skill by name.
|
||||
|
||||
Args:
|
||||
name: Skill name
|
||||
|
||||
Returns:
|
||||
True if activation succeeded
|
||||
"""
|
||||
skill = self.get_skill(name)
|
||||
if skill is None:
|
||||
return False
|
||||
return self.activate(skill)
|
||||
|
||||
def get_available_skills(self) -> List[SkillMetadata]:
|
||||
"""Get metadata for all available skills.
|
||||
|
||||
This is used for system prompt injection.
|
||||
|
||||
Returns:
|
||||
List of SkillMetadata instances
|
||||
"""
|
||||
return [skill.metadata for skill in self._skills.values()]
|
||||
|
||||
def to_prompt(self) -> str:
|
||||
"""Generate XML prompt for available skills.
|
||||
|
||||
Returns:
|
||||
XML string with <available_skills> block
|
||||
"""
|
||||
metadata_list = self.get_available_skills()
|
||||
return generate_skills_xml(metadata_list)
|
||||
|
||||
def get_instructions(self, name: str) -> Optional[str]:
|
||||
"""Get instructions for a skill, activating if needed.
|
||||
|
||||
Args:
|
||||
name: Skill name
|
||||
|
||||
Returns:
|
||||
Skill instructions or None if not found
|
||||
"""
|
||||
skill = self.get_skill(name)
|
||||
if skill is None:
|
||||
return None
|
||||
|
||||
if not skill.is_activated:
|
||||
self.activate(skill)
|
||||
|
||||
return skill.instructions
|
||||
|
||||
def load_resources(self, name: str) -> bool:
|
||||
"""Load all resources for a skill.
|
||||
|
||||
Args:
|
||||
name: Skill name
|
||||
|
||||
Returns:
|
||||
True if resources loaded successfully
|
||||
"""
|
||||
skill = self.get_skill(name)
|
||||
if skill is None:
|
||||
return False
|
||||
|
||||
self._loader.load_all_resources(skill)
|
||||
return True
|
||||
|
||||
def clear(self) -> None:
|
||||
"""Clear all loaded skills."""
|
||||
self._skills.clear()
|
||||
self._discovered = False
|
||||
|
||||
def __len__(self) -> int:
|
||||
"""Get number of loaded skills."""
|
||||
return len(self._skills)
|
||||
|
||||
def __contains__(self, name: str) -> bool:
|
||||
"""Check if a skill is loaded."""
|
||||
return name in self._skills
|
||||
|
||||
def __iter__(self):
|
||||
"""Iterate over loaded skills."""
|
||||
return iter(self._skills.values())
|
||||
@@ -0,0 +1,95 @@
|
||||
"""Data models for Agent Skills."""
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Optional
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
class ParseError(Exception):
|
||||
"""Raised when SKILL.md parsing fails."""
|
||||
pass
|
||||
|
||||
|
||||
class ValidationError(Exception):
|
||||
"""Raised when skill validation fails."""
|
||||
pass
|
||||
|
||||
|
||||
@dataclass
|
||||
class SkillProperties:
|
||||
"""Properties parsed from a skill's SKILL.md frontmatter.
|
||||
|
||||
Attributes:
|
||||
name: Skill name in kebab-case (required)
|
||||
description: What the skill does and when the model should use it (required)
|
||||
license: License for the skill (optional)
|
||||
compatibility: Compatibility information for the skill (optional)
|
||||
allowed_tools: Tool patterns the skill requires (optional, experimental)
|
||||
metadata: Key-value pairs for client-specific properties (defaults to
|
||||
empty dict; omitted from to_dict() output when empty)
|
||||
path: Path to the skill directory (optional, added for PraisonAI)
|
||||
"""
|
||||
|
||||
name: str
|
||||
description: str
|
||||
license: Optional[str] = None
|
||||
compatibility: Optional[str] = None
|
||||
allowed_tools: Optional[str] = None
|
||||
metadata: dict = field(default_factory=dict)
|
||||
path: Optional[Path] = None
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
"""Convert to dictionary, excluding None values and empty metadata."""
|
||||
result = {"name": self.name, "description": self.description}
|
||||
if self.license is not None:
|
||||
result["license"] = self.license
|
||||
if self.compatibility is not None:
|
||||
result["compatibility"] = self.compatibility
|
||||
if self.allowed_tools is not None:
|
||||
result["allowed-tools"] = self.allowed_tools
|
||||
if self.metadata:
|
||||
result["metadata"] = self.metadata
|
||||
return result
|
||||
|
||||
|
||||
@dataclass
|
||||
class SkillMetadata:
|
||||
"""Lightweight skill metadata for system prompt injection.
|
||||
|
||||
This is the minimal information needed to include a skill in the
|
||||
system prompt's <available_skills> block (~50-100 tokens per skill).
|
||||
|
||||
Attributes:
|
||||
name: Skill name
|
||||
description: What the skill does and when to use it
|
||||
location: Absolute path to the SKILL.md file
|
||||
"""
|
||||
|
||||
name: str
|
||||
description: str
|
||||
location: str
|
||||
|
||||
@classmethod
|
||||
def from_properties(cls, props: SkillProperties) -> "SkillMetadata":
|
||||
"""Create SkillMetadata from SkillProperties.
|
||||
|
||||
Args:
|
||||
props: SkillProperties instance with path set
|
||||
|
||||
Returns:
|
||||
SkillMetadata with location pointing to SKILL.md
|
||||
"""
|
||||
if props.path:
|
||||
# Look for SKILL.md or skill.md
|
||||
skill_md = props.path / "SKILL.md"
|
||||
if not skill_md.exists():
|
||||
skill_md = props.path / "skill.md"
|
||||
location = str(skill_md)
|
||||
else:
|
||||
location = ""
|
||||
|
||||
return cls(
|
||||
name=props.name,
|
||||
description=props.description,
|
||||
location=location
|
||||
)
|
||||
@@ -0,0 +1,165 @@
|
||||
"""YAML frontmatter parsing for SKILL.md files."""
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Optional, Tuple
|
||||
import re
|
||||
|
||||
from .models import ParseError, ValidationError, SkillProperties
|
||||
|
||||
|
||||
def find_skill_md(skill_dir: Path) -> Optional[Path]:
|
||||
"""Find the SKILL.md file in a skill directory.
|
||||
|
||||
Prefers SKILL.md (uppercase) but accepts skill.md (lowercase).
|
||||
|
||||
Args:
|
||||
skill_dir: Path to the skill directory
|
||||
|
||||
Returns:
|
||||
Path to the SKILL.md file, or None if not found
|
||||
"""
|
||||
for name in ("SKILL.md", "skill.md"):
|
||||
path = skill_dir / name
|
||||
if path.exists():
|
||||
return path
|
||||
return None
|
||||
|
||||
|
||||
def parse_frontmatter(content: str) -> Tuple[dict, str]:
|
||||
"""Parse YAML frontmatter from SKILL.md content.
|
||||
|
||||
Args:
|
||||
content: Raw content of SKILL.md file
|
||||
|
||||
Returns:
|
||||
Tuple of (metadata dict, markdown body)
|
||||
|
||||
Raises:
|
||||
ParseError: If frontmatter is missing or invalid
|
||||
"""
|
||||
if not content.startswith("---"):
|
||||
raise ParseError("SKILL.md must start with YAML frontmatter (---)")
|
||||
|
||||
parts = content.split("---", 2)
|
||||
if len(parts) < 3:
|
||||
raise ParseError("SKILL.md frontmatter not properly closed with ---")
|
||||
|
||||
frontmatter_str = parts[1]
|
||||
body = parts[2].strip()
|
||||
|
||||
try:
|
||||
metadata = _parse_yaml(frontmatter_str)
|
||||
except Exception as e:
|
||||
raise ParseError(f"Invalid YAML in frontmatter: {e}")
|
||||
|
||||
if not isinstance(metadata, dict):
|
||||
raise ParseError("SKILL.md frontmatter must be a YAML mapping")
|
||||
|
||||
# Convert metadata values to strings where needed
|
||||
if "metadata" in metadata and isinstance(metadata["metadata"], dict):
|
||||
metadata["metadata"] = {str(k): str(v) for k, v in metadata["metadata"].items()}
|
||||
|
||||
return metadata, body
|
||||
|
||||
|
||||
def _parse_yaml(yaml_str: str) -> dict:
|
||||
"""Parse YAML string to dictionary.
|
||||
|
||||
Uses a simple parser to avoid strictyaml dependency.
|
||||
Falls back to yaml if available.
|
||||
|
||||
Args:
|
||||
yaml_str: YAML string to parse
|
||||
|
||||
Returns:
|
||||
Parsed dictionary
|
||||
"""
|
||||
# Try to use PyYAML if available
|
||||
try:
|
||||
import yaml
|
||||
return yaml.safe_load(yaml_str) or {}
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
# Simple fallback parser for basic YAML
|
||||
result = {}
|
||||
current_key = None
|
||||
current_dict = None
|
||||
|
||||
for line in yaml_str.strip().split("\n"):
|
||||
if not line.strip():
|
||||
continue
|
||||
|
||||
# Check for nested dict
|
||||
if line.startswith(" ") and current_dict is not None:
|
||||
match = re.match(r"^\s+(\w+):\s*(.*)$", line)
|
||||
if match:
|
||||
key, value = match.groups()
|
||||
current_dict[key] = value.strip().strip('"').strip("'")
|
||||
continue
|
||||
|
||||
# Top-level key
|
||||
match = re.match(r"^(\S+):\s*(.*)$", line)
|
||||
if match:
|
||||
key, value = match.groups()
|
||||
value = value.strip()
|
||||
|
||||
if not value:
|
||||
# Could be start of nested dict
|
||||
result[key] = {}
|
||||
current_key = key
|
||||
current_dict = result[key]
|
||||
else:
|
||||
result[key] = value.strip('"').strip("'")
|
||||
current_dict = None
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def read_properties(skill_dir: Path) -> SkillProperties:
|
||||
"""Read skill properties from SKILL.md frontmatter.
|
||||
|
||||
This function parses the frontmatter and returns properties.
|
||||
It does NOT perform full validation. Use validate() for that.
|
||||
|
||||
Args:
|
||||
skill_dir: Path to the skill directory
|
||||
|
||||
Returns:
|
||||
SkillProperties with parsed metadata
|
||||
|
||||
Raises:
|
||||
ParseError: If SKILL.md is missing or has invalid YAML
|
||||
ValidationError: If required fields (name, description) are missing
|
||||
"""
|
||||
skill_dir = Path(skill_dir)
|
||||
skill_md = find_skill_md(skill_dir)
|
||||
|
||||
if skill_md is None:
|
||||
raise ParseError(f"SKILL.md not found in {skill_dir}")
|
||||
|
||||
content = skill_md.read_text()
|
||||
metadata, _ = parse_frontmatter(content)
|
||||
|
||||
if "name" not in metadata:
|
||||
raise ValidationError("Missing required field in frontmatter: name")
|
||||
if "description" not in metadata:
|
||||
raise ValidationError("Missing required field in frontmatter: description")
|
||||
|
||||
name = metadata["name"]
|
||||
description = metadata["description"]
|
||||
|
||||
if not isinstance(name, str) or not name.strip():
|
||||
raise ValidationError("Field 'name' must be a non-empty string")
|
||||
if not isinstance(description, str) or not description.strip():
|
||||
raise ValidationError("Field 'description' must be a non-empty string")
|
||||
|
||||
return SkillProperties(
|
||||
name=name.strip(),
|
||||
description=description.strip(),
|
||||
license=metadata.get("license"),
|
||||
compatibility=metadata.get("compatibility"),
|
||||
allowed_tools=metadata.get("allowed-tools"),
|
||||
metadata=metadata.get("metadata") or {},
|
||||
path=skill_dir,
|
||||
)
|
||||
@@ -0,0 +1,85 @@
|
||||
"""XML prompt generation for Agent Skills."""
|
||||
|
||||
from pathlib import Path
|
||||
from typing import List
|
||||
import html
|
||||
|
||||
from .models import SkillMetadata
|
||||
from .parser import read_properties
|
||||
|
||||
|
||||
def format_skill_for_prompt(skill: SkillMetadata) -> str:
|
||||
"""Format a single skill as XML for system prompt.
|
||||
|
||||
Args:
|
||||
skill: SkillMetadata instance
|
||||
|
||||
Returns:
|
||||
XML string for the skill
|
||||
"""
|
||||
name = html.escape(skill.name)
|
||||
description = html.escape(skill.description)
|
||||
location = html.escape(skill.location)
|
||||
|
||||
return f""" <skill>
|
||||
<name>{name}</name>
|
||||
<description>{description}</description>
|
||||
<location>{location}</location>
|
||||
</skill>"""
|
||||
|
||||
|
||||
def generate_skills_xml(skills: List[SkillMetadata], working_directory: str = None) -> str:
|
||||
"""Generate XML block for available skills.
|
||||
|
||||
Args:
|
||||
skills: List of SkillMetadata instances
|
||||
working_directory: Current working directory for path resolution
|
||||
|
||||
Returns:
|
||||
XML string with <available_skills> block
|
||||
"""
|
||||
import os
|
||||
cwd = working_directory or os.getcwd()
|
||||
|
||||
if not skills:
|
||||
return f"""<available_skills>
|
||||
<working_directory>{html.escape(cwd)}</working_directory>
|
||||
<note>When using run_skill_script, pass file paths relative to the working directory or as absolute paths. The tool will resolve relative paths automatically.</note>
|
||||
</available_skills>"""
|
||||
|
||||
skill_entries = "\n".join(format_skill_for_prompt(s) for s in skills)
|
||||
return f"""<available_skills>
|
||||
<working_directory>{html.escape(cwd)}</working_directory>
|
||||
<note>When using run_skill_script, pass file paths relative to the working directory or as absolute paths. The tool will resolve relative paths automatically.</note>
|
||||
{skill_entries}
|
||||
</available_skills>"""
|
||||
|
||||
|
||||
def to_prompt(skill_dirs: List[Path]) -> str:
|
||||
"""Generate prompt XML from skill directories.
|
||||
|
||||
This is the main entry point for generating the <available_skills>
|
||||
block to inject into agent system prompts.
|
||||
|
||||
Args:
|
||||
skill_dirs: List of paths to skill directories
|
||||
|
||||
Returns:
|
||||
XML string with <available_skills> block
|
||||
"""
|
||||
skills = []
|
||||
|
||||
for skill_dir in skill_dirs:
|
||||
skill_dir = Path(skill_dir)
|
||||
if not skill_dir.exists() or not skill_dir.is_dir():
|
||||
continue
|
||||
|
||||
try:
|
||||
props = read_properties(skill_dir)
|
||||
meta = SkillMetadata.from_properties(props)
|
||||
skills.append(meta)
|
||||
except Exception:
|
||||
# Skip invalid skills silently
|
||||
continue
|
||||
|
||||
return generate_skills_xml(skills)
|
||||
@@ -0,0 +1,176 @@
|
||||
"""Skill validation logic."""
|
||||
|
||||
import unicodedata
|
||||
from pathlib import Path
|
||||
from typing import Optional, List
|
||||
|
||||
from .models import ParseError
|
||||
from .parser import find_skill_md, parse_frontmatter
|
||||
|
||||
MAX_SKILL_NAME_LENGTH = 64
|
||||
MAX_DESCRIPTION_LENGTH = 1024
|
||||
MAX_COMPATIBILITY_LENGTH = 500
|
||||
|
||||
ALLOWED_FIELDS = {
|
||||
"name",
|
||||
"description",
|
||||
"license",
|
||||
"allowed-tools",
|
||||
"metadata",
|
||||
"compatibility",
|
||||
}
|
||||
|
||||
|
||||
def _validate_name(name: str, skill_dir: Optional[Path]) -> List[str]:
|
||||
"""Validate skill name format and directory match.
|
||||
|
||||
Skill names support i18n characters (Unicode letters) plus hyphens.
|
||||
Names must be lowercase and cannot start/end with hyphens.
|
||||
"""
|
||||
errors = []
|
||||
|
||||
if not name or not isinstance(name, str) or not name.strip():
|
||||
errors.append("Field 'name' must be a non-empty string")
|
||||
return errors
|
||||
|
||||
name = unicodedata.normalize("NFKC", name.strip())
|
||||
|
||||
if len(name) > MAX_SKILL_NAME_LENGTH:
|
||||
errors.append(
|
||||
f"Skill name '{name}' exceeds {MAX_SKILL_NAME_LENGTH} character limit "
|
||||
f"({len(name)} chars)"
|
||||
)
|
||||
|
||||
if name != name.lower():
|
||||
errors.append(f"Skill name '{name}' must be lowercase")
|
||||
|
||||
if name.startswith("-") or name.endswith("-"):
|
||||
errors.append("Skill name cannot start or end with a hyphen")
|
||||
|
||||
if "--" in name:
|
||||
errors.append("Skill name cannot contain consecutive hyphens")
|
||||
|
||||
if not all(c.isalnum() or c == "-" for c in name):
|
||||
errors.append(
|
||||
f"Skill name '{name}' contains invalid characters. "
|
||||
"Only letters, digits, and hyphens are allowed."
|
||||
)
|
||||
|
||||
if skill_dir:
|
||||
dir_name = unicodedata.normalize("NFKC", skill_dir.name)
|
||||
if dir_name != name:
|
||||
errors.append(
|
||||
f"Directory name '{skill_dir.name}' must match skill name '{name}'"
|
||||
)
|
||||
|
||||
return errors
|
||||
|
||||
|
||||
def _validate_description(description: str) -> List[str]:
|
||||
"""Validate description format."""
|
||||
errors = []
|
||||
|
||||
if not description or not isinstance(description, str) or not description.strip():
|
||||
errors.append("Field 'description' must be a non-empty string")
|
||||
return errors
|
||||
|
||||
if len(description) > MAX_DESCRIPTION_LENGTH:
|
||||
errors.append(
|
||||
f"Description exceeds {MAX_DESCRIPTION_LENGTH} character limit "
|
||||
f"({len(description)} chars)"
|
||||
)
|
||||
|
||||
return errors
|
||||
|
||||
|
||||
def _validate_compatibility(compatibility: str) -> List[str]:
|
||||
"""Validate compatibility format."""
|
||||
errors = []
|
||||
|
||||
if not isinstance(compatibility, str):
|
||||
errors.append("Field 'compatibility' must be a string")
|
||||
return errors
|
||||
|
||||
if len(compatibility) > MAX_COMPATIBILITY_LENGTH:
|
||||
errors.append(
|
||||
f"Compatibility exceeds {MAX_COMPATIBILITY_LENGTH} character limit "
|
||||
f"({len(compatibility)} chars)"
|
||||
)
|
||||
|
||||
return errors
|
||||
|
||||
|
||||
def _validate_metadata_fields(metadata: dict) -> List[str]:
|
||||
"""Validate that only allowed fields are present."""
|
||||
errors = []
|
||||
|
||||
extra_fields = set(metadata.keys()) - ALLOWED_FIELDS
|
||||
if extra_fields:
|
||||
errors.append(
|
||||
f"Unexpected fields in frontmatter: {', '.join(sorted(extra_fields))}. "
|
||||
f"Only {sorted(ALLOWED_FIELDS)} are allowed."
|
||||
)
|
||||
|
||||
return errors
|
||||
|
||||
|
||||
def validate_metadata(metadata: dict, skill_dir: Optional[Path] = None) -> List[str]:
|
||||
"""Validate parsed skill metadata.
|
||||
|
||||
This is the core validation function that works on already-parsed metadata,
|
||||
avoiding duplicate file I/O when called from the parser.
|
||||
|
||||
Args:
|
||||
metadata: Parsed YAML frontmatter dictionary
|
||||
skill_dir: Optional path to skill directory (for name-directory match check)
|
||||
|
||||
Returns:
|
||||
List of validation error messages. Empty list means valid.
|
||||
"""
|
||||
errors = []
|
||||
errors.extend(_validate_metadata_fields(metadata))
|
||||
|
||||
if "name" not in metadata:
|
||||
errors.append("Missing required field in frontmatter: name")
|
||||
else:
|
||||
errors.extend(_validate_name(metadata["name"], skill_dir))
|
||||
|
||||
if "description" not in metadata:
|
||||
errors.append("Missing required field in frontmatter: description")
|
||||
else:
|
||||
errors.extend(_validate_description(metadata["description"]))
|
||||
|
||||
if "compatibility" in metadata:
|
||||
errors.extend(_validate_compatibility(metadata["compatibility"]))
|
||||
|
||||
return errors
|
||||
|
||||
|
||||
def validate(skill_dir: Path) -> List[str]:
|
||||
"""Validate a skill directory.
|
||||
|
||||
Args:
|
||||
skill_dir: Path to the skill directory
|
||||
|
||||
Returns:
|
||||
List of validation error messages. Empty list means valid.
|
||||
"""
|
||||
skill_dir = Path(skill_dir)
|
||||
|
||||
if not skill_dir.exists():
|
||||
return [f"Path does not exist: {skill_dir}"]
|
||||
|
||||
if not skill_dir.is_dir():
|
||||
return [f"Not a directory: {skill_dir}"]
|
||||
|
||||
skill_md = find_skill_md(skill_dir)
|
||||
if skill_md is None:
|
||||
return ["Missing required file: SKILL.md"]
|
||||
|
||||
try:
|
||||
content = skill_md.read_text()
|
||||
metadata, _ = parse_frontmatter(content)
|
||||
except ParseError as e:
|
||||
return [str(e)]
|
||||
|
||||
return validate_metadata(metadata, skill_dir)
|
||||
@@ -218,6 +218,18 @@ TOOL_MAPPINGS = {
|
||||
'crawl4ai_extract_sync': ('.crawl4ai_tools', None),
|
||||
'Crawl4AITools': ('.crawl4ai_tools', 'Crawl4AITools'),
|
||||
'crawl4ai_tools': ('.crawl4ai_tools', None),
|
||||
|
||||
# Unified Web Search (auto-fallback across providers)
|
||||
'search_web': ('.web_search', None),
|
||||
'get_available_providers': ('.web_search', None),
|
||||
|
||||
# Skill Tools (for Agent Skills script execution)
|
||||
'run_skill_script': ('.skill_tools', None),
|
||||
'read_skill_file': ('.skill_tools', None),
|
||||
'list_skill_scripts': ('.skill_tools', None),
|
||||
'create_skill_tools': ('.skill_tools', None),
|
||||
'SkillTools': ('.skill_tools', 'SkillTools'),
|
||||
'skill_tools': ('.skill_tools', None),
|
||||
}
|
||||
|
||||
_instances = {} # Cache for class instances
|
||||
@@ -254,11 +266,13 @@ def __getattr__(name: str) -> Any:
|
||||
'exa', 'exa_search', 'exa_search_contents', 'exa_find_similar', 'exa_answer',
|
||||
'exa_search_async', 'exa_search_contents_async', 'exa_answer_async',
|
||||
'crawl4ai', 'crawl4ai_many', 'crawl4ai_extract', 'crawl4ai_llm_extract',
|
||||
'crawl4ai_sync', 'crawl4ai_extract_sync'
|
||||
'crawl4ai_sync', 'crawl4ai_extract_sync',
|
||||
'search_web', 'get_available_providers',
|
||||
'run_skill_script', 'read_skill_file', 'list_skill_scripts', 'create_skill_tools'
|
||||
]:
|
||||
return getattr(module, name)
|
||||
if name in ['file_tools', 'pandas_tools', 'wikipedia_tools',
|
||||
'newspaper_tools', 'arxiv_tools', 'spider_tools', 'duckdb_tools', 'mongodb_tools', 'csv_tools', 'json_tools', 'excel_tools', 'xml_tools', 'yaml_tools', 'calculator_tools', 'python_tools', 'shell_tools', 'cot_tools', 'tavily_tools', 'youdotcom_tools', 'exa_tools', 'crawl4ai_tools']:
|
||||
'newspaper_tools', 'arxiv_tools', 'spider_tools', 'duckdb_tools', 'mongodb_tools', 'csv_tools', 'json_tools', 'excel_tools', 'xml_tools', 'yaml_tools', 'calculator_tools', 'python_tools', 'shell_tools', 'cot_tools', 'tavily_tools', 'youdotcom_tools', 'exa_tools', 'crawl4ai_tools', 'skill_tools']:
|
||||
return module # Returns the callable module
|
||||
return getattr(module, name)
|
||||
else:
|
||||
|
||||
@@ -14,6 +14,7 @@ Usage:
|
||||
context, cleaned_prompt = parser.process("@file:main.py explain this code")
|
||||
"""
|
||||
|
||||
import os
|
||||
import re
|
||||
import logging
|
||||
from pathlib import Path
|
||||
@@ -43,10 +44,14 @@ class MentionsParser:
|
||||
"url": re.compile(r'@url:(https?://[^\s]+)'),
|
||||
}
|
||||
|
||||
# Default max file chars: 500K (~125K tokens) - fits GPT-4o (128K), Claude 3.5 (200K)
|
||||
DEFAULT_MAX_FILE_CHARS = 500000
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
workspace_path: Optional[str] = None,
|
||||
verbose: int = 0
|
||||
verbose: int = 0,
|
||||
max_file_chars: Optional[int] = None
|
||||
):
|
||||
"""
|
||||
Initialize MentionsParser.
|
||||
@@ -54,10 +59,24 @@ class MentionsParser:
|
||||
Args:
|
||||
workspace_path: Path to workspace/project root
|
||||
verbose: Verbosity level
|
||||
max_file_chars: Maximum characters to include from files.
|
||||
Default: 500000 (500K chars, ~125K tokens).
|
||||
Set to 0 for no limit.
|
||||
Can be overridden via PRAISON_MAX_FILE_CHARS env var.
|
||||
"""
|
||||
self.workspace_path = Path(workspace_path) if workspace_path else Path.cwd()
|
||||
self.verbose = verbose
|
||||
|
||||
# Configure max file chars: constructor > env var > default
|
||||
if max_file_chars is not None:
|
||||
self.max_file_chars = max_file_chars
|
||||
else:
|
||||
env_val = os.environ.get('PRAISON_MAX_FILE_CHARS')
|
||||
if env_val is not None:
|
||||
self.max_file_chars = int(env_val)
|
||||
else:
|
||||
self.max_file_chars = self.DEFAULT_MAX_FILE_CHARS
|
||||
|
||||
# Lazy-loaded managers
|
||||
self._docs_manager = None
|
||||
self._rules_manager = None
|
||||
@@ -158,9 +177,15 @@ class MentionsParser:
|
||||
|
||||
content = full_path.read_text(encoding="utf-8")
|
||||
|
||||
# Limit content size
|
||||
if len(content) > 50000:
|
||||
content = content[:50000] + "\n... (truncated)"
|
||||
# Limit content size (0 means no limit)
|
||||
if self.max_file_chars > 0 and len(content) > self.max_file_chars:
|
||||
original_size = len(content)
|
||||
content = content[:self.max_file_chars] + "\n... (truncated)"
|
||||
self._log(
|
||||
f"File {file_path} truncated from {original_size:,} to {self.max_file_chars:,} chars. "
|
||||
f"Set PRAISON_MAX_FILE_CHARS=0 for no limit.",
|
||||
logging.WARNING
|
||||
)
|
||||
|
||||
# Detect language for code fence
|
||||
ext = full_path.suffix.lower()
|
||||
@@ -253,9 +278,9 @@ class MentionsParser:
|
||||
with urllib.request.urlopen(req, timeout=10) as response:
|
||||
content = response.read().decode('utf-8', errors='ignore')
|
||||
|
||||
# Limit content size
|
||||
if len(content) > 50000:
|
||||
content = content[:50000] + "\n... (truncated)"
|
||||
# Limit content size (0 means no limit)
|
||||
if self.max_file_chars > 0 and len(content) > self.max_file_chars:
|
||||
content = content[:self.max_file_chars] + "\n... (truncated)"
|
||||
|
||||
# Try to extract text from HTML
|
||||
if '<html' in content.lower():
|
||||
|
||||
@@ -0,0 +1,280 @@
|
||||
"""Tools for executing Agent Skills scripts.
|
||||
|
||||
This module provides tools for running scripts bundled with Agent Skills.
|
||||
Skills are modular packages that extend agent capabilities through SKILL.md
|
||||
files and optional scripts.
|
||||
|
||||
Usage:
|
||||
from praisonaiagents.tools import run_skill_script
|
||||
result = run_skill_script("./my-skill/scripts/skill.py", "./data.csv")
|
||||
"""
|
||||
|
||||
import subprocess
|
||||
import os
|
||||
import logging
|
||||
|
||||
|
||||
class SkillTools:
|
||||
"""Tools for executing Agent Skills scripts."""
|
||||
|
||||
def __init__(self, working_directory: str = None):
|
||||
"""Initialize SkillTools.
|
||||
|
||||
Args:
|
||||
working_directory: Base directory for resolving relative paths.
|
||||
Defaults to current working directory.
|
||||
"""
|
||||
self._working_directory = working_directory or os.getcwd()
|
||||
|
||||
@property
|
||||
def working_directory(self) -> str:
|
||||
"""Get the working directory for path resolution."""
|
||||
return self._working_directory
|
||||
|
||||
@working_directory.setter
|
||||
def working_directory(self, value: str):
|
||||
"""Set the working directory for path resolution."""
|
||||
self._working_directory = value
|
||||
|
||||
def run_skill_script(
|
||||
self,
|
||||
script_path: str,
|
||||
args: str = "",
|
||||
timeout: int = 60
|
||||
) -> str:
|
||||
"""
|
||||
Execute a skill script from a skill's scripts/ directory.
|
||||
|
||||
Args:
|
||||
script_path: Path to the script file (e.g., ./csv-analyzer/scripts/skill.py)
|
||||
args: Arguments to pass to the script (e.g., file path to analyze)
|
||||
timeout: Maximum execution time in seconds (default: 60)
|
||||
|
||||
Returns:
|
||||
The output from running the script, or error message if failed
|
||||
"""
|
||||
try:
|
||||
# Resolve the script path
|
||||
script_path = os.path.expanduser(script_path)
|
||||
if not os.path.isabs(script_path):
|
||||
script_path = os.path.join(self._working_directory, script_path)
|
||||
script_path = os.path.abspath(script_path)
|
||||
|
||||
if not os.path.exists(script_path):
|
||||
return f"Error: Script not found at {script_path}"
|
||||
|
||||
# Resolve args paths relative to working directory
|
||||
resolved_args = []
|
||||
if args:
|
||||
for arg in args.split():
|
||||
# Skip flags
|
||||
if arg.startswith('-'):
|
||||
resolved_args.append(arg)
|
||||
continue
|
||||
|
||||
# Try to resolve as a path relative to working directory
|
||||
potential_path = os.path.join(self._working_directory, arg)
|
||||
if os.path.exists(potential_path):
|
||||
resolved_args.append(os.path.abspath(potential_path))
|
||||
elif os.path.isabs(arg) and os.path.exists(arg):
|
||||
# Already absolute and exists
|
||||
resolved_args.append(arg)
|
||||
else:
|
||||
# Keep original arg if path doesn't exist
|
||||
resolved_args.append(arg)
|
||||
|
||||
# Determine how to run the script based on extension
|
||||
ext = os.path.splitext(script_path)[1].lower()
|
||||
|
||||
if ext == '.py':
|
||||
cmd = ['python', script_path] + resolved_args
|
||||
elif ext == '.sh':
|
||||
cmd = ['bash', script_path] + resolved_args
|
||||
elif ext == '.js':
|
||||
cmd = ['node', script_path] + resolved_args
|
||||
else:
|
||||
# Try to run directly
|
||||
cmd = [script_path] + resolved_args
|
||||
|
||||
logging.debug(f"Running skill script: {' '.join(cmd)}")
|
||||
|
||||
result = subprocess.run(
|
||||
cmd,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=timeout,
|
||||
cwd=self._working_directory
|
||||
)
|
||||
|
||||
output = result.stdout
|
||||
if result.stderr:
|
||||
output += f"\nStderr: {result.stderr}"
|
||||
if result.returncode != 0:
|
||||
output += f"\nExit code: {result.returncode}"
|
||||
|
||||
return output if output else "Script executed successfully (no output)"
|
||||
|
||||
except subprocess.TimeoutExpired:
|
||||
return f"Error: Script execution timed out after {timeout} seconds"
|
||||
except FileNotFoundError as e:
|
||||
return f"Error: Could not find interpreter or script: {str(e)}"
|
||||
except PermissionError as e:
|
||||
return f"Error: Permission denied: {str(e)}"
|
||||
except Exception as e:
|
||||
return f"Error executing script: {str(e)}"
|
||||
|
||||
def read_skill_file(
|
||||
self,
|
||||
skill_path: str,
|
||||
file_path: str,
|
||||
encoding: str = 'utf-8'
|
||||
) -> str:
|
||||
"""
|
||||
Read a file from a skill directory.
|
||||
|
||||
Args:
|
||||
skill_path: Path to the skill directory (e.g., ./csv-analyzer)
|
||||
file_path: Relative path within the skill (e.g., SKILL.md, scripts/skill.py)
|
||||
encoding: File encoding (default: utf-8)
|
||||
|
||||
Returns:
|
||||
File contents or error message
|
||||
"""
|
||||
try:
|
||||
# Resolve skill path
|
||||
skill_path = os.path.expanduser(skill_path)
|
||||
if not os.path.isabs(skill_path):
|
||||
skill_path = os.path.join(self._working_directory, skill_path)
|
||||
skill_path = os.path.abspath(skill_path)
|
||||
|
||||
if not os.path.exists(skill_path):
|
||||
return f"Error: Skill directory not found at {skill_path}"
|
||||
|
||||
if not os.path.isdir(skill_path):
|
||||
return f"Error: {skill_path} is not a directory"
|
||||
|
||||
# Resolve file path within skill
|
||||
full_path = os.path.join(skill_path, file_path)
|
||||
full_path = os.path.abspath(full_path)
|
||||
|
||||
# Security check: ensure file is within skill directory
|
||||
if not full_path.startswith(skill_path):
|
||||
return f"Error: Path traversal detected - {file_path} is outside skill directory"
|
||||
|
||||
if not os.path.exists(full_path):
|
||||
return f"Error: File not found at {full_path}"
|
||||
|
||||
with open(full_path, 'r', encoding=encoding) as f:
|
||||
return f.read()
|
||||
|
||||
except UnicodeDecodeError as e:
|
||||
return f"Error: Could not decode file with {encoding} encoding: {str(e)}"
|
||||
except PermissionError as e:
|
||||
return f"Error: Permission denied reading file: {str(e)}"
|
||||
except Exception as e:
|
||||
return f"Error reading skill file: {str(e)}"
|
||||
|
||||
def list_skill_scripts(self, skill_path: str) -> str:
|
||||
"""
|
||||
List available scripts in a skill's scripts/ directory.
|
||||
|
||||
Args:
|
||||
skill_path: Path to the skill directory
|
||||
|
||||
Returns:
|
||||
JSON string with list of scripts or error message
|
||||
"""
|
||||
import json
|
||||
|
||||
try:
|
||||
# Resolve skill path
|
||||
skill_path = os.path.expanduser(skill_path)
|
||||
if not os.path.isabs(skill_path):
|
||||
skill_path = os.path.join(self._working_directory, skill_path)
|
||||
skill_path = os.path.abspath(skill_path)
|
||||
|
||||
if not os.path.exists(skill_path):
|
||||
return f"Error: Skill directory not found at {skill_path}"
|
||||
|
||||
scripts_dir = os.path.join(skill_path, "scripts")
|
||||
if not os.path.exists(scripts_dir):
|
||||
return json.dumps({"scripts": [], "message": "No scripts/ directory found"})
|
||||
|
||||
scripts = []
|
||||
for item in os.listdir(scripts_dir):
|
||||
item_path = os.path.join(scripts_dir, item)
|
||||
if os.path.isfile(item_path):
|
||||
ext = os.path.splitext(item)[1].lower()
|
||||
if ext in ['.py', '.sh', '.js', '.rb', '.pl']:
|
||||
scripts.append({
|
||||
"name": item,
|
||||
"path": os.path.join("scripts", item),
|
||||
"type": ext[1:] # Remove the dot
|
||||
})
|
||||
|
||||
return json.dumps({"scripts": scripts}, indent=2)
|
||||
|
||||
except Exception as e:
|
||||
return f"Error listing skill scripts: {str(e)}"
|
||||
|
||||
|
||||
# Create default instance for direct function access
|
||||
_skill_tools = SkillTools()
|
||||
|
||||
def run_skill_script(script_path: str, args: str = "", timeout: int = 60) -> str:
|
||||
"""
|
||||
Execute a skill script from a skill's scripts/ directory.
|
||||
|
||||
Args:
|
||||
script_path: Path to the script file (e.g., ./csv-analyzer/scripts/script.py)
|
||||
args: Arguments to pass to the script (e.g., file path to analyze)
|
||||
timeout: Maximum execution time in seconds (default: 60)
|
||||
|
||||
Returns:
|
||||
The output from running the script, or error message if failed
|
||||
"""
|
||||
return _skill_tools.run_skill_script(script_path, args, timeout)
|
||||
|
||||
|
||||
def read_skill_file(skill_path: str, file_path: str, encoding: str = 'utf-8') -> str:
|
||||
"""
|
||||
Read a file from a skill directory.
|
||||
|
||||
Args:
|
||||
skill_path: Path to the skill directory (e.g., ./csv-analyzer)
|
||||
file_path: Relative path within the skill (e.g., SKILL.md, scripts/script.py)
|
||||
encoding: File encoding (default: utf-8)
|
||||
|
||||
Returns:
|
||||
File contents or error message
|
||||
"""
|
||||
return _skill_tools.read_skill_file(skill_path, file_path, encoding)
|
||||
|
||||
|
||||
def list_skill_scripts(skill_path: str) -> str:
|
||||
"""
|
||||
List available scripts in a skill's scripts/ directory.
|
||||
|
||||
Args:
|
||||
skill_path: Path to the skill directory
|
||||
|
||||
Returns:
|
||||
JSON string with list of scripts or error message
|
||||
"""
|
||||
return _skill_tools.list_skill_scripts(skill_path)
|
||||
|
||||
|
||||
def create_skill_tools(working_directory: str = None) -> SkillTools:
|
||||
"""
|
||||
Create a SkillTools instance with a specific working directory.
|
||||
|
||||
This is useful when you need to resolve paths relative to a specific
|
||||
directory rather than the current working directory.
|
||||
|
||||
Args:
|
||||
working_directory: Base directory for resolving relative paths
|
||||
|
||||
Returns:
|
||||
SkillTools instance
|
||||
"""
|
||||
return SkillTools(working_directory)
|
||||
@@ -0,0 +1,329 @@
|
||||
"""Unified web search tool with automatic fallback across multiple providers.
|
||||
|
||||
This module provides a single `search_web` function that automatically tries
|
||||
multiple search providers in order, falling back to the next if one fails.
|
||||
|
||||
Search Provider Priority:
|
||||
1. Tavily (requires TAVILY_API_KEY + tavily-python)
|
||||
2. Exa (requires EXA_API_KEY + exa_py)
|
||||
3. You.com (requires YDC_API_KEY + youdotcom)
|
||||
4. DuckDuckGo (requires duckduckgo_search package, no API key)
|
||||
5. SearxNG (requires requests + running SearxNG instance)
|
||||
|
||||
Usage:
|
||||
from praisonaiagents.tools import search_web
|
||||
|
||||
# Simple search - automatically uses best available provider
|
||||
results = search_web("AI news 2024")
|
||||
|
||||
# With max results
|
||||
results = search_web("Python tutorials", max_results=10)
|
||||
"""
|
||||
|
||||
from typing import List, Dict, Any, Optional
|
||||
import logging
|
||||
import os
|
||||
from importlib import util
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _check_tavily() -> tuple[bool, Optional[str]]:
|
||||
"""Check if Tavily is available."""
|
||||
if util.find_spec("tavily") is None:
|
||||
return False, "tavily-python package not installed"
|
||||
if not os.environ.get("TAVILY_API_KEY"):
|
||||
return False, "TAVILY_API_KEY not set"
|
||||
return True, None
|
||||
|
||||
|
||||
def _check_exa() -> tuple[bool, Optional[str]]:
|
||||
"""Check if Exa is available."""
|
||||
if util.find_spec("exa_py") is None:
|
||||
return False, "exa_py package not installed"
|
||||
if not os.environ.get("EXA_API_KEY"):
|
||||
return False, "EXA_API_KEY not set"
|
||||
return True, None
|
||||
|
||||
|
||||
def _check_youdotcom() -> tuple[bool, Optional[str]]:
|
||||
"""Check if You.com is available."""
|
||||
if util.find_spec("youdotcom") is None:
|
||||
return False, "youdotcom package not installed"
|
||||
if not os.environ.get("YDC_API_KEY"):
|
||||
return False, "YDC_API_KEY not set"
|
||||
return True, None
|
||||
|
||||
|
||||
def _check_duckduckgo() -> tuple[bool, Optional[str]]:
|
||||
"""Check if DuckDuckGo is available."""
|
||||
if util.find_spec("duckduckgo_search") is None:
|
||||
return False, "duckduckgo_search package not installed"
|
||||
return True, None
|
||||
|
||||
|
||||
def _check_searxng() -> tuple[bool, Optional[str]]:
|
||||
"""Check if SearxNG is available."""
|
||||
if util.find_spec("requests") is None:
|
||||
return False, "requests package not installed"
|
||||
return True, None
|
||||
|
||||
|
||||
def _search_tavily(query: str, max_results: int = 5) -> List[Dict[str, Any]]:
|
||||
"""Search using Tavily."""
|
||||
from tavily import TavilyClient
|
||||
|
||||
client = TavilyClient(api_key=os.environ.get("TAVILY_API_KEY"))
|
||||
response = client.search(query=query, max_results=max_results)
|
||||
|
||||
results = []
|
||||
for r in response.get("results", []):
|
||||
results.append({
|
||||
"title": r.get("title", ""),
|
||||
"url": r.get("url", ""),
|
||||
"snippet": r.get("content", ""),
|
||||
"provider": "tavily"
|
||||
})
|
||||
return results
|
||||
|
||||
|
||||
def _search_exa(query: str, max_results: int = 5) -> List[Dict[str, Any]]:
|
||||
"""Search using Exa."""
|
||||
from exa_py import Exa
|
||||
|
||||
client = Exa(os.environ.get("EXA_API_KEY"))
|
||||
response = client.search(query=query, num_results=max_results)
|
||||
|
||||
results = []
|
||||
for r in response.results:
|
||||
results.append({
|
||||
"title": getattr(r, "title", "") or "",
|
||||
"url": r.url,
|
||||
"snippet": getattr(r, "text", "") or "",
|
||||
"provider": "exa"
|
||||
})
|
||||
return results
|
||||
|
||||
|
||||
def _search_youdotcom(query: str, max_results: int = 5) -> List[Dict[str, Any]]:
|
||||
"""Search using You.com."""
|
||||
from youdotcom import You
|
||||
|
||||
client = You(api_key_auth=os.environ.get("YDC_API_KEY"))
|
||||
response = client.search.unified(query=query, count=max_results)
|
||||
|
||||
results = []
|
||||
# Handle response object
|
||||
if hasattr(response, 'results'):
|
||||
web_results = getattr(response.results, 'web', []) or []
|
||||
for r in web_results[:max_results]:
|
||||
results.append({
|
||||
"title": getattr(r, "title", "") or "",
|
||||
"url": getattr(r, "url", "") or "",
|
||||
"snippet": getattr(r, "description", "") or "",
|
||||
"provider": "youdotcom"
|
||||
})
|
||||
elif isinstance(response, dict):
|
||||
web_results = response.get("results", {}).get("web", [])
|
||||
for r in web_results[:max_results]:
|
||||
results.append({
|
||||
"title": r.get("title", ""),
|
||||
"url": r.get("url", ""),
|
||||
"snippet": r.get("description", ""),
|
||||
"provider": "youdotcom"
|
||||
})
|
||||
return results
|
||||
|
||||
|
||||
def _search_duckduckgo(query: str, max_results: int = 5) -> List[Dict[str, Any]]:
|
||||
"""Search using DuckDuckGo."""
|
||||
from duckduckgo_search import DDGS
|
||||
|
||||
results = []
|
||||
ddgs = DDGS()
|
||||
for result in ddgs.text(keywords=query, max_results=max_results):
|
||||
results.append({
|
||||
"title": result.get("title", ""),
|
||||
"url": result.get("href", ""),
|
||||
"snippet": result.get("body", ""),
|
||||
"provider": "duckduckgo"
|
||||
})
|
||||
return results
|
||||
|
||||
|
||||
def _search_searxng(query: str, max_results: int = 5, searxng_url: Optional[str] = None) -> List[Dict[str, Any]]:
|
||||
"""Search using SearxNG."""
|
||||
import requests
|
||||
|
||||
url = searxng_url or os.environ.get("SEARXNG_URL", "http://localhost:32768/search")
|
||||
|
||||
params = {
|
||||
'q': query,
|
||||
'format': 'json',
|
||||
'engines': 'google,bing,duckduckgo',
|
||||
'safesearch': '1'
|
||||
}
|
||||
|
||||
response = requests.get(url, params=params, timeout=10)
|
||||
response.raise_for_status()
|
||||
|
||||
raw_results = response.json().get('results', [])
|
||||
|
||||
results = []
|
||||
for result in raw_results[:max_results]:
|
||||
results.append({
|
||||
"title": result.get("title", ""),
|
||||
"url": result.get("url", ""),
|
||||
"snippet": result.get("content", ""),
|
||||
"provider": "searxng"
|
||||
})
|
||||
return results
|
||||
|
||||
|
||||
# Provider configuration: (name, check_func, search_func)
|
||||
SEARCH_PROVIDERS = [
|
||||
("tavily", _check_tavily, _search_tavily),
|
||||
("exa", _check_exa, _search_exa),
|
||||
("youdotcom", _check_youdotcom, _search_youdotcom),
|
||||
("duckduckgo", _check_duckduckgo, _search_duckduckgo),
|
||||
("searxng", _check_searxng, _search_searxng),
|
||||
]
|
||||
|
||||
|
||||
def search_web(
|
||||
query: str,
|
||||
max_results: int = 5,
|
||||
providers: Optional[List[str]] = None,
|
||||
searxng_url: Optional[str] = None,
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Search the web using multiple providers with automatic fallback.
|
||||
|
||||
Tries each search provider in order until one succeeds. For each provider,
|
||||
it first checks if the API key is set (if required) and if the package is
|
||||
installed, then attempts the search. If the search fails, it moves to the
|
||||
next provider.
|
||||
|
||||
Args:
|
||||
query: The search query string
|
||||
max_results: Maximum number of results to return (default: 5)
|
||||
providers: Optional list of provider names to try, in order.
|
||||
If not specified, uses default order:
|
||||
["tavily", "exa", "youdotcom", "duckduckgo", "searxng"]
|
||||
searxng_url: Optional custom SearxNG instance URL
|
||||
|
||||
Returns:
|
||||
List of search results, each containing:
|
||||
- title: Result title
|
||||
- url: Result URL
|
||||
- snippet: Result description/snippet
|
||||
- provider: Name of the provider that returned the result
|
||||
|
||||
If all providers fail, returns a list with a single error dict.
|
||||
|
||||
Example:
|
||||
# Use default provider order
|
||||
results = search_web("AI news 2024")
|
||||
|
||||
# Specify providers to try
|
||||
results = search_web("Python tutorials", providers=["duckduckgo", "tavily"])
|
||||
|
||||
# With custom SearxNG URL
|
||||
results = search_web("tech news", searxng_url="http://my-searxng:8080/search")
|
||||
"""
|
||||
errors = []
|
||||
|
||||
# Determine which providers to try
|
||||
if providers:
|
||||
# Filter to only valid provider names
|
||||
provider_order = []
|
||||
for name in providers:
|
||||
for pname, check_fn, search_fn in SEARCH_PROVIDERS:
|
||||
if pname == name.lower():
|
||||
provider_order.append((pname, check_fn, search_fn))
|
||||
break
|
||||
else:
|
||||
provider_order = SEARCH_PROVIDERS
|
||||
|
||||
for provider_name, check_func, search_func in provider_order:
|
||||
# Step 1: Check if provider is available (API key + package)
|
||||
is_available, check_error = check_func()
|
||||
if not is_available:
|
||||
logger.debug(f"Skipping {provider_name}: {check_error}")
|
||||
errors.append(f"{provider_name}: {check_error}")
|
||||
continue
|
||||
|
||||
# Step 2: Try to search
|
||||
try:
|
||||
logger.debug(f"Trying {provider_name} for query: {query}")
|
||||
|
||||
if provider_name == "searxng":
|
||||
results = search_func(query, max_results, searxng_url)
|
||||
else:
|
||||
results = search_func(query, max_results)
|
||||
|
||||
if results:
|
||||
logger.info(f"search_web: Successfully used {provider_name}")
|
||||
return results
|
||||
else:
|
||||
errors.append(f"{provider_name}: No results returned")
|
||||
|
||||
except Exception as e:
|
||||
error_msg = f"{provider_name}: {str(e)}"
|
||||
logger.debug(f"Search failed with {provider_name}: {e}")
|
||||
errors.append(error_msg)
|
||||
continue
|
||||
|
||||
# All providers failed
|
||||
error_summary = "; ".join(errors) if errors else "No search providers available"
|
||||
logger.error(f"search_web: All providers failed - {error_summary}")
|
||||
return [{"error": f"All search providers failed: {error_summary}"}]
|
||||
|
||||
|
||||
def get_available_providers() -> List[Dict[str, Any]]:
|
||||
"""Get list of available search providers and their status.
|
||||
|
||||
Returns:
|
||||
List of dicts with provider info:
|
||||
- name: Provider name
|
||||
- available: Whether the provider is ready to use
|
||||
- reason: Reason if not available
|
||||
|
||||
Example:
|
||||
providers = get_available_providers()
|
||||
for p in providers:
|
||||
print(f"{p['name']}: {'✓' if p['available'] else '✗'} {p.get('reason', '')}")
|
||||
"""
|
||||
result = []
|
||||
for name, check_func, _ in SEARCH_PROVIDERS:
|
||||
is_available, error = check_func()
|
||||
result.append({
|
||||
"name": name,
|
||||
"available": is_available,
|
||||
"reason": error if not is_available else None
|
||||
})
|
||||
return result
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
# Example usage and provider status check
|
||||
print("=" * 60)
|
||||
print("Web Search Tool - Provider Status")
|
||||
print("=" * 60)
|
||||
|
||||
providers = get_available_providers()
|
||||
for p in providers:
|
||||
status = "✓ Available" if p["available"] else f"✗ {p['reason']}"
|
||||
print(f" {p['name']:12} {status}")
|
||||
|
||||
print("\n" + "=" * 60)
|
||||
print("Testing search_web...")
|
||||
print("=" * 60)
|
||||
|
||||
results = search_web("Python programming", max_results=3)
|
||||
if results and "error" not in results[0]:
|
||||
print(f"\nFound {len(results)} results using {results[0].get('provider', 'unknown')}:")
|
||||
for r in results:
|
||||
print(f"\n Title: {r.get('title', 'N/A')[:50]}...")
|
||||
print(f" URL: {r.get('url', 'N/A')}")
|
||||
else:
|
||||
print(f"\nSearch failed: {results}")
|
||||
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
||||
|
||||
[project]
|
||||
name = "praisonaiagents"
|
||||
version = "0.1.17"
|
||||
version = "0.1.27"
|
||||
description = "Praison AI agents for completing complex tasks with Self Reflection Agents"
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.10"
|
||||
|
||||
@@ -0,0 +1,217 @@
|
||||
"""
|
||||
Unit tests for configurable max_file_chars in MentionsParser.
|
||||
|
||||
TDD approach: Write tests first, then implement.
|
||||
"""
|
||||
|
||||
import os
|
||||
import pytest
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
|
||||
class TestMentionsParserConfigurableLimit:
|
||||
"""Test suite for configurable file truncation limit."""
|
||||
|
||||
@pytest.fixture
|
||||
def temp_workspace(self):
|
||||
"""Create a temporary workspace with test files."""
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
yield Path(tmpdir)
|
||||
|
||||
@pytest.fixture
|
||||
def small_file(self, temp_workspace):
|
||||
"""Create a small test file (1KB)."""
|
||||
file_path = temp_workspace / "small.py"
|
||||
content = "# Small file\n" + "x = 1\n" * 100 # ~700 chars
|
||||
file_path.write_text(content)
|
||||
return file_path
|
||||
|
||||
@pytest.fixture
|
||||
def large_file(self, temp_workspace):
|
||||
"""Create a large test file (100KB)."""
|
||||
file_path = temp_workspace / "large.py"
|
||||
content = "# Large file\n" + "x = 'a' * 100\n" * 7000 # ~100KB
|
||||
file_path.write_text(content)
|
||||
return file_path
|
||||
|
||||
@pytest.fixture
|
||||
def huge_file(self, temp_workspace):
|
||||
"""Create a huge test file (600KB)."""
|
||||
file_path = temp_workspace / "huge.py"
|
||||
content = "# Huge file\n" + "x = 'a' * 100\n" * 42000 # ~600KB
|
||||
file_path.write_text(content)
|
||||
return file_path
|
||||
|
||||
# ===== Test 1.1: Default value (500K) =====
|
||||
def test_default_max_file_chars_is_500k(self, temp_workspace):
|
||||
"""Default max_file_chars should be 500,000."""
|
||||
from praisonaiagents.tools.mentions import MentionsParser
|
||||
|
||||
parser = MentionsParser(workspace_path=str(temp_workspace))
|
||||
assert parser.max_file_chars == 500000
|
||||
|
||||
# ===== Test 1.2: Custom value via constructor =====
|
||||
def test_custom_max_file_chars_via_constructor(self, temp_workspace):
|
||||
"""max_file_chars should be configurable via constructor."""
|
||||
from praisonaiagents.tools.mentions import MentionsParser
|
||||
|
||||
parser = MentionsParser(
|
||||
workspace_path=str(temp_workspace),
|
||||
max_file_chars=100000
|
||||
)
|
||||
assert parser.max_file_chars == 100000
|
||||
|
||||
def test_custom_max_file_chars_zero_means_no_limit(self, temp_workspace):
|
||||
"""max_file_chars=0 should mean no truncation."""
|
||||
from praisonaiagents.tools.mentions import MentionsParser
|
||||
|
||||
parser = MentionsParser(
|
||||
workspace_path=str(temp_workspace),
|
||||
max_file_chars=0
|
||||
)
|
||||
assert parser.max_file_chars == 0
|
||||
|
||||
# ===== Test 1.3: Environment variable override =====
|
||||
def test_env_var_override(self, temp_workspace):
|
||||
"""PRAISON_MAX_FILE_CHARS env var should override default."""
|
||||
from praisonaiagents.tools.mentions import MentionsParser
|
||||
|
||||
with patch.dict(os.environ, {'PRAISON_MAX_FILE_CHARS': '200000'}):
|
||||
# Need to reimport to pick up env var
|
||||
parser = MentionsParser(workspace_path=str(temp_workspace))
|
||||
assert parser.max_file_chars == 200000
|
||||
|
||||
def test_constructor_takes_precedence_over_env_var(self, temp_workspace):
|
||||
"""Constructor value should take precedence over env var."""
|
||||
from praisonaiagents.tools.mentions import MentionsParser
|
||||
|
||||
with patch.dict(os.environ, {'PRAISON_MAX_FILE_CHARS': '200000'}):
|
||||
parser = MentionsParser(
|
||||
workspace_path=str(temp_workspace),
|
||||
max_file_chars=100000
|
||||
)
|
||||
assert parser.max_file_chars == 100000
|
||||
|
||||
# ===== Test 1.4: No-limit mode =====
|
||||
def test_no_truncation_when_limit_is_zero(self, temp_workspace, huge_file):
|
||||
"""File should not be truncated when max_file_chars=0."""
|
||||
from praisonaiagents.tools.mentions import MentionsParser
|
||||
|
||||
parser = MentionsParser(
|
||||
workspace_path=str(temp_workspace),
|
||||
max_file_chars=0
|
||||
)
|
||||
|
||||
context, _ = parser.process(f"@file:{huge_file.name} explain")
|
||||
|
||||
# Should contain full content, no truncation message
|
||||
assert "(truncated)" not in context
|
||||
# Should have the full file content
|
||||
original_size = huge_file.read_text().__len__()
|
||||
# Context includes header and code fence, so check it's close
|
||||
assert len(context) >= original_size
|
||||
|
||||
# ===== Test 1.5: Truncation behavior =====
|
||||
def test_truncation_at_custom_limit(self, temp_workspace, large_file):
|
||||
"""File should be truncated at custom limit."""
|
||||
from praisonaiagents.tools.mentions import MentionsParser
|
||||
|
||||
parser = MentionsParser(
|
||||
workspace_path=str(temp_workspace),
|
||||
max_file_chars=10000 # 10KB limit
|
||||
)
|
||||
|
||||
context, _ = parser.process(f"@file:{large_file.name} explain")
|
||||
|
||||
# Should be truncated
|
||||
assert "(truncated)" in context
|
||||
# Content should be around the limit (plus header/fence overhead)
|
||||
assert len(context) < 15000 # Some overhead for markdown
|
||||
|
||||
def test_no_truncation_when_file_under_limit(self, temp_workspace, small_file):
|
||||
"""Small files should not be truncated."""
|
||||
from praisonaiagents.tools.mentions import MentionsParser
|
||||
|
||||
parser = MentionsParser(
|
||||
workspace_path=str(temp_workspace),
|
||||
max_file_chars=500000
|
||||
)
|
||||
|
||||
context, _ = parser.process(f"@file:{small_file.name} explain")
|
||||
|
||||
# Should NOT be truncated
|
||||
assert "(truncated)" not in context
|
||||
|
||||
def test_truncation_warning_logged(self, temp_workspace, large_file, caplog):
|
||||
"""Warning should be logged when file is truncated."""
|
||||
from praisonaiagents.tools.mentions import MentionsParser
|
||||
import logging
|
||||
|
||||
parser = MentionsParser(
|
||||
workspace_path=str(temp_workspace),
|
||||
max_file_chars=10000,
|
||||
verbose=1
|
||||
)
|
||||
|
||||
with caplog.at_level(logging.WARNING):
|
||||
parser.process(f"@file:{large_file.name} explain")
|
||||
|
||||
# Should log a warning about truncation
|
||||
assert any("truncat" in record.message.lower() for record in caplog.records)
|
||||
|
||||
# ===== Test: URL truncation also uses configurable limit =====
|
||||
def test_url_truncation_uses_same_limit(self, temp_workspace):
|
||||
"""URL content should also respect max_file_chars."""
|
||||
from praisonaiagents.tools.mentions import MentionsParser
|
||||
|
||||
parser = MentionsParser(
|
||||
workspace_path=str(temp_workspace),
|
||||
max_file_chars=100000
|
||||
)
|
||||
|
||||
# URL truncation should use the same limit
|
||||
assert parser.max_file_chars == 100000
|
||||
|
||||
# ===== Test: Backward compatibility =====
|
||||
def test_backward_compatible_no_args(self, temp_workspace):
|
||||
"""Parser should work without max_file_chars argument."""
|
||||
from praisonaiagents.tools.mentions import MentionsParser
|
||||
|
||||
# Should not raise any errors
|
||||
parser = MentionsParser(workspace_path=str(temp_workspace))
|
||||
assert hasattr(parser, 'max_file_chars')
|
||||
|
||||
# ===== Test: Performance - no impact on small files =====
|
||||
def test_performance_small_file(self, temp_workspace, small_file):
|
||||
"""Processing small files should be fast."""
|
||||
from praisonaiagents.tools.mentions import MentionsParser
|
||||
import time
|
||||
|
||||
parser = MentionsParser(workspace_path=str(temp_workspace))
|
||||
|
||||
start = time.time()
|
||||
for _ in range(100):
|
||||
parser.process(f"@file:{small_file.name} explain")
|
||||
elapsed = time.time() - start
|
||||
|
||||
# 100 iterations should complete in under 1 second
|
||||
assert elapsed < 1.0, f"Performance regression: {elapsed}s for 100 iterations"
|
||||
|
||||
|
||||
class TestProcessMentionsFunction:
|
||||
"""Test the convenience function."""
|
||||
|
||||
def test_process_mentions_uses_default_limit(self):
|
||||
"""process_mentions function should use default limit."""
|
||||
from praisonaiagents.tools.mentions import process_mentions
|
||||
|
||||
# The function should create a parser with default settings
|
||||
# Just verify it works without errors
|
||||
context, cleaned = process_mentions("hello world")
|
||||
assert cleaned == "hello world"
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
pytest.main([__file__, "-v"])
|
||||
@@ -0,0 +1 @@
|
||||
"""Unit tests for PraisonAI Agents evaluation framework."""
|
||||
@@ -0,0 +1,146 @@
|
||||
"""Unit tests for BaseEvaluator class."""
|
||||
|
||||
import pytest
|
||||
import tempfile
|
||||
import json
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from praisonaiagents.eval.base import BaseEvaluator
|
||||
|
||||
|
||||
class ConcreteEvaluator(BaseEvaluator):
|
||||
"""Concrete implementation for testing."""
|
||||
|
||||
def run(self, **kwargs):
|
||||
return {"test": "result"}
|
||||
|
||||
|
||||
class TestBaseEvaluator:
|
||||
"""Tests for BaseEvaluator abstract class."""
|
||||
|
||||
def test_init_default_values(self):
|
||||
"""Test default initialization values."""
|
||||
evaluator = ConcreteEvaluator()
|
||||
assert evaluator.name.startswith("eval_")
|
||||
assert evaluator.eval_id is not None
|
||||
assert len(evaluator.eval_id) == 8
|
||||
assert evaluator.save_results_path is None
|
||||
assert evaluator.verbose is False
|
||||
|
||||
def test_init_custom_values(self):
|
||||
"""Test initialization with custom values."""
|
||||
evaluator = ConcreteEvaluator(
|
||||
name="my_eval",
|
||||
save_results_path="/tmp/results.json",
|
||||
verbose=True
|
||||
)
|
||||
assert evaluator.name == "my_eval"
|
||||
assert evaluator.save_results_path == "/tmp/results.json"
|
||||
assert evaluator.verbose is True
|
||||
|
||||
def test_eval_id_unique(self):
|
||||
"""Test that each evaluator gets a unique ID."""
|
||||
eval1 = ConcreteEvaluator()
|
||||
eval2 = ConcreteEvaluator()
|
||||
assert eval1.eval_id != eval2.eval_id
|
||||
|
||||
def test_before_run_verbose(self):
|
||||
"""Test before_run hook with verbose logging."""
|
||||
evaluator = ConcreteEvaluator(verbose=True)
|
||||
with patch('praisonaiagents.eval.base.logger') as mock_logger:
|
||||
evaluator.before_run()
|
||||
mock_logger.info.assert_called()
|
||||
|
||||
def test_after_run_verbose(self):
|
||||
"""Test after_run hook with verbose logging."""
|
||||
evaluator = ConcreteEvaluator(verbose=True)
|
||||
with patch('praisonaiagents.eval.base.logger') as mock_logger:
|
||||
evaluator.after_run({"result": "test"})
|
||||
mock_logger.info.assert_called()
|
||||
|
||||
def test_save_result_to_file(self):
|
||||
"""Test saving results to file."""
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
path = f"{tmpdir}/results.json"
|
||||
evaluator = ConcreteEvaluator(
|
||||
name="test_save",
|
||||
save_results_path=path
|
||||
)
|
||||
|
||||
result = MagicMock()
|
||||
result.to_dict.return_value = {"score": 8.0}
|
||||
|
||||
evaluator._save_result(result)
|
||||
|
||||
assert Path(path).exists()
|
||||
with open(path) as f:
|
||||
data = json.load(f)
|
||||
assert data["score"] == 8.0
|
||||
|
||||
def test_save_result_with_placeholders(self):
|
||||
"""Test saving results with path placeholders."""
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
path = f"{tmpdir}/{{name}}_{{eval_id}}.json"
|
||||
evaluator = ConcreteEvaluator(
|
||||
name="test_eval",
|
||||
save_results_path=path
|
||||
)
|
||||
|
||||
result = MagicMock()
|
||||
result.to_dict.return_value = {"score": 8.0}
|
||||
|
||||
evaluator._save_result(result)
|
||||
|
||||
expected_path = f"{tmpdir}/test_eval_{evaluator.eval_id}.json"
|
||||
assert Path(expected_path).exists()
|
||||
|
||||
def test_save_result_creates_directories(self):
|
||||
"""Test that save creates parent directories."""
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
path = f"{tmpdir}/nested/dir/results.json"
|
||||
evaluator = ConcreteEvaluator(save_results_path=path)
|
||||
|
||||
result = MagicMock()
|
||||
result.to_dict.return_value = {"score": 8.0}
|
||||
|
||||
evaluator._save_result(result)
|
||||
|
||||
assert Path(path).exists()
|
||||
|
||||
def test_run_abstract_method(self):
|
||||
"""Test that run method works in concrete implementation."""
|
||||
evaluator = ConcreteEvaluator()
|
||||
result = evaluator.run()
|
||||
assert result == {"test": "result"}
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_run_async_default(self):
|
||||
"""Test default async run calls sync run."""
|
||||
evaluator = ConcreteEvaluator()
|
||||
result = await evaluator.run_async()
|
||||
assert result == {"test": "result"}
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_before_run(self):
|
||||
"""Test async before_run hook."""
|
||||
evaluator = ConcreteEvaluator(verbose=True)
|
||||
with patch('praisonaiagents.eval.base.logger') as mock_logger:
|
||||
await evaluator.async_before_run()
|
||||
mock_logger.info.assert_called()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_after_run(self):
|
||||
"""Test async after_run hook."""
|
||||
evaluator = ConcreteEvaluator(verbose=True)
|
||||
with patch('praisonaiagents.eval.base.logger') as mock_logger:
|
||||
await evaluator.async_after_run({"result": "test"})
|
||||
mock_logger.info.assert_called()
|
||||
|
||||
def test_repr(self):
|
||||
"""Test string representation."""
|
||||
evaluator = ConcreteEvaluator(name="test_eval")
|
||||
repr_str = repr(evaluator)
|
||||
assert "ConcreteEvaluator" in repr_str
|
||||
assert "test_eval" in repr_str
|
||||
assert evaluator.eval_id in repr_str
|
||||
@@ -0,0 +1,425 @@
|
||||
"""Unit tests for evaluator classes."""
|
||||
|
||||
import pytest
|
||||
import time
|
||||
from unittest.mock import MagicMock, patch, AsyncMock
|
||||
|
||||
from praisonaiagents.eval.accuracy import AccuracyEvaluator
|
||||
from praisonaiagents.eval.performance import PerformanceEvaluator
|
||||
from praisonaiagents.eval.reliability import ReliabilityEvaluator
|
||||
from praisonaiagents.eval.criteria import CriteriaEvaluator
|
||||
from praisonaiagents.eval.results import (
|
||||
AccuracyResult,
|
||||
PerformanceResult,
|
||||
ReliabilityResult,
|
||||
CriteriaResult,
|
||||
)
|
||||
|
||||
|
||||
class TestAccuracyEvaluator:
|
||||
"""Tests for AccuracyEvaluator class."""
|
||||
|
||||
def test_init_with_func(self):
|
||||
"""Test initialization with function."""
|
||||
func = lambda x: "test output"
|
||||
evaluator = AccuracyEvaluator(
|
||||
func=func,
|
||||
input_text="test input",
|
||||
expected_output="test output"
|
||||
)
|
||||
assert evaluator.func == func
|
||||
assert evaluator.input_text == "test input"
|
||||
assert evaluator.expected_output == "test output"
|
||||
|
||||
def test_init_with_agent(self):
|
||||
"""Test initialization with agent."""
|
||||
agent = MagicMock()
|
||||
evaluator = AccuracyEvaluator(
|
||||
agent=agent,
|
||||
input_text="test",
|
||||
expected_output="expected"
|
||||
)
|
||||
assert evaluator.agent == agent
|
||||
|
||||
def test_init_requires_agent_or_func(self):
|
||||
"""Test that either agent or func must be provided."""
|
||||
with pytest.raises(ValueError, match="Either 'agent' or 'func' must be provided"):
|
||||
AccuracyEvaluator(
|
||||
input_text="test",
|
||||
expected_output="expected"
|
||||
)
|
||||
|
||||
def test_get_output_from_func(self):
|
||||
"""Test getting output from function."""
|
||||
func = lambda x: f"output for {x}"
|
||||
evaluator = AccuracyEvaluator(
|
||||
func=func,
|
||||
input_text="test",
|
||||
expected_output="expected"
|
||||
)
|
||||
output = evaluator._get_output()
|
||||
assert output == "output for test"
|
||||
|
||||
def test_get_output_from_agent_chat(self):
|
||||
"""Test getting output from agent with chat method."""
|
||||
agent = MagicMock()
|
||||
agent.chat.return_value = "agent response"
|
||||
|
||||
evaluator = AccuracyEvaluator(
|
||||
agent=agent,
|
||||
input_text="test",
|
||||
expected_output="expected"
|
||||
)
|
||||
output = evaluator._get_output()
|
||||
assert output == "agent response"
|
||||
agent.chat.assert_called_once_with("test")
|
||||
|
||||
def test_get_output_from_agent_start(self):
|
||||
"""Test getting output from agent with start method."""
|
||||
agent = MagicMock(spec=['start'])
|
||||
result = MagicMock()
|
||||
result.raw = "agent result"
|
||||
agent.start.return_value = result
|
||||
|
||||
evaluator = AccuracyEvaluator(
|
||||
agent=agent,
|
||||
input_text="test",
|
||||
expected_output="expected"
|
||||
)
|
||||
output = evaluator._get_output()
|
||||
assert output == "agent result"
|
||||
|
||||
def test_judge_output(self):
|
||||
"""Test LLM judging of output."""
|
||||
with patch('openai.OpenAI') as mock_openai_class:
|
||||
mock_client = MagicMock()
|
||||
mock_openai_class.return_value = mock_client
|
||||
|
||||
mock_response = MagicMock()
|
||||
mock_response.choices = [MagicMock()]
|
||||
mock_response.choices[0].message.content = "SCORE: 8\nREASONING: Good match"
|
||||
mock_client.chat.completions.create.return_value = mock_response
|
||||
|
||||
def test_func(x):
|
||||
return "4"
|
||||
|
||||
evaluator = AccuracyEvaluator(
|
||||
func=test_func,
|
||||
input_text="What is 2+2?",
|
||||
expected_output="4"
|
||||
)
|
||||
|
||||
score = evaluator._judge_output("4")
|
||||
assert score.score == 8.0
|
||||
assert "Good match" in score.reasoning
|
||||
|
||||
def test_run_returns_accuracy_result(self):
|
||||
"""Test that run returns AccuracyResult."""
|
||||
with patch('openai.OpenAI') as mock_openai_class:
|
||||
mock_client = MagicMock()
|
||||
mock_openai_class.return_value = mock_client
|
||||
|
||||
mock_response = MagicMock()
|
||||
mock_response.choices = [MagicMock()]
|
||||
mock_response.choices[0].message.content = "SCORE: 9\nREASONING: Excellent"
|
||||
mock_client.chat.completions.create.return_value = mock_response
|
||||
|
||||
def test_func(x):
|
||||
return "4"
|
||||
|
||||
evaluator = AccuracyEvaluator(
|
||||
func=test_func,
|
||||
input_text="What is 2+2?",
|
||||
expected_output="4",
|
||||
num_iterations=2
|
||||
)
|
||||
|
||||
result = evaluator.run()
|
||||
assert isinstance(result, AccuracyResult)
|
||||
assert len(result.evaluations) == 2
|
||||
|
||||
def test_evaluate_output(self):
|
||||
"""Test evaluate_output method for pre-generated outputs."""
|
||||
with patch('openai.OpenAI') as mock_openai_class:
|
||||
mock_client = MagicMock()
|
||||
mock_openai_class.return_value = mock_client
|
||||
|
||||
mock_response = MagicMock()
|
||||
mock_response.choices = [MagicMock()]
|
||||
mock_response.choices[0].message.content = "SCORE: 10\nREASONING: Perfect"
|
||||
mock_client.chat.completions.create.return_value = mock_response
|
||||
|
||||
def test_func(x):
|
||||
return "unused"
|
||||
|
||||
evaluator = AccuracyEvaluator(
|
||||
func=test_func,
|
||||
input_text="What is 2+2?",
|
||||
expected_output="4"
|
||||
)
|
||||
|
||||
result = evaluator.evaluate_output("4")
|
||||
assert isinstance(result, AccuracyResult)
|
||||
assert len(result.evaluations) == 1
|
||||
assert result.evaluations[0].score == 10.0
|
||||
|
||||
|
||||
class TestPerformanceEvaluator:
|
||||
"""Tests for PerformanceEvaluator class."""
|
||||
|
||||
def test_init_with_func(self):
|
||||
"""Test initialization with function."""
|
||||
func = lambda: time.sleep(0.01)
|
||||
evaluator = PerformanceEvaluator(
|
||||
func=func,
|
||||
num_iterations=5,
|
||||
warmup_runs=2
|
||||
)
|
||||
assert evaluator.func == func
|
||||
assert evaluator.num_iterations == 5
|
||||
assert evaluator.warmup_runs == 2
|
||||
|
||||
def test_init_with_agent(self):
|
||||
"""Test initialization with agent."""
|
||||
agent = MagicMock()
|
||||
evaluator = PerformanceEvaluator(
|
||||
agent=agent,
|
||||
input_text="test",
|
||||
num_iterations=3
|
||||
)
|
||||
assert evaluator.agent == agent
|
||||
assert evaluator.input_text == "test"
|
||||
|
||||
def test_init_requires_func_or_agent(self):
|
||||
"""Test that either func or agent must be provided."""
|
||||
with pytest.raises(ValueError, match="Either 'func' or 'agent' must be provided"):
|
||||
PerformanceEvaluator()
|
||||
|
||||
def test_run_measures_time(self):
|
||||
"""Test that run measures execution time."""
|
||||
call_count = 0
|
||||
def slow_func():
|
||||
nonlocal call_count
|
||||
call_count += 1
|
||||
time.sleep(0.01)
|
||||
|
||||
evaluator = PerformanceEvaluator(
|
||||
func=slow_func,
|
||||
num_iterations=3,
|
||||
warmup_runs=1,
|
||||
track_memory=False
|
||||
)
|
||||
|
||||
result = evaluator.run()
|
||||
assert isinstance(result, PerformanceResult)
|
||||
assert len(result.metrics) == 3
|
||||
assert all(m.run_time >= 0.01 for m in result.metrics)
|
||||
assert call_count == 4 # 1 warmup + 3 iterations
|
||||
|
||||
def test_run_with_agent(self):
|
||||
"""Test run with agent."""
|
||||
agent = MagicMock()
|
||||
agent.chat.return_value = "response"
|
||||
|
||||
evaluator = PerformanceEvaluator(
|
||||
agent=agent,
|
||||
input_text="test",
|
||||
num_iterations=2,
|
||||
warmup_runs=1,
|
||||
track_memory=False
|
||||
)
|
||||
|
||||
result = evaluator.run()
|
||||
assert isinstance(result, PerformanceResult)
|
||||
assert len(result.metrics) == 2
|
||||
assert agent.chat.call_count == 3 # 1 warmup + 2 iterations
|
||||
|
||||
|
||||
class TestReliabilityEvaluator:
|
||||
"""Tests for ReliabilityEvaluator class."""
|
||||
|
||||
def test_init(self):
|
||||
"""Test initialization."""
|
||||
agent = MagicMock()
|
||||
evaluator = ReliabilityEvaluator(
|
||||
agent=agent,
|
||||
input_text="test",
|
||||
expected_tools=["search", "calculate"]
|
||||
)
|
||||
assert evaluator.agent == agent
|
||||
assert evaluator.expected_tools == {"search", "calculate"}
|
||||
|
||||
def test_init_requires_agent(self):
|
||||
"""Test that agent must be provided."""
|
||||
with pytest.raises(ValueError, match="'agent' must be provided"):
|
||||
ReliabilityEvaluator(
|
||||
input_text="test",
|
||||
expected_tools=["search"]
|
||||
)
|
||||
|
||||
def test_evaluate_tool_calls(self):
|
||||
"""Test evaluate_tool_calls method."""
|
||||
agent = MagicMock()
|
||||
evaluator = ReliabilityEvaluator(
|
||||
agent=agent,
|
||||
input_text="test",
|
||||
expected_tools=["search", "calculate"],
|
||||
forbidden_tools=["delete"]
|
||||
)
|
||||
|
||||
result = evaluator.evaluate_tool_calls(["search", "calculate"])
|
||||
assert isinstance(result, ReliabilityResult)
|
||||
assert result.passed is True
|
||||
assert len(result.passed_calls) == 3 # 2 expected + 1 forbidden not called
|
||||
|
||||
def test_evaluate_tool_calls_with_failures(self):
|
||||
"""Test evaluate_tool_calls with missing tools."""
|
||||
agent = MagicMock()
|
||||
evaluator = ReliabilityEvaluator(
|
||||
agent=agent,
|
||||
input_text="test",
|
||||
expected_tools=["search", "calculate"]
|
||||
)
|
||||
|
||||
result = evaluator.evaluate_tool_calls(["search"]) # missing calculate
|
||||
assert result.passed is False
|
||||
assert len(result.failed_calls) == 1
|
||||
|
||||
def test_forbidden_tools(self):
|
||||
"""Test forbidden tools detection."""
|
||||
agent = MagicMock()
|
||||
evaluator = ReliabilityEvaluator(
|
||||
agent=agent,
|
||||
input_text="test",
|
||||
expected_tools=[],
|
||||
forbidden_tools=["delete", "drop"]
|
||||
)
|
||||
|
||||
result = evaluator.evaluate_tool_calls(["delete"]) # called forbidden tool
|
||||
assert result.passed is False
|
||||
assert len(result.failed_calls) == 1
|
||||
|
||||
|
||||
class TestCriteriaEvaluator:
|
||||
"""Tests for CriteriaEvaluator class."""
|
||||
|
||||
def test_init(self):
|
||||
"""Test initialization."""
|
||||
evaluator = CriteriaEvaluator(
|
||||
criteria="Response is helpful",
|
||||
func=lambda x: "test",
|
||||
input_text="test"
|
||||
)
|
||||
assert evaluator.criteria == "Response is helpful"
|
||||
assert evaluator.scoring_type == "numeric"
|
||||
assert evaluator.threshold == 7.0
|
||||
|
||||
def test_init_binary_scoring(self):
|
||||
"""Test initialization with binary scoring."""
|
||||
evaluator = CriteriaEvaluator(
|
||||
criteria="Response is helpful",
|
||||
func=lambda x: "test",
|
||||
scoring_type="binary"
|
||||
)
|
||||
assert evaluator.scoring_type == "binary"
|
||||
|
||||
def test_judge_output_numeric(self):
|
||||
"""Test numeric scoring."""
|
||||
with patch('openai.OpenAI') as mock_openai_class:
|
||||
mock_client = MagicMock()
|
||||
mock_openai_class.return_value = mock_client
|
||||
|
||||
mock_response = MagicMock()
|
||||
mock_response.choices = [MagicMock()]
|
||||
mock_response.choices[0].message.content = "SCORE: 8\nREASONING: Good"
|
||||
mock_client.chat.completions.create.return_value = mock_response
|
||||
|
||||
def test_func(x):
|
||||
return "test"
|
||||
|
||||
evaluator = CriteriaEvaluator(
|
||||
criteria="Be helpful",
|
||||
func=test_func,
|
||||
scoring_type="numeric",
|
||||
threshold=7.0
|
||||
)
|
||||
|
||||
score = evaluator._judge_output("test output")
|
||||
assert score.score == 8.0
|
||||
assert score.passed is True
|
||||
|
||||
def test_judge_output_binary(self):
|
||||
"""Test binary scoring."""
|
||||
with patch('openai.OpenAI') as mock_openai_class:
|
||||
mock_client = MagicMock()
|
||||
mock_openai_class.return_value = mock_client
|
||||
|
||||
mock_response = MagicMock()
|
||||
mock_response.choices = [MagicMock()]
|
||||
mock_response.choices[0].message.content = "RESULT: PASS\nREASONING: Meets criteria"
|
||||
mock_client.chat.completions.create.return_value = mock_response
|
||||
|
||||
def test_func(x):
|
||||
return "test"
|
||||
|
||||
evaluator = CriteriaEvaluator(
|
||||
criteria="Be helpful",
|
||||
func=test_func,
|
||||
scoring_type="binary"
|
||||
)
|
||||
|
||||
score = evaluator._judge_output("test output")
|
||||
assert score.score == 10.0
|
||||
assert score.passed is True
|
||||
|
||||
def test_on_fail_callback(self):
|
||||
"""Test on_fail callback is called."""
|
||||
with patch('openai.OpenAI') as mock_openai_class:
|
||||
mock_client = MagicMock()
|
||||
mock_openai_class.return_value = mock_client
|
||||
|
||||
mock_response = MagicMock()
|
||||
mock_response.choices = [MagicMock()]
|
||||
mock_response.choices[0].message.content = "SCORE: 3\nREASONING: Poor"
|
||||
mock_client.chat.completions.create.return_value = mock_response
|
||||
|
||||
on_fail_called = []
|
||||
def on_fail(score):
|
||||
on_fail_called.append(score)
|
||||
|
||||
def test_func(x):
|
||||
return "test"
|
||||
|
||||
evaluator = CriteriaEvaluator(
|
||||
criteria="Be helpful",
|
||||
func=test_func,
|
||||
threshold=7.0,
|
||||
on_fail=on_fail
|
||||
)
|
||||
|
||||
evaluator.run()
|
||||
assert len(on_fail_called) == 1
|
||||
assert on_fail_called[0].score == 3.0
|
||||
|
||||
def test_evaluate_output_criteria(self):
|
||||
"""Test evaluate_output method."""
|
||||
with patch('openai.OpenAI') as mock_openai_class:
|
||||
mock_client = MagicMock()
|
||||
mock_openai_class.return_value = mock_client
|
||||
|
||||
mock_response = MagicMock()
|
||||
mock_response.choices = [MagicMock()]
|
||||
mock_response.choices[0].message.content = "SCORE: 9\nREASONING: Great"
|
||||
mock_client.chat.completions.create.return_value = mock_response
|
||||
|
||||
def test_func(x):
|
||||
return "unused"
|
||||
|
||||
evaluator = CriteriaEvaluator(
|
||||
criteria="Be helpful",
|
||||
func=test_func
|
||||
)
|
||||
|
||||
result = evaluator.evaluate_output("pre-generated output")
|
||||
assert isinstance(result, CriteriaResult)
|
||||
assert len(result.evaluations) == 1
|
||||
@@ -0,0 +1,359 @@
|
||||
"""Unit tests for evaluation result dataclasses."""
|
||||
|
||||
import pytest
|
||||
import json
|
||||
from datetime import datetime
|
||||
|
||||
from praisonaiagents.eval.results import (
|
||||
EvaluationScore,
|
||||
AccuracyResult,
|
||||
PerformanceMetrics,
|
||||
PerformanceResult,
|
||||
ToolCallResult,
|
||||
ReliabilityResult,
|
||||
CriteriaScore,
|
||||
CriteriaResult,
|
||||
)
|
||||
|
||||
|
||||
class TestEvaluationScore:
|
||||
"""Tests for EvaluationScore dataclass."""
|
||||
|
||||
def test_creation(self):
|
||||
"""Test basic creation of EvaluationScore."""
|
||||
score = EvaluationScore(
|
||||
score=8.5,
|
||||
reasoning="Good response",
|
||||
input_text="What is 2+2?",
|
||||
output_text="4",
|
||||
expected_output="4"
|
||||
)
|
||||
assert score.score == 8.5
|
||||
assert score.reasoning == "Good response"
|
||||
assert score.input_text == "What is 2+2?"
|
||||
assert score.output_text == "4"
|
||||
assert score.expected_output == "4"
|
||||
|
||||
def test_timestamp_auto_generated(self):
|
||||
"""Test that timestamp is auto-generated."""
|
||||
score = EvaluationScore(
|
||||
score=8.0,
|
||||
reasoning="Test",
|
||||
input_text="test",
|
||||
output_text="test"
|
||||
)
|
||||
assert score.timestamp is not None
|
||||
datetime.fromisoformat(score.timestamp)
|
||||
|
||||
def test_metadata_default(self):
|
||||
"""Test that metadata defaults to empty dict."""
|
||||
score = EvaluationScore(
|
||||
score=8.0,
|
||||
reasoning="Test",
|
||||
input_text="test",
|
||||
output_text="test"
|
||||
)
|
||||
assert score.metadata == {}
|
||||
|
||||
|
||||
class TestAccuracyResult:
|
||||
"""Tests for AccuracyResult dataclass."""
|
||||
|
||||
def test_empty_result(self):
|
||||
"""Test empty AccuracyResult."""
|
||||
result = AccuracyResult()
|
||||
assert result.evaluations == []
|
||||
assert result.avg_score == 0.0
|
||||
assert result.min_score == 0.0
|
||||
assert result.max_score == 0.0
|
||||
assert result.std_dev == 0.0
|
||||
assert result.passed is False
|
||||
|
||||
def test_single_evaluation(self):
|
||||
"""Test AccuracyResult with single evaluation."""
|
||||
result = AccuracyResult()
|
||||
result.evaluations.append(EvaluationScore(
|
||||
score=8.0,
|
||||
reasoning="Good",
|
||||
input_text="test",
|
||||
output_text="test"
|
||||
))
|
||||
assert result.avg_score == 8.0
|
||||
assert result.min_score == 8.0
|
||||
assert result.max_score == 8.0
|
||||
assert result.std_dev == 0.0
|
||||
assert result.passed is True
|
||||
|
||||
def test_multiple_evaluations(self):
|
||||
"""Test AccuracyResult with multiple evaluations."""
|
||||
result = AccuracyResult()
|
||||
for score in [6.0, 8.0, 10.0]:
|
||||
result.evaluations.append(EvaluationScore(
|
||||
score=score,
|
||||
reasoning="Test",
|
||||
input_text="test",
|
||||
output_text="test"
|
||||
))
|
||||
assert result.avg_score == 8.0
|
||||
assert result.min_score == 6.0
|
||||
assert result.max_score == 10.0
|
||||
assert result.passed is True
|
||||
|
||||
def test_passed_threshold(self):
|
||||
"""Test passed property with different scores."""
|
||||
result = AccuracyResult()
|
||||
result.evaluations.append(EvaluationScore(
|
||||
score=6.0,
|
||||
reasoning="Below threshold",
|
||||
input_text="test",
|
||||
output_text="test"
|
||||
))
|
||||
assert result.passed is False
|
||||
|
||||
result.evaluations.append(EvaluationScore(
|
||||
score=8.0,
|
||||
reasoning="Above threshold",
|
||||
input_text="test",
|
||||
output_text="test"
|
||||
))
|
||||
assert result.passed is True
|
||||
|
||||
def test_to_dict(self):
|
||||
"""Test to_dict method."""
|
||||
result = AccuracyResult(eval_id="test123", name="test_eval")
|
||||
result.evaluations.append(EvaluationScore(
|
||||
score=8.0,
|
||||
reasoning="Good",
|
||||
input_text="test",
|
||||
output_text="test"
|
||||
))
|
||||
d = result.to_dict()
|
||||
assert d["eval_id"] == "test123"
|
||||
assert d["name"] == "test_eval"
|
||||
assert d["avg_score"] == 8.0
|
||||
assert d["num_evaluations"] == 1
|
||||
|
||||
def test_to_json(self):
|
||||
"""Test to_json method."""
|
||||
result = AccuracyResult(eval_id="test123")
|
||||
result.evaluations.append(EvaluationScore(
|
||||
score=8.0,
|
||||
reasoning="Good",
|
||||
input_text="test",
|
||||
output_text="test"
|
||||
))
|
||||
json_str = result.to_json()
|
||||
data = json.loads(json_str)
|
||||
assert data["eval_id"] == "test123"
|
||||
|
||||
|
||||
class TestPerformanceMetrics:
|
||||
"""Tests for PerformanceMetrics dataclass."""
|
||||
|
||||
def test_creation(self):
|
||||
"""Test basic creation."""
|
||||
metrics = PerformanceMetrics(
|
||||
run_time=0.5,
|
||||
memory_usage=10.5,
|
||||
iteration=1
|
||||
)
|
||||
assert metrics.run_time == 0.5
|
||||
assert metrics.memory_usage == 10.5
|
||||
assert metrics.iteration == 1
|
||||
|
||||
|
||||
class TestPerformanceResult:
|
||||
"""Tests for PerformanceResult dataclass."""
|
||||
|
||||
def test_empty_result(self):
|
||||
"""Test empty PerformanceResult."""
|
||||
result = PerformanceResult()
|
||||
assert result.metrics == []
|
||||
assert result.avg_run_time == 0.0
|
||||
assert result.min_run_time == 0.0
|
||||
assert result.max_run_time == 0.0
|
||||
assert result.avg_memory == 0.0
|
||||
|
||||
def test_with_metrics(self):
|
||||
"""Test PerformanceResult with metrics."""
|
||||
result = PerformanceResult()
|
||||
for i, (time, mem) in enumerate([(0.1, 5.0), (0.2, 10.0), (0.3, 15.0)]):
|
||||
result.metrics.append(PerformanceMetrics(
|
||||
run_time=time,
|
||||
memory_usage=mem,
|
||||
iteration=i + 1
|
||||
))
|
||||
assert result.avg_run_time == 0.2
|
||||
assert result.min_run_time == 0.1
|
||||
assert result.max_run_time == 0.3
|
||||
assert result.avg_memory == 10.0
|
||||
assert result.max_memory == 15.0
|
||||
|
||||
def test_median_and_p95(self):
|
||||
"""Test median and p95 calculations."""
|
||||
result = PerformanceResult()
|
||||
for i in range(100):
|
||||
result.metrics.append(PerformanceMetrics(
|
||||
run_time=float(i + 1) / 100,
|
||||
memory_usage=1.0,
|
||||
iteration=i + 1
|
||||
))
|
||||
assert 0.49 <= result.median_run_time <= 0.51
|
||||
assert result.p95_run_time >= 0.95
|
||||
|
||||
def test_to_dict(self):
|
||||
"""Test to_dict method."""
|
||||
result = PerformanceResult(warmup_runs=5, eval_id="perf123")
|
||||
result.metrics.append(PerformanceMetrics(
|
||||
run_time=0.5,
|
||||
memory_usage=10.0,
|
||||
iteration=1
|
||||
))
|
||||
d = result.to_dict()
|
||||
assert d["warmup_runs"] == 5
|
||||
assert d["eval_id"] == "perf123"
|
||||
assert d["num_iterations"] == 1
|
||||
|
||||
|
||||
class TestToolCallResult:
|
||||
"""Tests for ToolCallResult dataclass."""
|
||||
|
||||
def test_creation(self):
|
||||
"""Test basic creation."""
|
||||
tc = ToolCallResult(
|
||||
tool_name="search",
|
||||
expected=True,
|
||||
actual=True,
|
||||
passed=True
|
||||
)
|
||||
assert tc.tool_name == "search"
|
||||
assert tc.expected is True
|
||||
assert tc.actual is True
|
||||
assert tc.passed is True
|
||||
|
||||
|
||||
class TestReliabilityResult:
|
||||
"""Tests for ReliabilityResult dataclass."""
|
||||
|
||||
def test_empty_result(self):
|
||||
"""Test empty ReliabilityResult."""
|
||||
result = ReliabilityResult()
|
||||
assert result.tool_results == []
|
||||
assert result.pass_rate == 0.0
|
||||
assert result.status == "PASSED"
|
||||
assert result.passed is True
|
||||
|
||||
def test_with_tool_calls(self):
|
||||
"""Test ReliabilityResult with tool calls."""
|
||||
result = ReliabilityResult()
|
||||
result.tool_results.append(ToolCallResult(
|
||||
tool_name="search",
|
||||
expected=True,
|
||||
actual=True,
|
||||
passed=True
|
||||
))
|
||||
result.tool_results.append(ToolCallResult(
|
||||
tool_name="calculate",
|
||||
expected=True,
|
||||
actual=False,
|
||||
passed=False
|
||||
))
|
||||
assert len(result.passed_calls) == 1
|
||||
assert len(result.failed_calls) == 1
|
||||
assert result.pass_rate == 0.5
|
||||
assert result.status == "FAILED"
|
||||
assert result.passed is False
|
||||
|
||||
def test_all_passed(self):
|
||||
"""Test when all tool calls pass."""
|
||||
result = ReliabilityResult()
|
||||
for name in ["tool1", "tool2", "tool3"]:
|
||||
result.tool_results.append(ToolCallResult(
|
||||
tool_name=name,
|
||||
expected=True,
|
||||
actual=True,
|
||||
passed=True
|
||||
))
|
||||
assert result.pass_rate == 1.0
|
||||
assert result.passed is True
|
||||
|
||||
|
||||
class TestCriteriaScore:
|
||||
"""Tests for CriteriaScore dataclass."""
|
||||
|
||||
def test_creation(self):
|
||||
"""Test basic creation."""
|
||||
score = CriteriaScore(
|
||||
score=8.0,
|
||||
passed=True,
|
||||
reasoning="Meets criteria",
|
||||
output_text="test output",
|
||||
criteria="Be helpful"
|
||||
)
|
||||
assert score.score == 8.0
|
||||
assert score.passed is True
|
||||
assert score.reasoning == "Meets criteria"
|
||||
|
||||
|
||||
class TestCriteriaResult:
|
||||
"""Tests for CriteriaResult dataclass."""
|
||||
|
||||
def test_empty_result(self):
|
||||
"""Test empty CriteriaResult."""
|
||||
result = CriteriaResult(criteria="Be helpful")
|
||||
assert result.evaluations == []
|
||||
assert result.avg_score == 0.0
|
||||
assert result.pass_rate == 0.0
|
||||
|
||||
def test_numeric_scoring(self):
|
||||
"""Test numeric scoring mode."""
|
||||
result = CriteriaResult(
|
||||
criteria="Be helpful",
|
||||
scoring_type="numeric",
|
||||
threshold=7.0
|
||||
)
|
||||
result.evaluations.append(CriteriaScore(
|
||||
score=8.0,
|
||||
passed=True,
|
||||
reasoning="Good",
|
||||
output_text="test",
|
||||
criteria="Be helpful"
|
||||
))
|
||||
assert result.avg_score == 8.0
|
||||
assert result.passed is True
|
||||
|
||||
def test_binary_scoring(self):
|
||||
"""Test binary scoring mode."""
|
||||
result = CriteriaResult(
|
||||
criteria="Be helpful",
|
||||
scoring_type="binary"
|
||||
)
|
||||
result.evaluations.append(CriteriaScore(
|
||||
score=10.0,
|
||||
passed=True,
|
||||
reasoning="Pass",
|
||||
output_text="test",
|
||||
criteria="Be helpful"
|
||||
))
|
||||
result.evaluations.append(CriteriaScore(
|
||||
score=0.0,
|
||||
passed=False,
|
||||
reasoning="Fail",
|
||||
output_text="test",
|
||||
criteria="Be helpful"
|
||||
))
|
||||
assert result.pass_rate == 0.5
|
||||
assert result.passed is True
|
||||
|
||||
def test_to_dict(self):
|
||||
"""Test to_dict method."""
|
||||
result = CriteriaResult(
|
||||
criteria="Be helpful",
|
||||
scoring_type="numeric",
|
||||
threshold=7.0,
|
||||
eval_id="crit123"
|
||||
)
|
||||
d = result.to_dict()
|
||||
assert d["criteria"] == "Be helpful"
|
||||
assert d["scoring_type"] == "numeric"
|
||||
assert d["threshold"] == 7.0
|
||||
@@ -0,0 +1,222 @@
|
||||
"""Unit tests for evaluation utility functions."""
|
||||
|
||||
import json
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from praisonaiagents.eval.utils import (
|
||||
save_result_to_file,
|
||||
load_result_from_file,
|
||||
format_score,
|
||||
format_percentage,
|
||||
format_duration,
|
||||
format_memory,
|
||||
calculate_pass_rate,
|
||||
)
|
||||
|
||||
|
||||
class TestSaveResultToFile:
|
||||
"""Tests for save_result_to_file function."""
|
||||
|
||||
def test_save_with_to_dict(self):
|
||||
"""Test saving result with to_dict method."""
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
path = f"{tmpdir}/result.json"
|
||||
result = MagicMock()
|
||||
result.to_dict.return_value = {"score": 8.0, "passed": True}
|
||||
|
||||
success = save_result_to_file(result, path)
|
||||
|
||||
assert success is True
|
||||
assert Path(path).exists()
|
||||
with open(path) as f:
|
||||
data = json.load(f)
|
||||
assert data["score"] == 8.0
|
||||
|
||||
def test_save_with_dict_attr(self):
|
||||
"""Test saving result with __dict__ attribute."""
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
path = f"{tmpdir}/result.json"
|
||||
|
||||
class SimpleResult:
|
||||
def __init__(self):
|
||||
self.score = 9.0
|
||||
self.name = "test"
|
||||
|
||||
result = SimpleResult()
|
||||
success = save_result_to_file(result, path)
|
||||
|
||||
assert success is True
|
||||
with open(path) as f:
|
||||
data = json.load(f)
|
||||
assert data["score"] == 9.0
|
||||
|
||||
def test_save_with_placeholders(self):
|
||||
"""Test saving with path placeholders."""
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
path = f"{tmpdir}/{{name}}_{{eval_id}}.json"
|
||||
result = MagicMock()
|
||||
result.to_dict.return_value = {"score": 8.0}
|
||||
|
||||
success = save_result_to_file(
|
||||
result, path,
|
||||
eval_id="abc123",
|
||||
name="test_eval"
|
||||
)
|
||||
|
||||
assert success is True
|
||||
expected_path = f"{tmpdir}/test_eval_abc123.json"
|
||||
assert Path(expected_path).exists()
|
||||
|
||||
def test_save_creates_directories(self):
|
||||
"""Test that save creates parent directories."""
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
path = f"{tmpdir}/nested/dir/result.json"
|
||||
result = MagicMock()
|
||||
result.to_dict.return_value = {"score": 8.0}
|
||||
|
||||
success = save_result_to_file(result, path)
|
||||
|
||||
assert success is True
|
||||
assert Path(path).exists()
|
||||
|
||||
def test_save_failure_returns_false(self):
|
||||
"""Test that save failure returns False."""
|
||||
result = MagicMock()
|
||||
result.to_dict.side_effect = Exception("Test error")
|
||||
|
||||
success = save_result_to_file(result, "/invalid/path/result.json")
|
||||
|
||||
assert success is False
|
||||
|
||||
|
||||
class TestLoadResultFromFile:
|
||||
"""Tests for load_result_from_file function."""
|
||||
|
||||
def test_load_existing_file(self):
|
||||
"""Test loading existing result file."""
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
path = f"{tmpdir}/result.json"
|
||||
data = {"score": 8.0, "passed": True}
|
||||
with open(path, 'w') as f:
|
||||
json.dump(data, f)
|
||||
|
||||
loaded = load_result_from_file(path)
|
||||
|
||||
assert loaded == data
|
||||
|
||||
def test_load_nonexistent_file(self):
|
||||
"""Test loading nonexistent file returns None."""
|
||||
result = load_result_from_file("/nonexistent/path/result.json")
|
||||
assert result is None
|
||||
|
||||
def test_load_invalid_json(self):
|
||||
"""Test loading invalid JSON returns None."""
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
path = f"{tmpdir}/invalid.json"
|
||||
with open(path, 'w') as f:
|
||||
f.write("not valid json")
|
||||
|
||||
result = load_result_from_file(path)
|
||||
assert result is None
|
||||
|
||||
|
||||
class TestFormatScore:
|
||||
"""Tests for format_score function."""
|
||||
|
||||
def test_format_default_max(self):
|
||||
"""Test formatting with default max score."""
|
||||
assert format_score(8.5) == "8.5/10"
|
||||
|
||||
def test_format_custom_max(self):
|
||||
"""Test formatting with custom max score."""
|
||||
assert format_score(85.0, 100.0) == "85.0/100"
|
||||
|
||||
def test_format_zero(self):
|
||||
"""Test formatting zero score."""
|
||||
assert format_score(0.0) == "0.0/10"
|
||||
|
||||
|
||||
class TestFormatPercentage:
|
||||
"""Tests for format_percentage function."""
|
||||
|
||||
def test_format_full(self):
|
||||
"""Test formatting 100%."""
|
||||
assert format_percentage(1.0) == "100.0%"
|
||||
|
||||
def test_format_half(self):
|
||||
"""Test formatting 50%."""
|
||||
assert format_percentage(0.5) == "50.0%"
|
||||
|
||||
def test_format_zero(self):
|
||||
"""Test formatting 0%."""
|
||||
assert format_percentage(0.0) == "0.0%"
|
||||
|
||||
def test_format_decimal(self):
|
||||
"""Test formatting decimal percentage."""
|
||||
assert format_percentage(0.333) == "33.3%"
|
||||
|
||||
|
||||
class TestFormatDuration:
|
||||
"""Tests for format_duration function."""
|
||||
|
||||
def test_format_microseconds(self):
|
||||
"""Test formatting microseconds."""
|
||||
result = format_duration(0.0001)
|
||||
assert "μs" in result
|
||||
|
||||
def test_format_milliseconds(self):
|
||||
"""Test formatting milliseconds."""
|
||||
result = format_duration(0.5)
|
||||
assert "ms" in result
|
||||
|
||||
def test_format_seconds(self):
|
||||
"""Test formatting seconds."""
|
||||
result = format_duration(5.5)
|
||||
assert "s" in result
|
||||
assert "m" not in result
|
||||
|
||||
def test_format_minutes(self):
|
||||
"""Test formatting minutes."""
|
||||
result = format_duration(90.0)
|
||||
assert "m" in result
|
||||
|
||||
|
||||
class TestFormatMemory:
|
||||
"""Tests for format_memory function."""
|
||||
|
||||
def test_format_kilobytes(self):
|
||||
"""Test formatting kilobytes."""
|
||||
result = format_memory(0.5)
|
||||
assert "KB" in result
|
||||
|
||||
def test_format_megabytes(self):
|
||||
"""Test formatting megabytes."""
|
||||
result = format_memory(100.0)
|
||||
assert "MB" in result
|
||||
|
||||
def test_format_gigabytes(self):
|
||||
"""Test formatting gigabytes."""
|
||||
result = format_memory(2048.0)
|
||||
assert "GB" in result
|
||||
|
||||
|
||||
class TestCalculatePassRate:
|
||||
"""Tests for calculate_pass_rate function."""
|
||||
|
||||
def test_all_passed(self):
|
||||
"""Test 100% pass rate."""
|
||||
assert calculate_pass_rate(10, 10) == 1.0
|
||||
|
||||
def test_none_passed(self):
|
||||
"""Test 0% pass rate."""
|
||||
assert calculate_pass_rate(0, 10) == 0.0
|
||||
|
||||
def test_half_passed(self):
|
||||
"""Test 50% pass rate."""
|
||||
assert calculate_pass_rate(5, 10) == 0.5
|
||||
|
||||
def test_zero_total(self):
|
||||
"""Test with zero total."""
|
||||
assert calculate_pass_rate(0, 0) == 0.0
|
||||
@@ -0,0 +1 @@
|
||||
"""Unit tests for Agent Skills module."""
|
||||
@@ -0,0 +1,180 @@
|
||||
"""Tests for Agent Skills discovery."""
|
||||
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
|
||||
|
||||
class TestGetDefaultSkillDirs:
|
||||
"""Tests for get_default_skill_dirs function."""
|
||||
|
||||
def test_returns_list(self):
|
||||
"""Test that function returns a list."""
|
||||
from praisonaiagents.skills.discovery import get_default_skill_dirs
|
||||
|
||||
result = get_default_skill_dirs()
|
||||
|
||||
assert isinstance(result, list)
|
||||
|
||||
def test_returns_path_objects(self):
|
||||
"""Test that returned items are Path objects."""
|
||||
from praisonaiagents.skills.discovery import get_default_skill_dirs
|
||||
|
||||
result = get_default_skill_dirs()
|
||||
|
||||
for item in result:
|
||||
assert isinstance(item, Path)
|
||||
|
||||
|
||||
class TestDiscoverSkills:
|
||||
"""Tests for discover_skills function."""
|
||||
|
||||
def test_discover_empty_directory(self):
|
||||
"""Test discovering skills in empty directory."""
|
||||
from praisonaiagents.skills.discovery import discover_skills
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
result = discover_skills([tmpdir], include_defaults=False)
|
||||
|
||||
assert result == []
|
||||
|
||||
def test_discover_single_skill(self):
|
||||
"""Test discovering a single valid skill."""
|
||||
from praisonaiagents.skills.discovery import discover_skills
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
# Create a skill directory
|
||||
skill_dir = Path(tmpdir) / "test-skill"
|
||||
skill_dir.mkdir()
|
||||
(skill_dir / "SKILL.md").write_text("""---
|
||||
name: test-skill
|
||||
description: A test skill for discovery
|
||||
---
|
||||
|
||||
# Test Skill
|
||||
""")
|
||||
|
||||
result = discover_skills([tmpdir], include_defaults=False)
|
||||
|
||||
assert len(result) == 1
|
||||
assert result[0].name == "test-skill"
|
||||
assert result[0].description == "A test skill for discovery"
|
||||
|
||||
def test_discover_multiple_skills(self):
|
||||
"""Test discovering multiple skills."""
|
||||
from praisonaiagents.skills.discovery import discover_skills
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
# Create first skill
|
||||
skill1_dir = Path(tmpdir) / "skill-one"
|
||||
skill1_dir.mkdir()
|
||||
(skill1_dir / "SKILL.md").write_text("""---
|
||||
name: skill-one
|
||||
description: First skill
|
||||
---
|
||||
""")
|
||||
|
||||
# Create second skill
|
||||
skill2_dir = Path(tmpdir) / "skill-two"
|
||||
skill2_dir.mkdir()
|
||||
(skill2_dir / "SKILL.md").write_text("""---
|
||||
name: skill-two
|
||||
description: Second skill
|
||||
---
|
||||
""")
|
||||
|
||||
result = discover_skills([tmpdir], include_defaults=False)
|
||||
|
||||
assert len(result) == 2
|
||||
names = {s.name for s in result}
|
||||
assert "skill-one" in names
|
||||
assert "skill-two" in names
|
||||
|
||||
def test_discover_skips_invalid_skills(self):
|
||||
"""Test that invalid skills are skipped."""
|
||||
from praisonaiagents.skills.discovery import discover_skills
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
# Create valid skill
|
||||
valid_dir = Path(tmpdir) / "valid-skill"
|
||||
valid_dir.mkdir()
|
||||
(valid_dir / "SKILL.md").write_text("""---
|
||||
name: valid-skill
|
||||
description: A valid skill
|
||||
---
|
||||
""")
|
||||
|
||||
# Create invalid skill (no SKILL.md)
|
||||
invalid_dir = Path(tmpdir) / "invalid-skill"
|
||||
invalid_dir.mkdir()
|
||||
|
||||
result = discover_skills([tmpdir], include_defaults=False)
|
||||
|
||||
assert len(result) == 1
|
||||
assert result[0].name == "valid-skill"
|
||||
|
||||
def test_discover_from_multiple_directories(self):
|
||||
"""Test discovering from multiple directories."""
|
||||
from praisonaiagents.skills.discovery import discover_skills
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir1:
|
||||
with tempfile.TemporaryDirectory() as tmpdir2:
|
||||
# Create skill in first directory
|
||||
skill1_dir = Path(tmpdir1) / "skill-a"
|
||||
skill1_dir.mkdir()
|
||||
(skill1_dir / "SKILL.md").write_text("""---
|
||||
name: skill-a
|
||||
description: Skill A
|
||||
---
|
||||
""")
|
||||
|
||||
# Create skill in second directory
|
||||
skill2_dir = Path(tmpdir2) / "skill-b"
|
||||
skill2_dir.mkdir()
|
||||
(skill2_dir / "SKILL.md").write_text("""---
|
||||
name: skill-b
|
||||
description: Skill B
|
||||
---
|
||||
""")
|
||||
|
||||
result = discover_skills([tmpdir1, tmpdir2], include_defaults=False)
|
||||
|
||||
assert len(result) == 2
|
||||
|
||||
|
||||
class TestDiscoverSkill:
|
||||
"""Tests for discover_skill function (single skill)."""
|
||||
|
||||
def test_discover_valid_skill(self):
|
||||
"""Test discovering a single valid skill."""
|
||||
from praisonaiagents.skills.discovery import discover_skill
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
skill_dir = Path(tmpdir) / "my-skill"
|
||||
skill_dir.mkdir()
|
||||
(skill_dir / "SKILL.md").write_text("""---
|
||||
name: my-skill
|
||||
description: My skill
|
||||
---
|
||||
""")
|
||||
|
||||
result = discover_skill(str(skill_dir))
|
||||
|
||||
assert result is not None
|
||||
assert result.name == "my-skill"
|
||||
|
||||
def test_discover_nonexistent_path(self):
|
||||
"""Test discovering from nonexistent path."""
|
||||
from praisonaiagents.skills.discovery import discover_skill
|
||||
|
||||
result = discover_skill("/nonexistent/path")
|
||||
|
||||
assert result is None
|
||||
|
||||
def test_discover_directory_without_skill_md(self):
|
||||
"""Test discovering from directory without SKILL.md."""
|
||||
from praisonaiagents.skills.discovery import discover_skill
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
result = discover_skill(tmpdir)
|
||||
|
||||
assert result is None
|
||||
@@ -0,0 +1,243 @@
|
||||
"""Tests for Agent Skills loader."""
|
||||
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
|
||||
|
||||
class TestSkillLoader:
|
||||
"""Tests for SkillLoader class."""
|
||||
|
||||
def test_load_metadata_valid_skill(self):
|
||||
"""Test loading metadata from valid skill."""
|
||||
from praisonaiagents.skills.loader import SkillLoader
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
skill_dir = Path(tmpdir) / "test-skill"
|
||||
skill_dir.mkdir()
|
||||
(skill_dir / "SKILL.md").write_text("""---
|
||||
name: test-skill
|
||||
description: A test skill
|
||||
---
|
||||
|
||||
# Instructions
|
||||
""")
|
||||
|
||||
loader = SkillLoader()
|
||||
skill = loader.load_metadata(str(skill_dir))
|
||||
|
||||
assert skill is not None
|
||||
assert skill.properties.name == "test-skill"
|
||||
assert skill.instructions is None # Not activated yet
|
||||
|
||||
def test_load_metadata_nonexistent(self):
|
||||
"""Test loading from nonexistent path."""
|
||||
from praisonaiagents.skills.loader import SkillLoader
|
||||
|
||||
loader = SkillLoader()
|
||||
skill = loader.load_metadata("/nonexistent/path")
|
||||
|
||||
assert skill is None
|
||||
|
||||
def test_activate_skill(self):
|
||||
"""Test activating a skill loads instructions."""
|
||||
from praisonaiagents.skills.loader import SkillLoader
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
skill_dir = Path(tmpdir) / "test-skill"
|
||||
skill_dir.mkdir()
|
||||
(skill_dir / "SKILL.md").write_text("""---
|
||||
name: test-skill
|
||||
description: A test skill
|
||||
---
|
||||
|
||||
# Instructions
|
||||
|
||||
Follow these steps:
|
||||
1. Do this
|
||||
2. Do that
|
||||
""")
|
||||
|
||||
loader = SkillLoader()
|
||||
skill = loader.load_metadata(str(skill_dir))
|
||||
|
||||
assert not skill.is_activated
|
||||
|
||||
result = loader.activate(skill)
|
||||
|
||||
assert result is True
|
||||
assert skill.is_activated
|
||||
assert "Follow these steps" in skill.instructions
|
||||
|
||||
def test_activate_already_activated(self):
|
||||
"""Test activating already activated skill returns True."""
|
||||
from praisonaiagents.skills.loader import SkillLoader
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
skill_dir = Path(tmpdir) / "test-skill"
|
||||
skill_dir.mkdir()
|
||||
(skill_dir / "SKILL.md").write_text("""---
|
||||
name: test-skill
|
||||
description: A test skill
|
||||
---
|
||||
|
||||
Instructions.
|
||||
""")
|
||||
|
||||
loader = SkillLoader()
|
||||
skill = loader.load_metadata(str(skill_dir))
|
||||
loader.activate(skill)
|
||||
|
||||
# Activate again
|
||||
result = loader.activate(skill)
|
||||
|
||||
assert result is True
|
||||
|
||||
def test_load_scripts(self):
|
||||
"""Test loading scripts from skill."""
|
||||
from praisonaiagents.skills.loader import SkillLoader
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
skill_dir = Path(tmpdir) / "test-skill"
|
||||
skill_dir.mkdir()
|
||||
(skill_dir / "SKILL.md").write_text("""---
|
||||
name: test-skill
|
||||
description: A test skill
|
||||
---
|
||||
""")
|
||||
|
||||
# Create scripts directory
|
||||
scripts_dir = skill_dir / "scripts"
|
||||
scripts_dir.mkdir()
|
||||
(scripts_dir / "helper.py").write_text("print('hello')")
|
||||
(scripts_dir / "utils.sh").write_text("echo 'hello'")
|
||||
|
||||
loader = SkillLoader()
|
||||
skill = loader.load_metadata(str(skill_dir))
|
||||
scripts = loader.load_scripts(skill)
|
||||
|
||||
assert "helper.py" in scripts
|
||||
assert "utils.sh" in scripts
|
||||
assert "print('hello')" in scripts["helper.py"]
|
||||
|
||||
def test_load_references(self):
|
||||
"""Test loading references from skill."""
|
||||
from praisonaiagents.skills.loader import SkillLoader
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
skill_dir = Path(tmpdir) / "test-skill"
|
||||
skill_dir.mkdir()
|
||||
(skill_dir / "SKILL.md").write_text("""---
|
||||
name: test-skill
|
||||
description: A test skill
|
||||
---
|
||||
""")
|
||||
|
||||
# Create references directory
|
||||
refs_dir = skill_dir / "references"
|
||||
refs_dir.mkdir()
|
||||
(refs_dir / "api.md").write_text("# API Reference")
|
||||
|
||||
loader = SkillLoader()
|
||||
skill = loader.load_metadata(str(skill_dir))
|
||||
refs = loader.load_references(skill)
|
||||
|
||||
assert "api.md" in refs
|
||||
assert "# API Reference" in refs["api.md"]
|
||||
|
||||
def test_load_assets(self):
|
||||
"""Test loading assets from skill."""
|
||||
from praisonaiagents.skills.loader import SkillLoader
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
skill_dir = Path(tmpdir) / "test-skill"
|
||||
skill_dir.mkdir()
|
||||
(skill_dir / "SKILL.md").write_text("""---
|
||||
name: test-skill
|
||||
description: A test skill
|
||||
---
|
||||
""")
|
||||
|
||||
# Create assets directory
|
||||
assets_dir = skill_dir / "assets"
|
||||
assets_dir.mkdir()
|
||||
(assets_dir / "template.json").write_text("{}")
|
||||
|
||||
loader = SkillLoader()
|
||||
skill = loader.load_metadata(str(skill_dir))
|
||||
assets = loader.load_assets(skill)
|
||||
|
||||
assert "template.json" in assets
|
||||
# Path may be resolved differently on macOS (/var vs /private/var)
|
||||
assert assets["template.json"].endswith("test-skill/assets/template.json")
|
||||
|
||||
def test_class_method_load(self):
|
||||
"""Test convenience class method load."""
|
||||
from praisonaiagents.skills.loader import SkillLoader
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
skill_dir = Path(tmpdir) / "test-skill"
|
||||
skill_dir.mkdir()
|
||||
(skill_dir / "SKILL.md").write_text("""---
|
||||
name: test-skill
|
||||
description: A test skill
|
||||
---
|
||||
|
||||
Instructions here.
|
||||
""")
|
||||
|
||||
skill = SkillLoader.load(str(skill_dir), activate=True)
|
||||
|
||||
assert skill is not None
|
||||
assert skill.is_activated
|
||||
assert "Instructions here" in skill.instructions
|
||||
|
||||
|
||||
class TestLoadedSkill:
|
||||
"""Tests for LoadedSkill dataclass."""
|
||||
|
||||
def test_metadata_property(self):
|
||||
"""Test metadata property returns SkillMetadata."""
|
||||
from praisonaiagents.skills.loader import SkillLoader
|
||||
from praisonaiagents.skills.models import SkillMetadata
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
skill_dir = Path(tmpdir) / "test-skill"
|
||||
skill_dir.mkdir()
|
||||
(skill_dir / "SKILL.md").write_text("""---
|
||||
name: test-skill
|
||||
description: A test skill
|
||||
---
|
||||
""")
|
||||
|
||||
loader = SkillLoader()
|
||||
skill = loader.load_metadata(str(skill_dir))
|
||||
|
||||
meta = skill.metadata
|
||||
|
||||
assert isinstance(meta, SkillMetadata)
|
||||
assert meta.name == "test-skill"
|
||||
assert meta.description == "A test skill"
|
||||
|
||||
def test_is_activated_property(self):
|
||||
"""Test is_activated property."""
|
||||
from praisonaiagents.skills.loader import SkillLoader
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
skill_dir = Path(tmpdir) / "test-skill"
|
||||
skill_dir.mkdir()
|
||||
(skill_dir / "SKILL.md").write_text("""---
|
||||
name: test-skill
|
||||
description: A test skill
|
||||
---
|
||||
|
||||
Body.
|
||||
""")
|
||||
|
||||
loader = SkillLoader()
|
||||
skill = loader.load_metadata(str(skill_dir))
|
||||
|
||||
assert skill.is_activated is False
|
||||
|
||||
loader.activate(skill)
|
||||
|
||||
assert skill.is_activated is True
|
||||
@@ -0,0 +1,254 @@
|
||||
"""Tests for Agent Skills manager."""
|
||||
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
|
||||
|
||||
class TestSkillManager:
|
||||
"""Tests for SkillManager class."""
|
||||
|
||||
def test_init_empty(self):
|
||||
"""Test initializing empty manager."""
|
||||
from praisonaiagents.skills.manager import SkillManager
|
||||
|
||||
manager = SkillManager()
|
||||
|
||||
assert len(manager) == 0
|
||||
assert manager.skill_names == []
|
||||
|
||||
def test_discover_skills(self):
|
||||
"""Test discovering skills from directory."""
|
||||
from praisonaiagents.skills.manager import SkillManager
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
# Create a skill
|
||||
skill_dir = Path(tmpdir) / "test-skill"
|
||||
skill_dir.mkdir()
|
||||
(skill_dir / "SKILL.md").write_text("""---
|
||||
name: test-skill
|
||||
description: A test skill
|
||||
---
|
||||
""")
|
||||
|
||||
manager = SkillManager()
|
||||
count = manager.discover([tmpdir], include_defaults=False)
|
||||
|
||||
assert count == 1
|
||||
assert len(manager) == 1
|
||||
assert "test-skill" in manager
|
||||
|
||||
def test_add_skill(self):
|
||||
"""Test adding a single skill."""
|
||||
from praisonaiagents.skills.manager import SkillManager
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
skill_dir = Path(tmpdir) / "my-skill"
|
||||
skill_dir.mkdir()
|
||||
(skill_dir / "SKILL.md").write_text("""---
|
||||
name: my-skill
|
||||
description: My skill
|
||||
---
|
||||
""")
|
||||
|
||||
manager = SkillManager()
|
||||
skill = manager.add_skill(str(skill_dir))
|
||||
|
||||
assert skill is not None
|
||||
assert skill.properties.name == "my-skill"
|
||||
assert "my-skill" in manager
|
||||
|
||||
def test_get_skill(self):
|
||||
"""Test getting a skill by name."""
|
||||
from praisonaiagents.skills.manager import SkillManager
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
skill_dir = Path(tmpdir) / "test-skill"
|
||||
skill_dir.mkdir()
|
||||
(skill_dir / "SKILL.md").write_text("""---
|
||||
name: test-skill
|
||||
description: A test skill
|
||||
---
|
||||
""")
|
||||
|
||||
manager = SkillManager()
|
||||
manager.add_skill(str(skill_dir))
|
||||
|
||||
skill = manager.get_skill("test-skill")
|
||||
|
||||
assert skill is not None
|
||||
assert skill.properties.name == "test-skill"
|
||||
|
||||
def test_get_skill_not_found(self):
|
||||
"""Test getting nonexistent skill."""
|
||||
from praisonaiagents.skills.manager import SkillManager
|
||||
|
||||
manager = SkillManager()
|
||||
skill = manager.get_skill("nonexistent")
|
||||
|
||||
assert skill is None
|
||||
|
||||
def test_activate_by_name(self):
|
||||
"""Test activating skill by name."""
|
||||
from praisonaiagents.skills.manager import SkillManager
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
skill_dir = Path(tmpdir) / "test-skill"
|
||||
skill_dir.mkdir()
|
||||
(skill_dir / "SKILL.md").write_text("""---
|
||||
name: test-skill
|
||||
description: A test skill
|
||||
---
|
||||
|
||||
# Instructions
|
||||
""")
|
||||
|
||||
manager = SkillManager()
|
||||
manager.add_skill(str(skill_dir))
|
||||
|
||||
result = manager.activate_by_name("test-skill")
|
||||
|
||||
assert result is True
|
||||
skill = manager.get_skill("test-skill")
|
||||
assert skill.is_activated
|
||||
|
||||
def test_get_available_skills(self):
|
||||
"""Test getting available skills metadata."""
|
||||
from praisonaiagents.skills.manager import SkillManager
|
||||
from praisonaiagents.skills.models import SkillMetadata
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
skill_dir = Path(tmpdir) / "test-skill"
|
||||
skill_dir.mkdir()
|
||||
(skill_dir / "SKILL.md").write_text("""---
|
||||
name: test-skill
|
||||
description: A test skill
|
||||
---
|
||||
""")
|
||||
|
||||
manager = SkillManager()
|
||||
manager.add_skill(str(skill_dir))
|
||||
|
||||
skills = manager.get_available_skills()
|
||||
|
||||
assert len(skills) == 1
|
||||
assert isinstance(skills[0], SkillMetadata)
|
||||
assert skills[0].name == "test-skill"
|
||||
|
||||
def test_to_prompt(self):
|
||||
"""Test generating prompt XML."""
|
||||
from praisonaiagents.skills.manager import SkillManager
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
skill_dir = Path(tmpdir) / "test-skill"
|
||||
skill_dir.mkdir()
|
||||
(skill_dir / "SKILL.md").write_text("""---
|
||||
name: test-skill
|
||||
description: A test skill
|
||||
---
|
||||
""")
|
||||
|
||||
manager = SkillManager()
|
||||
manager.add_skill(str(skill_dir))
|
||||
|
||||
prompt = manager.to_prompt()
|
||||
|
||||
assert "<available_skills>" in prompt
|
||||
assert "test-skill" in prompt
|
||||
assert "A test skill" in prompt
|
||||
|
||||
def test_get_instructions(self):
|
||||
"""Test getting instructions for a skill."""
|
||||
from praisonaiagents.skills.manager import SkillManager
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
skill_dir = Path(tmpdir) / "test-skill"
|
||||
skill_dir.mkdir()
|
||||
(skill_dir / "SKILL.md").write_text("""---
|
||||
name: test-skill
|
||||
description: A test skill
|
||||
---
|
||||
|
||||
# Instructions
|
||||
|
||||
Do this and that.
|
||||
""")
|
||||
|
||||
manager = SkillManager()
|
||||
manager.add_skill(str(skill_dir))
|
||||
|
||||
instructions = manager.get_instructions("test-skill")
|
||||
|
||||
assert instructions is not None
|
||||
assert "Do this and that" in instructions
|
||||
|
||||
def test_load_resources(self):
|
||||
"""Test loading resources for a skill."""
|
||||
from praisonaiagents.skills.manager import SkillManager
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
skill_dir = Path(tmpdir) / "test-skill"
|
||||
skill_dir.mkdir()
|
||||
(skill_dir / "SKILL.md").write_text("""---
|
||||
name: test-skill
|
||||
description: A test skill
|
||||
---
|
||||
""")
|
||||
|
||||
# Create scripts directory
|
||||
scripts_dir = skill_dir / "scripts"
|
||||
scripts_dir.mkdir()
|
||||
(scripts_dir / "helper.py").write_text("print('hello')")
|
||||
|
||||
manager = SkillManager()
|
||||
manager.add_skill(str(skill_dir))
|
||||
|
||||
result = manager.load_resources("test-skill")
|
||||
|
||||
assert result is True
|
||||
skill = manager.get_skill("test-skill")
|
||||
assert skill.resources_loaded
|
||||
|
||||
def test_clear(self):
|
||||
"""Test clearing all skills."""
|
||||
from praisonaiagents.skills.manager import SkillManager
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
skill_dir = Path(tmpdir) / "test-skill"
|
||||
skill_dir.mkdir()
|
||||
(skill_dir / "SKILL.md").write_text("""---
|
||||
name: test-skill
|
||||
description: A test skill
|
||||
---
|
||||
""")
|
||||
|
||||
manager = SkillManager()
|
||||
manager.add_skill(str(skill_dir))
|
||||
assert len(manager) == 1
|
||||
|
||||
manager.clear()
|
||||
|
||||
assert len(manager) == 0
|
||||
|
||||
def test_iteration(self):
|
||||
"""Test iterating over skills."""
|
||||
from praisonaiagents.skills.manager import SkillManager
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
# Create two skills
|
||||
for name in ["skill-a", "skill-b"]:
|
||||
skill_dir = Path(tmpdir) / name
|
||||
skill_dir.mkdir()
|
||||
(skill_dir / "SKILL.md").write_text(f"""---
|
||||
name: {name}
|
||||
description: Skill {name}
|
||||
---
|
||||
""")
|
||||
|
||||
manager = SkillManager()
|
||||
manager.discover([tmpdir], include_defaults=False)
|
||||
|
||||
names = [s.properties.name for s in manager]
|
||||
|
||||
assert len(names) == 2
|
||||
assert "skill-a" in names
|
||||
assert "skill-b" in names
|
||||
@@ -0,0 +1,184 @@
|
||||
"""Tests for Agent Skills models."""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
class TestSkillProperties:
|
||||
"""Tests for SkillProperties dataclass."""
|
||||
|
||||
def test_skill_properties_required_fields(self):
|
||||
"""Test that SkillProperties requires name and description."""
|
||||
from praisonaiagents.skills.models import SkillProperties
|
||||
|
||||
skill = SkillProperties(
|
||||
name="test-skill",
|
||||
description="A test skill for unit testing"
|
||||
)
|
||||
|
||||
assert skill.name == "test-skill"
|
||||
assert skill.description == "A test skill for unit testing"
|
||||
|
||||
def test_skill_properties_optional_fields_default_none(self):
|
||||
"""Test that optional fields default to None."""
|
||||
from praisonaiagents.skills.models import SkillProperties
|
||||
|
||||
skill = SkillProperties(
|
||||
name="test-skill",
|
||||
description="A test skill"
|
||||
)
|
||||
|
||||
assert skill.license is None
|
||||
assert skill.compatibility is None
|
||||
assert skill.allowed_tools is None
|
||||
assert skill.path is None
|
||||
|
||||
def test_skill_properties_metadata_defaults_empty_dict(self):
|
||||
"""Test that metadata defaults to empty dict."""
|
||||
from praisonaiagents.skills.models import SkillProperties
|
||||
|
||||
skill = SkillProperties(
|
||||
name="test-skill",
|
||||
description="A test skill"
|
||||
)
|
||||
|
||||
assert skill.metadata == {}
|
||||
|
||||
def test_skill_properties_all_fields(self):
|
||||
"""Test SkillProperties with all fields populated."""
|
||||
from praisonaiagents.skills.models import SkillProperties
|
||||
|
||||
skill = SkillProperties(
|
||||
name="pdf-processing",
|
||||
description="Extract text and tables from PDF files",
|
||||
license="Apache-2.0",
|
||||
compatibility="Requires pypdf package",
|
||||
allowed_tools="Read Grep Bash",
|
||||
metadata={"author": "test-org", "version": "1.0"},
|
||||
path=Path("/path/to/skill")
|
||||
)
|
||||
|
||||
assert skill.name == "pdf-processing"
|
||||
assert skill.description == "Extract text and tables from PDF files"
|
||||
assert skill.license == "Apache-2.0"
|
||||
assert skill.compatibility == "Requires pypdf package"
|
||||
assert skill.allowed_tools == "Read Grep Bash"
|
||||
assert skill.metadata == {"author": "test-org", "version": "1.0"}
|
||||
assert skill.path == Path("/path/to/skill")
|
||||
|
||||
def test_skill_properties_to_dict(self):
|
||||
"""Test to_dict method excludes None values."""
|
||||
from praisonaiagents.skills.models import SkillProperties
|
||||
|
||||
skill = SkillProperties(
|
||||
name="test-skill",
|
||||
description="A test skill",
|
||||
license="MIT"
|
||||
)
|
||||
|
||||
result = skill.to_dict()
|
||||
|
||||
assert result["name"] == "test-skill"
|
||||
assert result["description"] == "A test skill"
|
||||
assert result["license"] == "MIT"
|
||||
assert "compatibility" not in result
|
||||
assert "allowed-tools" not in result
|
||||
assert "metadata" not in result # Empty metadata excluded
|
||||
|
||||
def test_skill_properties_to_dict_with_metadata(self):
|
||||
"""Test to_dict includes non-empty metadata."""
|
||||
from praisonaiagents.skills.models import SkillProperties
|
||||
|
||||
skill = SkillProperties(
|
||||
name="test-skill",
|
||||
description="A test skill",
|
||||
metadata={"version": "1.0"}
|
||||
)
|
||||
|
||||
result = skill.to_dict()
|
||||
|
||||
assert result["metadata"] == {"version": "1.0"}
|
||||
|
||||
def test_skill_properties_to_dict_allowed_tools_key(self):
|
||||
"""Test to_dict uses 'allowed-tools' key (hyphenated)."""
|
||||
from praisonaiagents.skills.models import SkillProperties
|
||||
|
||||
skill = SkillProperties(
|
||||
name="test-skill",
|
||||
description="A test skill",
|
||||
allowed_tools="Read Write"
|
||||
)
|
||||
|
||||
result = skill.to_dict()
|
||||
|
||||
assert "allowed-tools" in result
|
||||
assert result["allowed-tools"] == "Read Write"
|
||||
|
||||
|
||||
class TestSkillMetadata:
|
||||
"""Tests for SkillMetadata dataclass (lightweight for system prompt)."""
|
||||
|
||||
def test_skill_metadata_required_fields(self):
|
||||
"""Test SkillMetadata with required fields."""
|
||||
from praisonaiagents.skills.models import SkillMetadata
|
||||
|
||||
meta = SkillMetadata(
|
||||
name="test-skill",
|
||||
description="A test skill",
|
||||
location="/path/to/skill/SKILL.md"
|
||||
)
|
||||
|
||||
assert meta.name == "test-skill"
|
||||
assert meta.description == "A test skill"
|
||||
assert meta.location == "/path/to/skill/SKILL.md"
|
||||
|
||||
def test_skill_metadata_from_properties(self):
|
||||
"""Test creating SkillMetadata from SkillProperties."""
|
||||
from praisonaiagents.skills.models import SkillProperties, SkillMetadata
|
||||
|
||||
props = SkillProperties(
|
||||
name="pdf-skill",
|
||||
description="Process PDF files",
|
||||
path=Path("/skills/pdf-skill")
|
||||
)
|
||||
|
||||
meta = SkillMetadata.from_properties(props)
|
||||
|
||||
assert meta.name == "pdf-skill"
|
||||
assert meta.description == "Process PDF files"
|
||||
assert "/skills/pdf-skill" in meta.location
|
||||
|
||||
|
||||
class TestParseError:
|
||||
"""Tests for ParseError exception."""
|
||||
|
||||
def test_parse_error_message(self):
|
||||
"""Test ParseError stores message correctly."""
|
||||
from praisonaiagents.skills.models import ParseError
|
||||
|
||||
error = ParseError("Invalid YAML frontmatter")
|
||||
|
||||
assert str(error) == "Invalid YAML frontmatter"
|
||||
|
||||
def test_parse_error_is_exception(self):
|
||||
"""Test ParseError is an Exception subclass."""
|
||||
from praisonaiagents.skills.models import ParseError
|
||||
|
||||
assert issubclass(ParseError, Exception)
|
||||
|
||||
|
||||
class TestValidationError:
|
||||
"""Tests for ValidationError exception."""
|
||||
|
||||
def test_validation_error_message(self):
|
||||
"""Test ValidationError stores message correctly."""
|
||||
from praisonaiagents.skills.models import ValidationError
|
||||
|
||||
error = ValidationError("Missing required field: name")
|
||||
|
||||
assert str(error) == "Missing required field: name"
|
||||
|
||||
def test_validation_error_is_exception(self):
|
||||
"""Test ValidationError is an Exception subclass."""
|
||||
from praisonaiagents.skills.models import ValidationError
|
||||
|
||||
assert issubclass(ValidationError, Exception)
|
||||
@@ -0,0 +1,288 @@
|
||||
"""Tests for Agent Skills parser."""
|
||||
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
|
||||
|
||||
class TestParseFrontmatter:
|
||||
"""Tests for parse_frontmatter function."""
|
||||
|
||||
def test_parse_valid_frontmatter(self):
|
||||
"""Test parsing valid YAML frontmatter."""
|
||||
from praisonaiagents.skills.parser import parse_frontmatter
|
||||
|
||||
content = """---
|
||||
name: test-skill
|
||||
description: A test skill for unit testing
|
||||
---
|
||||
|
||||
# Test Skill
|
||||
|
||||
Instructions go here.
|
||||
"""
|
||||
metadata, body = parse_frontmatter(content)
|
||||
|
||||
assert metadata["name"] == "test-skill"
|
||||
assert metadata["description"] == "A test skill for unit testing"
|
||||
assert "# Test Skill" in body
|
||||
|
||||
def test_parse_frontmatter_with_optional_fields(self):
|
||||
"""Test parsing frontmatter with all optional fields."""
|
||||
from praisonaiagents.skills.parser import parse_frontmatter
|
||||
|
||||
content = """---
|
||||
name: pdf-processing
|
||||
description: Extract text from PDFs
|
||||
license: Apache-2.0
|
||||
compatibility: Requires pypdf
|
||||
allowed-tools: Read Grep
|
||||
metadata:
|
||||
author: test-org
|
||||
version: "1.0"
|
||||
---
|
||||
|
||||
Body content.
|
||||
"""
|
||||
metadata, body = parse_frontmatter(content)
|
||||
|
||||
assert metadata["name"] == "pdf-processing"
|
||||
assert metadata["license"] == "Apache-2.0"
|
||||
assert metadata["compatibility"] == "Requires pypdf"
|
||||
assert metadata["allowed-tools"] == "Read Grep"
|
||||
assert metadata["metadata"]["author"] == "test-org"
|
||||
assert metadata["metadata"]["version"] == "1.0"
|
||||
|
||||
def test_parse_frontmatter_missing_opening_delimiter(self):
|
||||
"""Test error when frontmatter doesn't start with ---."""
|
||||
from praisonaiagents.skills.parser import parse_frontmatter
|
||||
from praisonaiagents.skills.models import ParseError
|
||||
|
||||
content = """name: test-skill
|
||||
description: Missing opening delimiter
|
||||
---
|
||||
|
||||
Body content.
|
||||
"""
|
||||
try:
|
||||
parse_frontmatter(content)
|
||||
assert False, "Should have raised ParseError"
|
||||
except ParseError as e:
|
||||
assert "must start with" in str(e).lower() or "---" in str(e)
|
||||
|
||||
def test_parse_frontmatter_missing_closing_delimiter(self):
|
||||
"""Test error when frontmatter is not closed."""
|
||||
from praisonaiagents.skills.parser import parse_frontmatter
|
||||
from praisonaiagents.skills.models import ParseError
|
||||
|
||||
content = """---
|
||||
name: test-skill
|
||||
description: Missing closing delimiter
|
||||
|
||||
Body content without closing delimiter.
|
||||
"""
|
||||
try:
|
||||
parse_frontmatter(content)
|
||||
assert False, "Should have raised ParseError"
|
||||
except ParseError as e:
|
||||
assert "closed" in str(e).lower() or "---" in str(e)
|
||||
|
||||
def test_parse_frontmatter_invalid_yaml(self):
|
||||
"""Test error when YAML is invalid."""
|
||||
from praisonaiagents.skills.parser import parse_frontmatter
|
||||
from praisonaiagents.skills.models import ParseError
|
||||
|
||||
content = """---
|
||||
name: test-skill
|
||||
description: [invalid yaml
|
||||
- missing bracket
|
||||
---
|
||||
|
||||
Body content.
|
||||
"""
|
||||
try:
|
||||
parse_frontmatter(content)
|
||||
assert False, "Should have raised ParseError"
|
||||
except ParseError as e:
|
||||
assert "yaml" in str(e).lower() or "invalid" in str(e).lower()
|
||||
|
||||
def test_parse_frontmatter_empty_body(self):
|
||||
"""Test parsing when body is empty."""
|
||||
from praisonaiagents.skills.parser import parse_frontmatter
|
||||
|
||||
content = """---
|
||||
name: test-skill
|
||||
description: A skill with no body
|
||||
---
|
||||
"""
|
||||
metadata, body = parse_frontmatter(content)
|
||||
|
||||
assert metadata["name"] == "test-skill"
|
||||
assert body.strip() == ""
|
||||
|
||||
|
||||
class TestFindSkillMd:
|
||||
"""Tests for find_skill_md function."""
|
||||
|
||||
def test_find_skill_md_uppercase(self):
|
||||
"""Test finding SKILL.md (uppercase)."""
|
||||
from praisonaiagents.skills.parser import find_skill_md
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
skill_dir = Path(tmpdir)
|
||||
skill_md = skill_dir / "SKILL.md"
|
||||
skill_md.write_text("---\nname: test\ndescription: test\n---\n")
|
||||
|
||||
result = find_skill_md(skill_dir)
|
||||
|
||||
assert result is not None
|
||||
assert result.name == "SKILL.md"
|
||||
|
||||
def test_find_skill_md_lowercase(self):
|
||||
"""Test finding skill.md (lowercase) as fallback."""
|
||||
from praisonaiagents.skills.parser import find_skill_md
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
skill_dir = Path(tmpdir)
|
||||
skill_md = skill_dir / "skill.md"
|
||||
skill_md.write_text("---\nname: test\ndescription: test\n---\n")
|
||||
|
||||
result = find_skill_md(skill_dir)
|
||||
|
||||
assert result is not None
|
||||
# On case-insensitive filesystems (macOS), SKILL.md may match skill.md
|
||||
assert result.name.lower() == "skill.md"
|
||||
|
||||
def test_find_skill_md_prefers_uppercase(self):
|
||||
"""Test that SKILL.md is preferred over skill.md."""
|
||||
from praisonaiagents.skills.parser import find_skill_md
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
skill_dir = Path(tmpdir)
|
||||
(skill_dir / "SKILL.md").write_text("uppercase")
|
||||
(skill_dir / "skill.md").write_text("lowercase")
|
||||
|
||||
result = find_skill_md(skill_dir)
|
||||
|
||||
assert result is not None
|
||||
assert result.name == "SKILL.md"
|
||||
|
||||
def test_find_skill_md_not_found(self):
|
||||
"""Test returning None when no SKILL.md exists."""
|
||||
from praisonaiagents.skills.parser import find_skill_md
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
skill_dir = Path(tmpdir)
|
||||
|
||||
result = find_skill_md(skill_dir)
|
||||
|
||||
assert result is None
|
||||
|
||||
|
||||
class TestReadProperties:
|
||||
"""Tests for read_properties function."""
|
||||
|
||||
def test_read_properties_valid_skill(self):
|
||||
"""Test reading properties from a valid skill directory."""
|
||||
from praisonaiagents.skills.parser import read_properties
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
skill_dir = Path(tmpdir) / "test-skill"
|
||||
skill_dir.mkdir()
|
||||
(skill_dir / "SKILL.md").write_text("""---
|
||||
name: test-skill
|
||||
description: A test skill for unit testing
|
||||
---
|
||||
|
||||
# Test Skill
|
||||
""")
|
||||
|
||||
props = read_properties(skill_dir)
|
||||
|
||||
assert props.name == "test-skill"
|
||||
assert props.description == "A test skill for unit testing"
|
||||
|
||||
def test_read_properties_with_all_fields(self):
|
||||
"""Test reading properties with all optional fields."""
|
||||
from praisonaiagents.skills.parser import read_properties
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
skill_dir = Path(tmpdir) / "pdf-skill"
|
||||
skill_dir.mkdir()
|
||||
(skill_dir / "SKILL.md").write_text("""---
|
||||
name: pdf-skill
|
||||
description: Process PDF files
|
||||
license: MIT
|
||||
compatibility: Requires pypdf
|
||||
allowed-tools: Read Write
|
||||
metadata:
|
||||
author: test
|
||||
version: "2.0"
|
||||
---
|
||||
|
||||
Instructions.
|
||||
""")
|
||||
|
||||
props = read_properties(skill_dir)
|
||||
|
||||
assert props.name == "pdf-skill"
|
||||
assert props.license == "MIT"
|
||||
assert props.compatibility == "Requires pypdf"
|
||||
assert props.allowed_tools == "Read Write"
|
||||
assert props.metadata["author"] == "test"
|
||||
|
||||
def test_read_properties_missing_skill_md(self):
|
||||
"""Test error when SKILL.md is missing."""
|
||||
from praisonaiagents.skills.parser import read_properties
|
||||
from praisonaiagents.skills.models import ParseError
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
skill_dir = Path(tmpdir) / "empty-skill"
|
||||
skill_dir.mkdir()
|
||||
|
||||
try:
|
||||
read_properties(skill_dir)
|
||||
assert False, "Should have raised ParseError"
|
||||
except ParseError as e:
|
||||
assert "not found" in str(e).lower() or "skill.md" in str(e).lower()
|
||||
|
||||
def test_read_properties_missing_name(self):
|
||||
"""Test error when name field is missing."""
|
||||
from praisonaiagents.skills.parser import read_properties
|
||||
from praisonaiagents.skills.models import ValidationError
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
skill_dir = Path(tmpdir) / "no-name"
|
||||
skill_dir.mkdir()
|
||||
(skill_dir / "SKILL.md").write_text("""---
|
||||
description: Missing name field
|
||||
---
|
||||
|
||||
Body.
|
||||
""")
|
||||
|
||||
try:
|
||||
read_properties(skill_dir)
|
||||
assert False, "Should have raised ValidationError"
|
||||
except ValidationError as e:
|
||||
assert "name" in str(e).lower()
|
||||
|
||||
def test_read_properties_missing_description(self):
|
||||
"""Test error when description field is missing."""
|
||||
from praisonaiagents.skills.parser import read_properties
|
||||
from praisonaiagents.skills.models import ValidationError
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
skill_dir = Path(tmpdir) / "no-desc"
|
||||
skill_dir.mkdir()
|
||||
(skill_dir / "SKILL.md").write_text("""---
|
||||
name: no-desc
|
||||
---
|
||||
|
||||
Body.
|
||||
""")
|
||||
|
||||
try:
|
||||
read_properties(skill_dir)
|
||||
assert False, "Should have raised ValidationError"
|
||||
except ValidationError as e:
|
||||
assert "description" in str(e).lower()
|
||||
@@ -0,0 +1,180 @@
|
||||
"""Tests for Agent Skills prompt generation."""
|
||||
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
|
||||
|
||||
class TestToPrompt:
|
||||
"""Tests for to_prompt function (XML generation)."""
|
||||
|
||||
def test_to_prompt_empty_list(self):
|
||||
"""Test generating prompt with no skills."""
|
||||
from praisonaiagents.skills.prompt import to_prompt
|
||||
|
||||
result = to_prompt([])
|
||||
|
||||
assert "<available_skills>" in result
|
||||
assert "</available_skills>" in result
|
||||
|
||||
def test_to_prompt_single_skill(self):
|
||||
"""Test generating prompt with one skill."""
|
||||
from praisonaiagents.skills.prompt import to_prompt
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
skill_dir = Path(tmpdir) / "test-skill"
|
||||
skill_dir.mkdir()
|
||||
(skill_dir / "SKILL.md").write_text("""---
|
||||
name: test-skill
|
||||
description: A test skill for prompt generation
|
||||
---
|
||||
|
||||
# Test Skill
|
||||
""")
|
||||
|
||||
result = to_prompt([skill_dir])
|
||||
|
||||
assert "<available_skills>" in result
|
||||
assert "</available_skills>" in result
|
||||
assert "<skill>" in result
|
||||
assert "</skill>" in result
|
||||
assert "<name>" in result
|
||||
assert "test-skill" in result
|
||||
assert "<description>" in result
|
||||
assert "A test skill for prompt generation" in result
|
||||
assert "<location>" in result
|
||||
|
||||
def test_to_prompt_multiple_skills(self):
|
||||
"""Test generating prompt with multiple skills."""
|
||||
from praisonaiagents.skills.prompt import to_prompt
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
# Create first skill
|
||||
skill1_dir = Path(tmpdir) / "skill-one"
|
||||
skill1_dir.mkdir()
|
||||
(skill1_dir / "SKILL.md").write_text("""---
|
||||
name: skill-one
|
||||
description: First skill
|
||||
---
|
||||
""")
|
||||
|
||||
# Create second skill
|
||||
skill2_dir = Path(tmpdir) / "skill-two"
|
||||
skill2_dir.mkdir()
|
||||
(skill2_dir / "SKILL.md").write_text("""---
|
||||
name: skill-two
|
||||
description: Second skill
|
||||
---
|
||||
""")
|
||||
|
||||
result = to_prompt([skill1_dir, skill2_dir])
|
||||
|
||||
assert result.count("<skill>") == 2
|
||||
assert result.count("</skill>") == 2
|
||||
assert "skill-one" in result
|
||||
assert "skill-two" in result
|
||||
assert "First skill" in result
|
||||
assert "Second skill" in result
|
||||
|
||||
def test_to_prompt_escapes_html(self):
|
||||
"""Test that special characters are HTML escaped."""
|
||||
from praisonaiagents.skills.prompt import to_prompt
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
skill_dir = Path(tmpdir) / "escape-test"
|
||||
skill_dir.mkdir()
|
||||
(skill_dir / "SKILL.md").write_text("""---
|
||||
name: escape-test
|
||||
description: Test <script> & "quotes" handling
|
||||
---
|
||||
""")
|
||||
|
||||
result = to_prompt([skill_dir])
|
||||
|
||||
# Should escape < > & "
|
||||
assert "<script>" not in result or "<script>" in result
|
||||
assert "&" in result or "& " not in result
|
||||
|
||||
def test_to_prompt_includes_location(self):
|
||||
"""Test that location points to SKILL.md file."""
|
||||
from praisonaiagents.skills.prompt import to_prompt
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
skill_dir = Path(tmpdir) / "location-test"
|
||||
skill_dir.mkdir()
|
||||
(skill_dir / "SKILL.md").write_text("""---
|
||||
name: location-test
|
||||
description: Test location field
|
||||
---
|
||||
""")
|
||||
|
||||
result = to_prompt([skill_dir])
|
||||
|
||||
assert "<location>" in result
|
||||
assert "SKILL.md" in result
|
||||
assert "</location>" in result
|
||||
|
||||
|
||||
class TestGenerateSkillsXml:
|
||||
"""Tests for generate_skills_xml helper function."""
|
||||
|
||||
def test_generate_skills_xml_from_metadata_list(self):
|
||||
"""Test generating XML from SkillMetadata list."""
|
||||
from praisonaiagents.skills.prompt import generate_skills_xml
|
||||
from praisonaiagents.skills.models import SkillMetadata
|
||||
|
||||
skills = [
|
||||
SkillMetadata(
|
||||
name="pdf-skill",
|
||||
description="Process PDF files",
|
||||
location="/path/to/pdf-skill/SKILL.md"
|
||||
),
|
||||
SkillMetadata(
|
||||
name="excel-skill",
|
||||
description="Process Excel files",
|
||||
location="/path/to/excel-skill/SKILL.md"
|
||||
)
|
||||
]
|
||||
|
||||
result = generate_skills_xml(skills)
|
||||
|
||||
assert "<available_skills>" in result
|
||||
assert "pdf-skill" in result
|
||||
assert "excel-skill" in result
|
||||
assert "Process PDF files" in result
|
||||
assert "Process Excel files" in result
|
||||
|
||||
def test_generate_skills_xml_empty_list(self):
|
||||
"""Test generating XML with empty list."""
|
||||
from praisonaiagents.skills.prompt import generate_skills_xml
|
||||
|
||||
result = generate_skills_xml([])
|
||||
|
||||
assert "<available_skills>" in result
|
||||
assert "</available_skills>" in result
|
||||
assert "<skill>" not in result
|
||||
|
||||
|
||||
class TestFormatSkillForPrompt:
|
||||
"""Tests for format_skill_for_prompt helper."""
|
||||
|
||||
def test_format_single_skill(self):
|
||||
"""Test formatting a single skill for prompt."""
|
||||
from praisonaiagents.skills.prompt import format_skill_for_prompt
|
||||
from praisonaiagents.skills.models import SkillMetadata
|
||||
|
||||
skill = SkillMetadata(
|
||||
name="test-skill",
|
||||
description="A test skill",
|
||||
location="/path/to/SKILL.md"
|
||||
)
|
||||
|
||||
result = format_skill_for_prompt(skill)
|
||||
|
||||
assert "<skill>" in result
|
||||
assert "</skill>" in result
|
||||
assert "<name>" in result
|
||||
assert "test-skill" in result
|
||||
assert "<description>" in result
|
||||
assert "A test skill" in result
|
||||
assert "<location>" in result
|
||||
assert "/path/to/SKILL.md" in result
|
||||
@@ -0,0 +1,258 @@
|
||||
"""Tests for Agent Skills validator."""
|
||||
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
|
||||
|
||||
class TestValidateName:
|
||||
"""Tests for skill name validation."""
|
||||
|
||||
def test_valid_name_simple(self):
|
||||
"""Test valid simple skill name."""
|
||||
from praisonaiagents.skills.validator import _validate_name
|
||||
|
||||
errors = _validate_name("test-skill", None)
|
||||
|
||||
assert errors == []
|
||||
|
||||
def test_valid_name_with_numbers(self):
|
||||
"""Test valid name with numbers."""
|
||||
from praisonaiagents.skills.validator import _validate_name
|
||||
|
||||
errors = _validate_name("skill-v2", None)
|
||||
|
||||
assert errors == []
|
||||
|
||||
def test_invalid_name_uppercase(self):
|
||||
"""Test that uppercase letters are rejected."""
|
||||
from praisonaiagents.skills.validator import _validate_name
|
||||
|
||||
errors = _validate_name("Test-Skill", None)
|
||||
|
||||
assert len(errors) > 0
|
||||
assert any("lowercase" in e.lower() for e in errors)
|
||||
|
||||
def test_invalid_name_starts_with_hyphen(self):
|
||||
"""Test that names starting with hyphen are rejected."""
|
||||
from praisonaiagents.skills.validator import _validate_name
|
||||
|
||||
errors = _validate_name("-skill", None)
|
||||
|
||||
assert len(errors) > 0
|
||||
assert any("hyphen" in e.lower() for e in errors)
|
||||
|
||||
def test_invalid_name_ends_with_hyphen(self):
|
||||
"""Test that names ending with hyphen are rejected."""
|
||||
from praisonaiagents.skills.validator import _validate_name
|
||||
|
||||
errors = _validate_name("skill-", None)
|
||||
|
||||
assert len(errors) > 0
|
||||
assert any("hyphen" in e.lower() for e in errors)
|
||||
|
||||
def test_invalid_name_consecutive_hyphens(self):
|
||||
"""Test that consecutive hyphens are rejected."""
|
||||
from praisonaiagents.skills.validator import _validate_name
|
||||
|
||||
errors = _validate_name("test--skill", None)
|
||||
|
||||
assert len(errors) > 0
|
||||
assert any("consecutive" in e.lower() for e in errors)
|
||||
|
||||
def test_invalid_name_too_long(self):
|
||||
"""Test that names over 64 characters are rejected."""
|
||||
from praisonaiagents.skills.validator import _validate_name
|
||||
|
||||
long_name = "a" * 65
|
||||
errors = _validate_name(long_name, None)
|
||||
|
||||
assert len(errors) > 0
|
||||
assert any("64" in e or "limit" in e.lower() for e in errors)
|
||||
|
||||
def test_invalid_name_special_characters(self):
|
||||
"""Test that special characters are rejected."""
|
||||
from praisonaiagents.skills.validator import _validate_name
|
||||
|
||||
errors = _validate_name("test_skill", None)
|
||||
|
||||
assert len(errors) > 0
|
||||
assert any("invalid" in e.lower() or "character" in e.lower() for e in errors)
|
||||
|
||||
def test_name_must_match_directory(self):
|
||||
"""Test that name must match parent directory name."""
|
||||
from praisonaiagents.skills.validator import _validate_name
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
skill_dir = Path(tmpdir) / "actual-name"
|
||||
skill_dir.mkdir()
|
||||
|
||||
errors = _validate_name("different-name", skill_dir)
|
||||
|
||||
assert len(errors) > 0
|
||||
assert any("match" in e.lower() for e in errors)
|
||||
|
||||
|
||||
class TestValidateDescription:
|
||||
"""Tests for skill description validation."""
|
||||
|
||||
def test_valid_description(self):
|
||||
"""Test valid description."""
|
||||
from praisonaiagents.skills.validator import _validate_description
|
||||
|
||||
errors = _validate_description("A helpful skill that does things")
|
||||
|
||||
assert errors == []
|
||||
|
||||
def test_invalid_description_empty(self):
|
||||
"""Test that empty description is rejected."""
|
||||
from praisonaiagents.skills.validator import _validate_description
|
||||
|
||||
errors = _validate_description("")
|
||||
|
||||
assert len(errors) > 0
|
||||
assert any("empty" in e.lower() or "non-empty" in e.lower() for e in errors)
|
||||
|
||||
def test_invalid_description_too_long(self):
|
||||
"""Test that descriptions over 1024 characters are rejected."""
|
||||
from praisonaiagents.skills.validator import _validate_description
|
||||
|
||||
long_desc = "a" * 1025
|
||||
errors = _validate_description(long_desc)
|
||||
|
||||
assert len(errors) > 0
|
||||
assert any("1024" in e or "limit" in e.lower() for e in errors)
|
||||
|
||||
|
||||
class TestValidateCompatibility:
|
||||
"""Tests for skill compatibility validation."""
|
||||
|
||||
def test_valid_compatibility(self):
|
||||
"""Test valid compatibility string."""
|
||||
from praisonaiagents.skills.validator import _validate_compatibility
|
||||
|
||||
errors = _validate_compatibility("Requires Python 3.11+")
|
||||
|
||||
assert errors == []
|
||||
|
||||
def test_invalid_compatibility_too_long(self):
|
||||
"""Test that compatibility over 500 characters is rejected."""
|
||||
from praisonaiagents.skills.validator import _validate_compatibility
|
||||
|
||||
long_compat = "a" * 501
|
||||
errors = _validate_compatibility(long_compat)
|
||||
|
||||
assert len(errors) > 0
|
||||
assert any("500" in e or "limit" in e.lower() for e in errors)
|
||||
|
||||
|
||||
class TestValidateMetadata:
|
||||
"""Tests for full metadata validation."""
|
||||
|
||||
def test_valid_metadata(self):
|
||||
"""Test valid complete metadata."""
|
||||
from praisonaiagents.skills.validator import validate_metadata
|
||||
|
||||
metadata = {
|
||||
"name": "test-skill",
|
||||
"description": "A test skill"
|
||||
}
|
||||
|
||||
errors = validate_metadata(metadata)
|
||||
|
||||
assert errors == []
|
||||
|
||||
def test_missing_name(self):
|
||||
"""Test error when name is missing."""
|
||||
from praisonaiagents.skills.validator import validate_metadata
|
||||
|
||||
metadata = {
|
||||
"description": "A test skill"
|
||||
}
|
||||
|
||||
errors = validate_metadata(metadata)
|
||||
|
||||
assert len(errors) > 0
|
||||
assert any("name" in e.lower() for e in errors)
|
||||
|
||||
def test_missing_description(self):
|
||||
"""Test error when description is missing."""
|
||||
from praisonaiagents.skills.validator import validate_metadata
|
||||
|
||||
metadata = {
|
||||
"name": "test-skill"
|
||||
}
|
||||
|
||||
errors = validate_metadata(metadata)
|
||||
|
||||
assert len(errors) > 0
|
||||
assert any("description" in e.lower() for e in errors)
|
||||
|
||||
def test_unexpected_fields_rejected(self):
|
||||
"""Test that unexpected fields are rejected."""
|
||||
from praisonaiagents.skills.validator import validate_metadata
|
||||
|
||||
metadata = {
|
||||
"name": "test-skill",
|
||||
"description": "A test skill",
|
||||
"unknown-field": "value"
|
||||
}
|
||||
|
||||
errors = validate_metadata(metadata)
|
||||
|
||||
assert len(errors) > 0
|
||||
assert any("unexpected" in e.lower() or "unknown-field" in e for e in errors)
|
||||
|
||||
|
||||
class TestValidate:
|
||||
"""Tests for full skill directory validation."""
|
||||
|
||||
def test_validate_valid_skill(self):
|
||||
"""Test validating a valid skill directory."""
|
||||
from praisonaiagents.skills.validator import validate
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
skill_dir = Path(tmpdir) / "test-skill"
|
||||
skill_dir.mkdir()
|
||||
(skill_dir / "SKILL.md").write_text("""---
|
||||
name: test-skill
|
||||
description: A test skill for validation
|
||||
---
|
||||
|
||||
# Test Skill
|
||||
""")
|
||||
|
||||
errors = validate(skill_dir)
|
||||
|
||||
assert errors == []
|
||||
|
||||
def test_validate_missing_skill_md(self):
|
||||
"""Test error when SKILL.md is missing."""
|
||||
from praisonaiagents.skills.validator import validate
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
skill_dir = Path(tmpdir) / "empty-skill"
|
||||
skill_dir.mkdir()
|
||||
|
||||
errors = validate(skill_dir)
|
||||
|
||||
assert len(errors) > 0
|
||||
assert any("skill.md" in e.lower() for e in errors)
|
||||
|
||||
def test_validate_nonexistent_directory(self):
|
||||
"""Test error when directory doesn't exist."""
|
||||
from praisonaiagents.skills.validator import validate
|
||||
|
||||
errors = validate(Path("/nonexistent/path"))
|
||||
|
||||
assert len(errors) > 0
|
||||
assert any("exist" in e.lower() for e in errors)
|
||||
|
||||
def test_validate_not_a_directory(self):
|
||||
"""Test error when path is not a directory."""
|
||||
from praisonaiagents.skills.validator import validate
|
||||
|
||||
with tempfile.NamedTemporaryFile() as tmpfile:
|
||||
errors = validate(Path(tmpfile.name))
|
||||
|
||||
assert len(errors) > 0
|
||||
assert any("directory" in e.lower() for e in errors)
|
||||
Generated
+1
-1
@@ -2422,7 +2422,7 @@ wheels = [
|
||||
|
||||
[[package]]
|
||||
name = "praisonaiagents"
|
||||
version = "0.1.17"
|
||||
version = "0.1.27"
|
||||
source = { editable = "." }
|
||||
dependencies = [
|
||||
{ name = "aiohttp" },
|
||||
|
||||
+400
-217
@@ -6,10 +6,13 @@
|
||||
</picture>
|
||||
</p>
|
||||
|
||||
<!-- mcp-name: io.github.MervinPraison/praisonai -->
|
||||
|
||||
<p align="center">
|
||||
<a href="https://github.com/MervinPraison/PraisonAI"><img src="https://static.pepy.tech/badge/PraisonAI" alt="Total Downloads" /></a>
|
||||
<a href="https://github.com/MervinPraison/PraisonAI"><img src="https://img.shields.io/github/v/release/MervinPraison/PraisonAI" alt="Latest Stable Version" /></a>
|
||||
<a href="https://github.com/MervinPraison/PraisonAI"><img src="https://img.shields.io/badge/License-MIT-yellow.svg" alt="License" /></a>
|
||||
<a href="https://registry.modelcontextprotocol.io/servers/io.github.MervinPraison/praisonai"><img src="https://img.shields.io/badge/MCP-Registry-blue" alt="MCP Registry" /></a>
|
||||
</p>
|
||||
|
||||
<div align="center">
|
||||
@@ -168,12 +171,13 @@ npm install praisonai
|
||||
|
||||
| Feature | Code | Docs |
|
||||
|---------|:----:|:----:|
|
||||
| 🚀 Single Agent | [Example](examples/python/agents/single-agent.py) | [📖](https://docs.praison.ai/agents/single) |
|
||||
| 🤝 Multi Agents | [Example](examples/python/general/mini_agents_example.py) | [📖](https://docs.praison.ai/concepts/agents) |
|
||||
| 🤖 Auto Agents | [Example](examples/python/general/auto_agents_example.py) | [📖](https://docs.praison.ai/features/autoagents) |
|
||||
| 🔄 Self Reflection AI Agents | [Example](examples/python/concepts/self-reflection-details.py) | [📖](https://docs.praison.ai/features/selfreflection) |
|
||||
| 🧠 Reasoning AI Agents | [Example](examples/python/concepts/reasoning-extraction.py) | [📖](https://docs.praison.ai/features/reasoning) |
|
||||
| 👁️ Multi Modal AI Agents | [Example](examples/python/general/multimodal.py) | [📖](https://docs.praison.ai/features/multimodal) |
|
||||
| **🤖 Core Agents** | | |
|
||||
| ↳ Single Agent | [Example](examples/python/agents/single-agent.py) | [📖](https://docs.praison.ai/agents/single) |
|
||||
| ↳ Multi Agents | [Example](examples/python/general/mini_agents_example.py) | [📖](https://docs.praison.ai/concepts/agents) |
|
||||
| ↳ Auto Agents | [Example](examples/python/general/auto_agents_example.py) | [📖](https://docs.praison.ai/features/autoagents) |
|
||||
| ↳ Self Reflection AI Agents | [Example](examples/python/concepts/self-reflection-details.py) | [📖](https://docs.praison.ai/features/selfreflection) |
|
||||
| ↳ Reasoning AI Agents | [Example](examples/python/concepts/reasoning-extraction.py) | [📖](https://docs.praison.ai/features/reasoning) |
|
||||
| ↳ Multi Modal AI Agents | [Example](examples/python/general/multimodal.py) | [📖](https://docs.praison.ai/features/multimodal) |
|
||||
| **🔄 Workflows** | | |
|
||||
| ↳ Simple Workflow | [Example](examples/python/workflows/simple_workflow.py) | [📖](https://docs.praison.ai/features/workflows) |
|
||||
| ↳ Workflow with Agents | [Example](examples/python/workflows/workflow_with_agents.py) | [📖](https://docs.praison.ai/features/workflows) |
|
||||
@@ -185,69 +189,111 @@ npm install praisonai
|
||||
| ↳ Workflow Branching | [Example](examples/python/workflows/workflow_branching.py) | [📖](https://docs.praison.ai/features/workflows) |
|
||||
| ↳ Workflow Early Stop | [Example](examples/python/workflows/workflow_early_stop.py) | [📖](https://docs.praison.ai/features/workflows) |
|
||||
| ↳ Workflow Checkpoints | [Example](examples/python/workflows/workflow_checkpoints.py) | [📖](https://docs.praison.ai/features/workflows) |
|
||||
| 📚 Add Custom Knowledge | [Example](examples/python/concepts/knowledge-agents.py) | [📖](https://docs.praison.ai/features/knowledge) |
|
||||
| 🧠 Memory (Short & Long Term) | [Example](examples/python/general/memory_example.py) | [📖](https://docs.praison.ai/concepts/memory) |
|
||||
| 📄 Chat with PDF Agents | [Example](examples/python/concepts/chat-with-pdf.py) | [📖](https://docs.praison.ai/features/chat-with-pdf) |
|
||||
| 💻 Code Interpreter Agents | [Example](examples/python/agents/code-agent.py) | [📖](https://docs.praison.ai/features/codeagent) |
|
||||
| ✏️ AI Code Editing Tools | [Example](#ai-code-editing) | [📖](https://docs.praison.ai/features/code) |
|
||||
| 📚 RAG Agents | [Example](examples/python/concepts/rag-agents.py) | [📖](https://docs.praison.ai/features/rag) |
|
||||
| 🤔 Async & Parallel Processing | [Example](examples/python/general/async_example.py) | [📖](https://docs.praison.ai/features/async) |
|
||||
| 🔢 Math Agents | [Example](examples/python/agents/math-agent.py) | [📖](https://docs.praison.ai/features/mathagent) |
|
||||
| 🎯 Structured Output Agents | [Example](examples/python/general/structured_agents_example.py) | [📖](https://docs.praison.ai/features/structured) |
|
||||
| 🔗 LangChain Integrated Agents | [Example](examples/python/general/langchain_example.py) | [📖](https://docs.praison.ai/features/langchain) |
|
||||
| 📞 Callback Agents | [Example](examples/python/general/advanced-callback-systems.py) | [📖](https://docs.praison.ai/features/callbacks) |
|
||||
| 🛠️ 100+ Custom Tools | [Example](examples/python/general/tools_example.py) | [📖](https://docs.praison.ai/tools/tools) |
|
||||
| 📄 YAML Configuration | [Example](examples/cookbooks/yaml/secondary_market_research_agents.yaml) | [📖](https://docs.praison.ai/developers/agents-playbook) |
|
||||
| 💯 100+ LLM Support | [Example](examples/python/providers/openai/openai_gpt4_example.py) | [📖](https://docs.praison.ai/models) |
|
||||
| 🔬 Deep Research Agents | [Example](examples/python/agents/research-agent.py) | [📖](https://docs.praison.ai/agents/deep-research) |
|
||||
| 🔄 Query Rewriter Agent | [Example](#5-query-rewriter-agent) | [📖](https://docs.praison.ai/agents/query-rewriter) |
|
||||
| 🌐 Native Web Search | [Example](examples/python/agents/websearch-agent.py) | [📖](https://docs.praison.ai/agents/websearch) |
|
||||
| 📥 Web Fetch (Anthropic) | [Example](#web-search-web-fetch--prompt-caching) | [📖](https://docs.praison.ai/features/model-capabilities) |
|
||||
| 💾 Prompt Caching | [Example](#web-search-web-fetch--prompt-caching) | [📖](https://docs.praison.ai/features/model-capabilities) |
|
||||
| 🧠 Claude Memory Tool | [Example](#claude-memory-tool-cli) | [📖](https://docs.praison.ai/features/claude-memory-tool) |
|
||||
| 💾 File-Based Memory | [Example](examples/python/general/memory_example.py) | [📖](https://docs.praison.ai/concepts/memory) |
|
||||
| 🔍 Built-in Search Tools | [Example](examples/python/agents/websearch-agent.py) | [📖](https://docs.praison.ai/tools/tavily) |
|
||||
| 📋 Planning Mode | [Example](examples/python/agents/planning-agent.py) | [📖](https://docs.praison.ai/features/planning-mode) |
|
||||
| 🔧 Planning Tools | [Example](#3-agent-with-planning-mode) | [📖](https://docs.praison.ai/features/planning-mode) |
|
||||
| 🧠 Planning Reasoning | [Example](#3-agent-with-planning-mode) | [📖](https://docs.praison.ai/features/planning-mode) |
|
||||
| 🔌 MCP Transports | [Example](examples/python/mcp/mcp-transports-overview.py) | [📖](https://docs.praison.ai/mcp/transports) |
|
||||
| 🌐 WebSocket MCP | [Example](examples/python/mcp/websocket-mcp.py) | [📖](https://docs.praison.ai/mcp/sse-transport) |
|
||||
| 🔐 MCP Security | [Example](examples/python/mcp/mcp-security.py) | [📖](https://docs.praison.ai/mcp/transports) |
|
||||
| 🔄 MCP Resumability | [Example](examples/python/mcp/mcp-resumability.py) | [📖](https://docs.praison.ai/mcp/sse-transport) |
|
||||
| ⚡ Fast Context | [Example](examples/context/00_agent_fast_context_basic.py) | [📖](https://docs.praison.ai/features/fast-context) |
|
||||
| 🖼️ Image Generation Agent | [Example](examples/python/image/image-agent.py) | [📖](https://docs.praison.ai/features/image-generation) |
|
||||
| 📷 Image to Text Agent | [Example](examples/python/agents/image-to-text-agent.py) | [📖](https://docs.praison.ai/agents/image-to-text) |
|
||||
| 🎬 Video Agent | [Example](examples/python/agents/video-agent.py) | [📖](https://docs.praison.ai/agents/video) |
|
||||
| 📊 Data Analyst Agent | [Example](examples/python/agents/data-analyst-agent.py) | [📖](https://docs.praison.ai/agents/data-analyst) |
|
||||
| 💰 Finance Agent | [Example](examples/python/agents/finance-agent.py) | [📖](https://docs.praison.ai/agents/finance) |
|
||||
| 🛒 Shopping Agent | [Example](examples/python/agents/shopping-agent.py) | [📖](https://docs.praison.ai/agents/shopping) |
|
||||
| ⭐ Recommendation Agent | [Example](examples/python/agents/recommendation-agent.py) | [📖](https://docs.praison.ai/agents/recommendation) |
|
||||
| 📖 Wikipedia Agent | [Example](examples/python/agents/wikipedia-agent.py) | [📖](https://docs.praison.ai/agents/wikipedia) |
|
||||
| 💻 Programming Agent | [Example](examples/python/agents/programming-agent.py) | [📖](https://docs.praison.ai/agents/programming) |
|
||||
| 📝 Markdown Agent | [Example](examples/python/agents/markdown-agent.py) | [📖](https://docs.praison.ai/agents/markdown) |
|
||||
| 📝 Prompt Expander Agent | [Example](#prompt-expansion) | [📖](https://docs.praison.ai/agents/prompt-expander) |
|
||||
| 🔀 Router Agent | [Example](examples/python/agents/router-agent-cost-optimization.py) | [📖](https://docs.praison.ai/features/routing) |
|
||||
| ⛓️ Prompt Chaining | [Example](examples/python/general/prompt_chaining.py) | [📖](https://docs.praison.ai/features/promptchaining) |
|
||||
| 🔍 Evaluator Optimiser | [Example](examples/python/general/evaluator-optimiser.py) | [📖](https://docs.praison.ai/features/evaluator-optimiser) |
|
||||
| 👷 Orchestrator Workers | [Example](examples/python/general/orchestrator-workers.py) | [📖](https://docs.praison.ai/features/orchestrator-worker) |
|
||||
| ⚡ Parallelisation | [Example](examples/python/general/parallelisation.py) | [📖](https://docs.praison.ai/features/parallelisation) |
|
||||
| 🔁 Repetitive Agents | [Example](examples/python/concepts/repetitive-agents.py) | [📖](https://docs.praison.ai/features/repetitive) |
|
||||
| 🤝 Agent Handoffs | [Example](examples/python/handoff/handoff_basic.py) | [📖](https://docs.praison.ai/features/handoffs) |
|
||||
| 🛡️ Guardrails | [Example](examples/python/guardrails/comprehensive-guardrails-example.py) | [📖](https://docs.praison.ai/features/guardrails) |
|
||||
| 💬 Sessions Management | [Example](examples/python/sessions/comprehensive-session-management.py) | [📖](https://docs.praison.ai/features/sessions) |
|
||||
| ✅ Human Approval | [Example](examples/python/general/human_approval_example.py) | [📖](https://docs.praison.ai/features/approval) |
|
||||
| 🔄 Stateful Agents | [Example](examples/python/stateful/workflow-state-example.py) | [📖](https://docs.praison.ai/features/stateful-agents) |
|
||||
| 🤖 Autonomous Workflow | [Example](examples/python/general/autonomous-agent.py) | [📖](https://docs.praison.ai/features/autonomous-workflow) |
|
||||
| 📜 Rules & Instructions | [Example](#6-rules--instructions) | [📖](https://docs.praison.ai/features/rules) |
|
||||
| 🪝 Hooks | [Example](#9-hooks) | [📖](https://docs.praison.ai/features/hooks) |
|
||||
| 📈 Telemetry | [Example](examples/python/telemetry/production-telemetry-example.py) | [📖](https://docs.praison.ai/features/telemetry) |
|
||||
| 📹 Camera Integration | [Example](examples/python/camera/) | [📖](https://docs.praison.ai/features/camera-integration) |
|
||||
| 📄 Project Docs (.praison/docs/) | [Example](#docs-cli) | [📖](https://docs.praison.ai/cli/docs) |
|
||||
| 🔌 MCP Config Management | [Example](#mcp-config-cli) | [📖](https://docs.praison.ai/cli/mcp) |
|
||||
| 💬 AI Commit Messages | [Example](#ai-commit-cli) | [📖](https://docs.praison.ai/cli/commit) |
|
||||
| @ @Mentions in Prompts | [Example](#mentions-in-prompts) | [📖](https://docs.praison.ai/cli/mentions) |
|
||||
| 💾 Auto-Save Sessions | [Example](#session-management-python) | [📖](https://docs.praison.ai/cli/session) |
|
||||
| 📜 History in Context | [Example](#session-management-python) | [📖](https://docs.praison.ai/cli/session) |
|
||||
| **💻 Code & Development** | | |
|
||||
| ↳ Code Interpreter Agents | [Example](examples/python/agents/code-agent.py) | [📖](https://docs.praison.ai/features/codeagent) |
|
||||
| ↳ AI Code Editing Tools | [Example](examples/python/code/code_editing_example.py) | [📖](https://docs.praison.ai/code/editing) |
|
||||
| ↳ External Agents (All) | [Example](examples/python/code/external_agents_example.py) | [📖](https://docs.praison.ai/code/external-agents) |
|
||||
| ↳ Claude Code CLI | [Example](examples/python/code/claude_code_example.py) | [📖](https://docs.praison.ai/code/claude-code) |
|
||||
| ↳ Gemini CLI | [Example](examples/python/code/gemini_cli_example.py) | [📖](https://docs.praison.ai/code/gemini-cli) |
|
||||
| ↳ Codex CLI | [Example](examples/python/code/codex_cli_example.py) | [📖](https://docs.praison.ai/code/codex-cli) |
|
||||
| ↳ Cursor CLI | [Example](examples/python/code/cursor_cli_example.py) | [📖](https://docs.praison.ai/code/cursor-cli) |
|
||||
| **🧠 Memory & Knowledge** | | |
|
||||
| ↳ Memory (Short & Long Term) | [Example](examples/python/general/memory_example.py) | [📖](https://docs.praison.ai/concepts/memory) |
|
||||
| ↳ File-Based Memory | [Example](examples/python/general/memory_example.py) | [📖](https://docs.praison.ai/concepts/memory) |
|
||||
| ↳ Claude Memory Tool | [Example](#claude-memory-tool-cli) | [📖](https://docs.praison.ai/features/claude-memory-tool) |
|
||||
| ↳ Add Custom Knowledge | [Example](examples/python/concepts/knowledge-agents.py) | [📖](https://docs.praison.ai/features/knowledge) |
|
||||
| ↳ RAG Agents | [Example](examples/python/concepts/rag-agents.py) | [📖](https://docs.praison.ai/features/rag) |
|
||||
| ↳ Chat with PDF Agents | [Example](examples/python/concepts/chat-with-pdf.py) | [📖](https://docs.praison.ai/features/chat-with-pdf) |
|
||||
| **🔬 Research & Intelligence** | | |
|
||||
| ↳ Deep Research Agents | [Example](examples/python/agents/research-agent.py) | [📖](https://docs.praison.ai/agents/deep-research) |
|
||||
| ↳ Query Rewriter Agent | [Example](#5-query-rewriter-agent) | [📖](https://docs.praison.ai/agents/query-rewriter) |
|
||||
| ↳ Native Web Search | [Example](examples/python/agents/websearch-agent.py) | [📖](https://docs.praison.ai/agents/websearch) |
|
||||
| ↳ Built-in Search Tools | [Example](examples/python/agents/websearch-agent.py) | [📖](https://docs.praison.ai/tools/tavily) |
|
||||
| ↳ Unified Web Search | [Example](src/praisonai-agents/examples/web_search_example.py) | [📖](https://docs.praison.ai/tools/web-search) |
|
||||
| ↳ Web Fetch (Anthropic) | [Example](#web-search-web-fetch--prompt-caching) | [📖](https://docs.praison.ai/features/model-capabilities) |
|
||||
| **📋 Planning & Execution** | | |
|
||||
| ↳ Planning Mode | [Example](examples/python/agents/planning-agent.py) | [📖](https://docs.praison.ai/features/planning-mode) |
|
||||
| ↳ Planning Tools | [Example](#3-agent-with-planning-mode) | [📖](https://docs.praison.ai/features/planning-mode) |
|
||||
| ↳ Planning Reasoning | [Example](#3-agent-with-planning-mode) | [📖](https://docs.praison.ai/features/planning-mode) |
|
||||
| ↳ Prompt Chaining | [Example](examples/python/general/prompt_chaining.py) | [📖](https://docs.praison.ai/features/promptchaining) |
|
||||
| ↳ Evaluator Optimiser | [Example](examples/python/general/evaluator-optimiser.py) | [📖](https://docs.praison.ai/features/evaluator-optimiser) |
|
||||
| ↳ Orchestrator Workers | [Example](examples/python/general/orchestrator-workers.py) | [📖](https://docs.praison.ai/features/orchestrator-worker) |
|
||||
| **👥 Specialized Agents** | | |
|
||||
| ↳ Data Analyst Agent | [Example](examples/python/agents/data-analyst-agent.py) | [📖](https://docs.praison.ai/agents/data-analyst) |
|
||||
| ↳ Finance Agent | [Example](examples/python/agents/finance-agent.py) | [📖](https://docs.praison.ai/agents/finance) |
|
||||
| ↳ Shopping Agent | [Example](examples/python/agents/shopping-agent.py) | [📖](https://docs.praison.ai/agents/shopping) |
|
||||
| ↳ Recommendation Agent | [Example](examples/python/agents/recommendation-agent.py) | [📖](https://docs.praison.ai/agents/recommendation) |
|
||||
| ↳ Wikipedia Agent | [Example](examples/python/agents/wikipedia-agent.py) | [📖](https://docs.praison.ai/agents/wikipedia) |
|
||||
| ↳ Programming Agent | [Example](examples/python/agents/programming-agent.py) | [📖](https://docs.praison.ai/agents/programming) |
|
||||
| ↳ Math Agents | [Example](examples/python/agents/math-agent.py) | [📖](https://docs.praison.ai/features/mathagent) |
|
||||
| ↳ Markdown Agent | [Example](examples/python/agents/markdown-agent.py) | [📖](https://docs.praison.ai/agents/markdown) |
|
||||
| ↳ Prompt Expander Agent | [Example](#prompt-expansion) | [📖](https://docs.praison.ai/agents/prompt-expander) |
|
||||
| **🎨 Media & Multimodal** | | |
|
||||
| ↳ Image Generation Agent | [Example](examples/python/image/image-agent.py) | [📖](https://docs.praison.ai/features/image-generation) |
|
||||
| ↳ Image to Text Agent | [Example](examples/python/agents/image-to-text-agent.py) | [📖](https://docs.praison.ai/agents/image-to-text) |
|
||||
| ↳ Video Agent | [Example](examples/python/agents/video-agent.py) | [📖](https://docs.praison.ai/agents/video) |
|
||||
| ↳ Camera Integration | [Example](examples/python/camera/) | [📖](https://docs.praison.ai/features/camera-integration) |
|
||||
| **🔌 Protocols & Integration** | | |
|
||||
| ↳ MCP Transports | [Example](examples/python/mcp/mcp-transports-overview.py) | [📖](https://docs.praison.ai/mcp/transports) |
|
||||
| ↳ WebSocket MCP | [Example](examples/python/mcp/websocket-mcp.py) | [📖](https://docs.praison.ai/mcp/sse-transport) |
|
||||
| ↳ MCP Security | [Example](examples/python/mcp/mcp-security.py) | [📖](https://docs.praison.ai/mcp/transports) |
|
||||
| ↳ MCP Resumability | [Example](examples/python/mcp/mcp-resumability.py) | [📖](https://docs.praison.ai/mcp/sse-transport) |
|
||||
| ↳ MCP Config Management | [Example](#mcp-config-cli) | [📖](https://docs.praison.ai/docs/cli/mcp) |
|
||||
| ↳ LangChain Integrated Agents | [Example](examples/python/general/langchain_example.py) | [📖](https://docs.praison.ai/features/langchain) |
|
||||
| **🛡️ Safety & Control** | | |
|
||||
| ↳ Guardrails | [Example](examples/python/guardrails/comprehensive-guardrails-example.py) | [📖](https://docs.praison.ai/features/guardrails) |
|
||||
| ↳ Human Approval | [Example](examples/python/general/human_approval_example.py) | [📖](https://docs.praison.ai/features/approval) |
|
||||
| ↳ Rules & Instructions | [Example](#6-rules--instructions) | [📖](https://docs.praison.ai/features/rules) |
|
||||
| **⚙️ Advanced Features** | | |
|
||||
| ↳ Async & Parallel Processing | [Example](examples/python/general/async_example.py) | [📖](https://docs.praison.ai/features/async) |
|
||||
| ↳ Parallelisation | [Example](examples/python/general/parallelisation.py) | [📖](https://docs.praison.ai/features/parallelisation) |
|
||||
| ↳ Repetitive Agents | [Example](examples/python/concepts/repetitive-agents.py) | [📖](https://docs.praison.ai/features/repetitive) |
|
||||
| ↳ Agent Handoffs | [Example](examples/python/handoff/handoff_basic.py) | [📖](https://docs.praison.ai/features/handoffs) |
|
||||
| ↳ Stateful Agents | [Example](examples/python/stateful/workflow-state-example.py) | [📖](https://docs.praison.ai/features/stateful-agents) |
|
||||
| ↳ Autonomous Workflow | [Example](examples/python/general/autonomous-agent.py) | [📖](https://docs.praison.ai/features/autonomous-workflow) |
|
||||
| ↳ Structured Output Agents | [Example](examples/python/general/structured_agents_example.py) | [📖](https://docs.praison.ai/features/structured) |
|
||||
| ↳ Model Router | [Example](examples/python/agents/router-agent-cost-optimization.py) | [📖](https://docs.praison.ai/features/model-router) |
|
||||
| ↳ Prompt Caching | [Example](#web-search-web-fetch--prompt-caching) | [📖](https://docs.praison.ai/features/model-capabilities) |
|
||||
| ↳ Fast Context | [Example](examples/context/00_agent_fast_context_basic.py) | [📖](https://docs.praison.ai/features/fast-context) |
|
||||
| **🛠️ Tools & Configuration** | | |
|
||||
| ↳ 100+ Custom Tools | [Example](examples/python/general/tools_example.py) | [📖](https://docs.praison.ai/tools/tools) |
|
||||
| ↳ YAML Configuration | [Example](examples/cookbooks/yaml/secondary_market_research_agents.yaml) | [📖](https://docs.praison.ai/developers/agents-playbook) |
|
||||
| ↳ 100+ LLM Support | [Example](examples/python/providers/openai/openai_gpt4_example.py) | [📖](https://docs.praison.ai/models) |
|
||||
| ↳ Callback Agents | [Example](examples/python/general/advanced-callback-systems.py) | [📖](https://docs.praison.ai/features/callbacks) |
|
||||
| ↳ Hooks | [Example](#9-hooks) | [📖](https://docs.praison.ai/features/hooks) |
|
||||
| **📊 Monitoring & Management** | | |
|
||||
| ↳ Sessions Management | [Example](examples/python/sessions/comprehensive-session-management.py) | [📖](https://docs.praison.ai/features/sessions) |
|
||||
| ↳ Auto-Save Sessions | [Example](#session-management-python) | [📖](https://docs.praison.ai/docs/cli/session) |
|
||||
| ↳ History in Context | [Example](#session-management-python) | [📖](https://docs.praison.ai/docs/cli/session) |
|
||||
| ↳ Telemetry | [Example](examples/python/telemetry/production-telemetry-example.py) | [📖](https://docs.praison.ai/features/telemetry) |
|
||||
| ↳ Project Docs (.praison/docs/) | [Example](#docs-cli) | [📖](https://docs.praison.ai/docs/cli/docs) |
|
||||
| ↳ AI Commit Messages | [Example](#ai-commit-cli) | [📖](https://docs.praison.ai/docs/cli/commit) |
|
||||
| ↳ @Mentions in Prompts | [Example](#mentions-in-prompts) | [📖](https://docs.praison.ai/docs/cli/mentions) |
|
||||
| **🖥️ CLI Features** | | |
|
||||
| ↳ Slash Commands | [Example](examples/python/cli/slash_commands_example.py) | [📖](https://docs.praison.ai/docs/cli/slash-commands) |
|
||||
| ↳ Autonomy Modes | [Example](examples/python/cli/autonomy_modes_example.py) | [📖](https://docs.praison.ai/docs/cli/autonomy-modes) |
|
||||
| ↳ Cost Tracking | [Example](examples/python/cli/cost_tracking_example.py) | [📖](https://docs.praison.ai/docs/cli/cost-tracking) |
|
||||
| ↳ Repository Map | [Example](examples/python/cli/repo_map_example.py) | [📖](https://docs.praison.ai/docs/cli/repo-map) |
|
||||
| ↳ Interactive TUI | [Example](examples/python/cli/interactive_tui_example.py) | [📖](https://docs.praison.ai/docs/cli/interactive-tui) |
|
||||
| ↳ Git Integration | [Example](examples/python/cli/git_integration_example.py) | [📖](https://docs.praison.ai/docs/cli/git-integration) |
|
||||
| ↳ Sandbox Execution | [Example](examples/python/cli/sandbox_execution_example.py) | [📖](https://docs.praison.ai/docs/cli/sandbox-execution) |
|
||||
| ↳ CLI Compare | [Example](examples/compare/cli_compare_basic.py) | [📖](https://docs.praison.ai/docs/cli/compare) |
|
||||
| ↳ Profile/Benchmark | [Example](#profile-benchmark) | [📖](https://docs.praison.ai/docs/cli/profile) |
|
||||
| ↳ Auto Mode | [Example](#auto-mode) | [📖](https://docs.praison.ai/docs/cli/auto) |
|
||||
| ↳ Init | [Example](#init) | [📖](https://docs.praison.ai/docs/cli/init) |
|
||||
| ↳ File Input | [Example](#file-input) | [📖](https://docs.praison.ai/docs/cli/file-input) |
|
||||
| ↳ Final Agent | [Example](#final-agent) | [📖](https://docs.praison.ai/docs/cli/final-agent) |
|
||||
| ↳ Max Tokens | [Example](#max-tokens) | [📖](https://docs.praison.ai/docs/cli/max-tokens) |
|
||||
| **🧪 Evaluation** | | |
|
||||
| ↳ Accuracy Evaluation | [Example](examples/eval/accuracy_example.py) | [📖](https://docs.praison.ai/docs/cli/eval) |
|
||||
| ↳ Performance Evaluation | [Example](examples/eval/performance_example.py) | [📖](https://docs.praison.ai/docs/cli/eval) |
|
||||
| ↳ Reliability Evaluation | [Example](examples/eval/reliability_example.py) | [📖](https://docs.praison.ai/docs/cli/eval) |
|
||||
| ↳ Criteria Evaluation | [Example](examples/eval/criteria_example.py) | [📖](https://docs.praison.ai/docs/cli/eval) |
|
||||
| **🎯 Agent Skills** | | |
|
||||
| ↳ Skills Management | [Example](examples/skills/basic_skill_usage.py) | [📖](https://docs.praison.ai/features/skills) |
|
||||
| ↳ Custom Skills | [Example](examples/skills/custom_skill_example.py) | [📖](https://docs.praison.ai/features/skills) |
|
||||
| **⏰ 24/7 Scheduling** | | |
|
||||
| ↳ Agent Scheduler | [Example](examples/python/scheduled_agents/news_checker_live.py) | [📖](https://docs.praison.ai/docs/cli/scheduler) |
|
||||
|
||||
---
|
||||
|
||||
@@ -941,6 +987,124 @@ Run with the same simple command:
|
||||
praisonai agents.yaml
|
||||
```
|
||||
|
||||
### 12. MCP (Model Context Protocol)
|
||||
|
||||
PraisonAI supports MCP Protocol Revision 2025-11-25 with multiple transports.
|
||||
|
||||
#### MCP Client (Consume MCP Servers)
|
||||
```python
|
||||
from praisonaiagents import Agent, MCP
|
||||
|
||||
# stdio - Local NPX/Python servers
|
||||
agent = Agent(tools=MCP("npx @modelcontextprotocol/server-memory"))
|
||||
|
||||
# Streamable HTTP - Production servers
|
||||
agent = Agent(tools=MCP("https://api.example.com/mcp"))
|
||||
|
||||
# WebSocket - Real-time bidirectional
|
||||
agent = Agent(tools=MCP("wss://api.example.com/mcp", auth_token="token"))
|
||||
|
||||
# SSE (Legacy) - Backward compatibility
|
||||
agent = Agent(tools=MCP("http://localhost:8080/sse"))
|
||||
|
||||
# With environment variables
|
||||
agent = Agent(
|
||||
tools=MCP(
|
||||
command="npx",
|
||||
args=["-y", "@modelcontextprotocol/server-brave-search"],
|
||||
env={"BRAVE_API_KEY": "your-key"}
|
||||
)
|
||||
)
|
||||
|
||||
# Multiple MCP servers + regular functions
|
||||
def my_custom_tool(query: str) -> str:
|
||||
"""Custom tool function."""
|
||||
return f"Result: {query}"
|
||||
|
||||
agent = Agent(
|
||||
name="MultiToolAgent",
|
||||
instructions="Agent with multiple MCP servers",
|
||||
tools=[
|
||||
MCP("uvx mcp-server-time"), # Time tools
|
||||
MCP("npx @modelcontextprotocol/server-memory"), # Memory tools
|
||||
my_custom_tool # Regular function
|
||||
]
|
||||
)
|
||||
```
|
||||
|
||||
#### MCP Server (Expose Tools as MCP Server)
|
||||
|
||||
Expose your Python functions as MCP tools for Claude Desktop, Cursor, and other MCP clients:
|
||||
|
||||
```python
|
||||
from praisonaiagents.mcp import ToolsMCPServer
|
||||
|
||||
def search_web(query: str, max_results: int = 5) -> dict:
|
||||
"""Search the web for information."""
|
||||
return {"results": [f"Result for {query}"]}
|
||||
|
||||
def calculate(expression: str) -> dict:
|
||||
"""Evaluate a mathematical expression."""
|
||||
return {"result": eval(expression)}
|
||||
|
||||
# Create and run MCP server
|
||||
server = ToolsMCPServer(name="my-tools")
|
||||
server.register_tools([search_web, calculate])
|
||||
server.run() # stdio for Claude Desktop
|
||||
# server.run_sse(host="0.0.0.0", port=8080) # SSE for web clients
|
||||
```
|
||||
|
||||
#### MCP Features
|
||||
| Feature | Description |
|
||||
|---------|-------------|
|
||||
| Session Management | Automatic Mcp-Session-Id handling |
|
||||
| Protocol Versioning | Mcp-Protocol-Version header |
|
||||
| Resumability | SSE stream recovery via Last-Event-ID |
|
||||
| Security | Origin validation, DNS rebinding prevention |
|
||||
| WebSocket | Auto-reconnect with exponential backoff |
|
||||
|
||||
### 13. A2A (Agent2Agent Protocol)
|
||||
|
||||
PraisonAI supports the [A2A Protocol](https://a2a-protocol.org) for agent-to-agent communication, enabling your agents to be discovered and collaborate with other AI agents.
|
||||
|
||||
#### A2A Server (Expose Agent as A2A Server)
|
||||
```python
|
||||
from praisonaiagents import Agent, A2A
|
||||
from fastapi import FastAPI
|
||||
|
||||
# Create an agent with tools
|
||||
def search_web(query: str) -> str:
|
||||
"""Search the web for information."""
|
||||
return f"Results for: {query}"
|
||||
|
||||
agent = Agent(
|
||||
name="Research Assistant",
|
||||
role="Research Analyst",
|
||||
goal="Help users research topics",
|
||||
tools=[search_web]
|
||||
)
|
||||
|
||||
# Expose as A2A Server
|
||||
a2a = A2A(agent=agent, url="http://localhost:8000/a2a")
|
||||
|
||||
app = FastAPI()
|
||||
app.include_router(a2a.get_router())
|
||||
|
||||
# Run: uvicorn app:app --reload
|
||||
# Agent Card: GET /.well-known/agent.json
|
||||
# Status: GET /status
|
||||
```
|
||||
|
||||
#### A2A Features
|
||||
| Feature | Description |
|
||||
|---------|-------------|
|
||||
| Agent Card | JSON metadata for agent discovery |
|
||||
| Skills Extraction | Auto-generate skills from tools |
|
||||
| Task Management | Stateful task lifecycle |
|
||||
| Streaming | SSE streaming for real-time updates |
|
||||
|
||||
> **Documentation**: [docs.praison.ai/a2a](https://docs.praison.ai/a2a) | **Examples**: [examples/python/a2a](https://github.com/MervinPraison/PraisonAI/tree/main/examples/python/a2a)
|
||||
|
||||
---
|
||||
|
||||
## 🎯 CLI / No-Code Interface
|
||||
@@ -954,6 +1118,32 @@ export OPENAI_API_KEY=xxxxxxxxxxxxxxxxxxxxxx
|
||||
praisonai --auto create a movie script about Robots in Mars
|
||||
```
|
||||
|
||||
### Interactive Mode CLI:
|
||||
```bash
|
||||
# Start interactive terminal mode (inspired by Gemini CLI, Codex CLI, Claude Code)
|
||||
praisonai --interactive
|
||||
praisonai -i
|
||||
|
||||
# Features:
|
||||
# - Streaming responses (no boxes)
|
||||
# - Built-in tools: read_file, write_file, list_files, execute_command, internet_search
|
||||
# - Slash commands: /help, /exit, /tools, /clear
|
||||
|
||||
# Chat mode - single prompt with interactive style (for testing/scripting)
|
||||
# Use --chat (or --chat-mode for backward compatibility)
|
||||
praisonai "list files in current folder" --chat
|
||||
praisonai "search the web for AI news" --chat
|
||||
praisonai "read README.md" --chat
|
||||
```
|
||||
|
||||
### Chat UI (Web Interface):
|
||||
```bash
|
||||
# Start web-based Chainlit chat interface (requires praisonai[chat])
|
||||
pip install "praisonai[chat]"
|
||||
praisonai chat
|
||||
# Opens browser at http://localhost:8084
|
||||
```
|
||||
|
||||
### Query Rewriting (works with any command):
|
||||
```bash
|
||||
# Rewrite query for better results (uses QueryRewriterAgent)
|
||||
@@ -1014,6 +1204,20 @@ praisonai "Complex analysis task" --planning --planning-reasoning
|
||||
praisonai "Task" --planning --auto-approve-plan
|
||||
```
|
||||
|
||||
### Tool Approval CLI:
|
||||
```bash
|
||||
# Auto-approve ALL tool executions (use with caution!)
|
||||
praisonai "run ls command" --trust
|
||||
|
||||
# Auto-approve tools up to a risk level (prompt for higher)
|
||||
# Levels: low, medium, high, critical
|
||||
praisonai "write to file" --approve-level high # Prompts for critical tools only
|
||||
praisonai "task" --approve-level medium # Prompts for high and critical
|
||||
|
||||
# Default behavior (no flags): prompts for all dangerous tools
|
||||
praisonai "run shell command" # Will prompt for approval
|
||||
```
|
||||
|
||||
### Memory CLI:
|
||||
```bash
|
||||
# Enable memory for agent (persists across sessions)
|
||||
@@ -1120,6 +1324,15 @@ praisonai workflow auto "Research AI trends" --pattern parallel
|
||||
# Generate routing workflow (classifier routes to specialists)
|
||||
praisonai workflow auto "Build a chatbot" --pattern routing
|
||||
|
||||
# Generate orchestrator-workers workflow (central orchestrator delegates)
|
||||
praisonai workflow auto "Comprehensive market analysis" --pattern orchestrator-workers
|
||||
|
||||
# Generate evaluator-optimizer workflow (iterative refinement)
|
||||
praisonai workflow auto "Write and refine article" --pattern evaluator-optimizer
|
||||
|
||||
# Specify output file
|
||||
praisonai workflow auto "Build a chatbot" --pattern routing
|
||||
|
||||
# Specify output file
|
||||
praisonai workflow auto "Research AI" --pattern sequential --output my_workflow.yaml
|
||||
```
|
||||
@@ -1129,7 +1342,7 @@ praisonai workflow auto "Research AI" --pattern sequential --output my_workflow.
|
||||
|------|-------------|
|
||||
| `--workflow-var key=value` | Set workflow variable (can be repeated) |
|
||||
| `--var key=value` | Set variable for YAML workflows |
|
||||
| `--pattern <pattern>` | Pattern for auto-generation (sequential, parallel, routing, loop) |
|
||||
| `--pattern <pattern>` | Pattern for auto-generation (sequential, parallel, routing, loop, orchestrator-workers, evaluator-optimizer) |
|
||||
| `--output <file>` | Output file for auto-generation |
|
||||
| `--llm <model>` | LLM model (e.g., openai/gpt-4o-mini) |
|
||||
| `--tools <tools>` | Tools (comma-separated, e.g., tavily) |
|
||||
@@ -1175,6 +1388,21 @@ praisonai "Analyze this data" --metrics
|
||||
praisonai "Complex task" --metrics --planning
|
||||
```
|
||||
|
||||
### Scheduler CLI:
|
||||
|
||||
```bash
|
||||
praisonai schedule start <name> "task" --interval hourly
|
||||
praisonai schedule list
|
||||
praisonai schedule logs <name> [--follow]
|
||||
praisonai schedule stop <name>
|
||||
praisonai schedule restart <name>
|
||||
praisonai schedule delete <name>
|
||||
praisonai schedule describe <name>
|
||||
praisonai schedule save <name> [file.yaml]
|
||||
praisonai schedule "task" --interval hourly # foreground mode
|
||||
praisonai schedule agents.yaml # foreground mode
|
||||
```
|
||||
|
||||
### Image Processing CLI:
|
||||
```bash
|
||||
# Process images with vision-based tasks
|
||||
@@ -1361,8 +1589,13 @@ praisonai "Complex analysis" --router --router-provider anthropic
|
||||
# Router automatically selects:
|
||||
# - Simple tasks → gpt-4o-mini, claude-3-haiku
|
||||
# - Complex tasks → gpt-4-turbo, claude-3-opus
|
||||
|
||||
# Create workflow with model routing template
|
||||
praisonai workflow create --template model-routing --output my_workflow.yaml
|
||||
```
|
||||
|
||||
Custom models can be configured in `agents.yaml`. See [Model Router Docs](https://docs.praison.ai/features/model-router) for details.
|
||||
|
||||
### Flow Display CLI:
|
||||
```bash
|
||||
# Enable visual workflow tracking
|
||||
@@ -1414,13 +1647,31 @@ praisonai mcp help
|
||||
|
||||
### AI Commit CLI:
|
||||
```bash
|
||||
# Generate AI commit message for staged changes
|
||||
# Full auto mode: stage all, security check, commit, and push
|
||||
praisonai commit -a
|
||||
|
||||
# Interactive mode (requires git add first)
|
||||
praisonai commit
|
||||
|
||||
# Generate, commit, and push
|
||||
# Interactive with auto-push
|
||||
praisonai commit --push
|
||||
|
||||
# Skip security check (not recommended)
|
||||
praisonai commit -a --no-verify
|
||||
```
|
||||
|
||||
**Features:**
|
||||
- 🤖 AI-generated conventional commit messages
|
||||
- 🔒 Built-in security scanning (API keys, passwords, secrets, sensitive files)
|
||||
- 📦 Auto-staging with `-a` flag
|
||||
- 🚀 Auto-push in full auto mode
|
||||
- ✏️ Edit message before commit in interactive mode
|
||||
|
||||
**Security Detection:**
|
||||
- API keys, secrets, tokens (AWS, GitHub, GitLab, Slack)
|
||||
- Passwords and private keys
|
||||
- Sensitive files (`.env`, `id_rsa`, `.pem`, `.key`, etc.)
|
||||
|
||||
### Serve CLI (API Server):
|
||||
```bash
|
||||
# Start API server for agents defined in YAML
|
||||
@@ -1451,6 +1702,55 @@ export N8N_API_KEY="your-api-key"
|
||||
praisonai agents.yaml --n8n
|
||||
```
|
||||
|
||||
### External Agents CLI:
|
||||
|
||||
Use external AI coding CLI tools (Claude Code, Gemini CLI, Codex CLI, Cursor CLI) as agent tools:
|
||||
|
||||
```bash
|
||||
# Use Claude Code for coding tasks
|
||||
praisonai "Refactor the auth module" --external-agent claude
|
||||
|
||||
# Use Gemini CLI for code analysis
|
||||
praisonai "Analyze codebase architecture" --external-agent gemini
|
||||
|
||||
# Use OpenAI Codex CLI
|
||||
praisonai "Fix all bugs in src/" --external-agent codex
|
||||
|
||||
# Use Cursor CLI
|
||||
praisonai "Add comprehensive tests" --external-agent cursor
|
||||
```
|
||||
|
||||
**Python API:**
|
||||
```python
|
||||
from praisonai.integrations import (
|
||||
ClaudeCodeIntegration,
|
||||
GeminiCLIIntegration,
|
||||
CodexCLIIntegration,
|
||||
CursorCLIIntegration
|
||||
)
|
||||
|
||||
# Create integration
|
||||
claude = ClaudeCodeIntegration(workspace="/project")
|
||||
|
||||
# Execute a coding task
|
||||
result = await claude.execute("Refactor the auth module")
|
||||
|
||||
# Use as agent tool
|
||||
from praisonai import Agent
|
||||
tool = claude.as_tool()
|
||||
agent = Agent(tools=[tool])
|
||||
```
|
||||
|
||||
**Environment Variables:**
|
||||
```bash
|
||||
export ANTHROPIC_API_KEY=your-key # Claude Code
|
||||
export GEMINI_API_KEY=your-key # Gemini CLI
|
||||
export OPENAI_API_KEY=your-key # Codex CLI
|
||||
export CURSOR_API_KEY=your-key # Cursor CLI
|
||||
```
|
||||
|
||||
See [External Agents Documentation](https://docs.praison.ai/code/external-agents) for more details.
|
||||
|
||||
### @Mentions in Prompts:
|
||||
```bash
|
||||
# Include file content in prompt
|
||||
@@ -1554,154 +1854,36 @@ agent = Agent(
|
||||
| Web Fetch | Anthropic |
|
||||
| Prompt Caching | OpenAI (auto), Anthropic, Bedrock, Deepseek |
|
||||
|
||||
## MCP (Model Context Protocol)
|
||||
|
||||
PraisonAI supports MCP Protocol Revision 2025-11-25 with multiple transports.
|
||||
|
||||
### MCP Client (Consume MCP Servers)
|
||||
```python
|
||||
from praisonaiagents import Agent, MCP
|
||||
|
||||
# stdio - Local NPX/Python servers
|
||||
agent = Agent(tools=MCP("npx @modelcontextprotocol/server-memory"))
|
||||
|
||||
# Streamable HTTP - Production servers
|
||||
agent = Agent(tools=MCP("https://api.example.com/mcp"))
|
||||
|
||||
# WebSocket - Real-time bidirectional
|
||||
agent = Agent(tools=MCP("wss://api.example.com/mcp", auth_token="token"))
|
||||
|
||||
# SSE (Legacy) - Backward compatibility
|
||||
agent = Agent(tools=MCP("http://localhost:8080/sse"))
|
||||
|
||||
# With environment variables
|
||||
agent = Agent(
|
||||
tools=MCP(
|
||||
command="npx",
|
||||
args=["-y", "@modelcontextprotocol/server-brave-search"],
|
||||
env={"BRAVE_API_KEY": "your-key"}
|
||||
)
|
||||
)
|
||||
|
||||
# Multiple MCP servers + regular functions
|
||||
def my_custom_tool(query: str) -> str:
|
||||
"""Custom tool function."""
|
||||
return f"Result: {query}"
|
||||
|
||||
agent = Agent(
|
||||
name="MultiToolAgent",
|
||||
instructions="Agent with multiple MCP servers",
|
||||
tools=[
|
||||
MCP("uvx mcp-server-time"), # Time tools
|
||||
MCP("npx @modelcontextprotocol/server-memory"), # Memory tools
|
||||
my_custom_tool # Regular function
|
||||
]
|
||||
)
|
||||
```
|
||||
|
||||
### MCP Server (Expose Tools as MCP Server)
|
||||
|
||||
Expose your Python functions as MCP tools for Claude Desktop, Cursor, and other MCP clients:
|
||||
|
||||
```python
|
||||
from praisonaiagents.mcp import ToolsMCPServer
|
||||
|
||||
def search_web(query: str, max_results: int = 5) -> dict:
|
||||
"""Search the web for information."""
|
||||
return {"results": [f"Result for {query}"]}
|
||||
|
||||
def calculate(expression: str) -> dict:
|
||||
"""Evaluate a mathematical expression."""
|
||||
return {"result": eval(expression)}
|
||||
|
||||
# Create and run MCP server
|
||||
server = ToolsMCPServer(name="my-tools")
|
||||
server.register_tools([search_web, calculate])
|
||||
server.run() # stdio for Claude Desktop
|
||||
# server.run_sse(host="0.0.0.0", port=8080) # SSE for web clients
|
||||
```
|
||||
|
||||
### MCP Features
|
||||
| Feature | Description |
|
||||
|---------|-------------|
|
||||
| Session Management | Automatic Mcp-Session-Id handling |
|
||||
| Protocol Versioning | Mcp-Protocol-Version header |
|
||||
| Resumability | SSE stream recovery via Last-Event-ID |
|
||||
| Security | Origin validation, DNS rebinding prevention |
|
||||
| WebSocket | Auto-reconnect with exponential backoff |
|
||||
|
||||
## A2A (Agent2Agent Protocol)
|
||||
|
||||
PraisonAI supports the [A2A Protocol](https://a2a-protocol.org) for agent-to-agent communication, enabling your agents to be discovered and collaborate with other AI agents.
|
||||
|
||||
### A2A Server (Expose Agent as A2A Server)
|
||||
```python
|
||||
from praisonaiagents import Agent, A2A
|
||||
from fastapi import FastAPI
|
||||
|
||||
# Create an agent with tools
|
||||
def search_web(query: str) -> str:
|
||||
"""Search the web for information."""
|
||||
return f"Results for: {query}"
|
||||
|
||||
agent = Agent(
|
||||
name="Research Assistant",
|
||||
role="Research Analyst",
|
||||
goal="Help users research topics",
|
||||
tools=[search_web]
|
||||
)
|
||||
|
||||
# Expose as A2A Server
|
||||
a2a = A2A(agent=agent, url="http://localhost:8000/a2a")
|
||||
|
||||
app = FastAPI()
|
||||
app.include_router(a2a.get_router())
|
||||
|
||||
# Run: uvicorn app:app --reload
|
||||
# Agent Card: GET /.well-known/agent.json
|
||||
# Status: GET /status
|
||||
```
|
||||
|
||||
### A2A Features
|
||||
| Feature | Description |
|
||||
|---------|-------------|
|
||||
| Agent Card | JSON metadata for agent discovery |
|
||||
| Skills Extraction | Auto-generate skills from tools |
|
||||
| Task Management | Stateful task lifecycle |
|
||||
| Streaming | SSE streaming for real-time updates |
|
||||
|
||||
> **Documentation**: [docs.praison.ai/a2a](https://docs.praison.ai/a2a) | **Examples**: [examples/python/a2a](https://github.com/MervinPraison/PraisonAI/tree/main/examples/python/a2a)
|
||||
|
||||
## CLI Features
|
||||
|
||||
| Feature | Docs |
|
||||
|---------|:----:|
|
||||
| 🔄 Query Rewrite - RAG optimization | [📖](https://docs.praison.ai/cli/query-rewrite) |
|
||||
| 🔬 Deep Research - Automated research | [📖](https://docs.praison.ai/cli/deep-research) |
|
||||
| 📋 Planning - Step-by-step execution | [📖](https://docs.praison.ai/cli/planning) |
|
||||
| 💾 Memory - Persistent agent memory | [📖](https://docs.praison.ai/cli/memory) |
|
||||
| 📜 Rules - Auto-discovered instructions | [📖](https://docs.praison.ai/cli/rules) |
|
||||
| 🔄 Workflow - Multi-step workflows | [📖](https://docs.praison.ai/cli/workflow) |
|
||||
| 🪝 Hooks - Event-driven actions | [📖](https://docs.praison.ai/cli/hooks) |
|
||||
| 🧠 Claude Memory - Anthropic memory tool | [📖](https://docs.praison.ai/cli/claude-memory) |
|
||||
| 🛡️ Guardrail - Output validation | [📖](https://docs.praison.ai/cli/guardrail) |
|
||||
| 📊 Metrics - Token usage tracking | [📖](https://docs.praison.ai/cli/metrics) |
|
||||
| 🖼️ Image - Vision processing | [📖](https://docs.praison.ai/cli/image) |
|
||||
| 📡 Telemetry - Usage monitoring | [📖](https://docs.praison.ai/cli/telemetry) |
|
||||
| 🔌 MCP - Model Context Protocol | [📖](https://docs.praison.ai/cli/mcp) |
|
||||
| ⚡ Fast Context - Codebase search | [📖](https://docs.praison.ai/cli/fast-context) |
|
||||
| 📚 Knowledge - RAG management | [📖](https://docs.praison.ai/cli/knowledge) |
|
||||
| 💬 Session - Conversation management | [📖](https://docs.praison.ai/cli/session) |
|
||||
| 🔧 Tools - Tool discovery | [📖](https://docs.praison.ai/cli/tools) |
|
||||
| 🤝 Handoff - Agent delegation | [📖](https://docs.praison.ai/cli/handoff) |
|
||||
| 🧠 Auto Memory - Memory extraction | [📖](https://docs.praison.ai/cli/auto-memory) |
|
||||
| 📋 Todo - Task management | [📖](https://docs.praison.ai/cli/todo) |
|
||||
| 🎯 Router - Smart model selection | [📖](https://docs.praison.ai/cli/router) |
|
||||
| 📈 Flow Display - Visual workflow | [📖](https://docs.praison.ai/cli/flow-display) |
|
||||
| ✨ Prompt Expansion - Detailed prompts | [📖](https://docs.praison.ai/cli/prompt-expansion) |
|
||||
| 🌐 Web Search - Real-time search | [📖](https://docs.praison.ai/cli/web-search) |
|
||||
| 📥 Web Fetch - URL content retrieval | [📖](https://docs.praison.ai/cli/web-fetch) |
|
||||
| 💾 Prompt Caching - Cost reduction | [📖](https://docs.praison.ai/cli/prompt-caching) |
|
||||
| 🔄 Query Rewrite - RAG optimization | [📖](https://docs.praison.ai/docs/cli/query-rewrite) |
|
||||
| 🔬 Deep Research - Automated research | [📖](https://docs.praison.ai/docs/cli/deep-research) |
|
||||
| 📋 Planning - Step-by-step execution | [📖](https://docs.praison.ai/docs/cli/planning) |
|
||||
| 💾 Memory - Persistent agent memory | [📖](https://docs.praison.ai/docs/cli/memory) |
|
||||
| 📜 Rules - Auto-discovered instructions | [📖](https://docs.praison.ai/docs/cli/rules) |
|
||||
| 🔄 Workflow - Multi-step workflows | [📖](https://docs.praison.ai/docs/cli/workflow) |
|
||||
| 🪝 Hooks - Event-driven actions | [📖](https://docs.praison.ai/docs/cli/hooks) |
|
||||
| 🧠 Claude Memory - Anthropic memory tool | [📖](https://docs.praison.ai/docs/cli/claude-memory) |
|
||||
| 🛡️ Guardrail - Output validation | [📖](https://docs.praison.ai/docs/cli/guardrail) |
|
||||
| 📊 Metrics - Token usage tracking | [📖](https://docs.praison.ai/docs/cli/metrics) |
|
||||
| 🖼️ Image - Vision processing | [📖](https://docs.praison.ai/docs/cli/image) |
|
||||
| 📡 Telemetry - Usage monitoring | [📖](https://docs.praison.ai/docs/cli/telemetry) |
|
||||
| 🔌 MCP - Model Context Protocol | [📖](https://docs.praison.ai/docs/cli/mcp) |
|
||||
| ⚡ Fast Context - Codebase search | [📖](https://docs.praison.ai/docs/cli/fast-context) |
|
||||
| 📚 Knowledge - RAG management | [📖](https://docs.praison.ai/docs/cli/knowledge) |
|
||||
| 💬 Session - Conversation management | [📖](https://docs.praison.ai/docs/cli/session) |
|
||||
| 🔧 Tools - Tool discovery | [📖](https://docs.praison.ai/docs/cli/tools) |
|
||||
| 🤝 Handoff - Agent delegation | [📖](https://docs.praison.ai/docs/cli/handoff) |
|
||||
| 🧠 Auto Memory - Memory extraction | [📖](https://docs.praison.ai/docs/cli/auto-memory) |
|
||||
| 📋 Todo - Task management | [📖](https://docs.praison.ai/docs/cli/todo) |
|
||||
| 🎯 Router - Smart model selection | [📖](https://docs.praison.ai/docs/cli/router) |
|
||||
| 📈 Flow Display - Visual workflow | [📖](https://docs.praison.ai/docs/cli/flow-display) |
|
||||
| ✨ Prompt Expansion - Detailed prompts | [📖](https://docs.praison.ai/docs/cli/prompt-expansion) |
|
||||
| 🌐 Web Search - Real-time search | [📖](https://docs.praison.ai/docs/cli/web-search) |
|
||||
| 📥 Web Fetch - URL content retrieval | [📖](https://docs.praison.ai/docs/cli/web-fetch) |
|
||||
| 💾 Prompt Caching - Cost reduction | [📖](https://docs.praison.ai/docs/cli/prompt-caching) |
|
||||
|
||||
---
|
||||
|
||||
@@ -2253,7 +2435,7 @@ PraisonAI provides zero-dependency persistent memory for agents. For detailed ex
|
||||
- 📖 **Wikipedia Agent** - Search and extract Wikipedia information
|
||||
- 💻 **Programming Agent** - Code development and analysis
|
||||
- 📝 **Markdown Agent** - Generate and format Markdown content
|
||||
- 🔀 **Router Agent** - Dynamic task routing with cost optimisation
|
||||
- 🔀 **Model Router** - Smart model selection based on task complexity
|
||||
|
||||
### MCP Protocol
|
||||
|
||||
@@ -2274,6 +2456,7 @@ PraisonAI provides zero-dependency persistent memory for agents. For detailed ex
|
||||
- 🤝 **Agent Handoffs** - Transfer context between specialised agents
|
||||
- 🛡️ **Guardrails** - Input/output validation and safety checks
|
||||
- ✅ **Human Approval** - Require human confirmation for critical actions
|
||||
- 🔐 **Tool Approval CLI** - `--trust` (auto-approve all) and `--approve-level` (risk-based approval)
|
||||
- 💬 **Sessions Management** - Isolated conversation contexts
|
||||
- 🔄 **Stateful Agents** - Maintain state across interactions
|
||||
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
# PraisonAI CLI - Benchmark Results
|
||||
|
||||
**Generated:** 2025-12-19 16:04:40
|
||||
|
||||
## Import Times
|
||||
|
||||
| Module | Time (ms) | Target | Status |
|
||||
|--------|-----------|--------|--------|
|
||||
| praisonai | 4.11 | <10 | ✅ |
|
||||
| message_queue | 6.87 | <50 | ✅ |
|
||||
| at_mentions | 0.58 | <50 | ✅ |
|
||||
| profiler | 0.90 | <10 | ✅ |
|
||||
|
||||
## Instantiation Times
|
||||
|
||||
| Class | Time (μs) | Target | Status |
|
||||
|-------|-----------|--------|--------|
|
||||
| MessageQueue | 0.26 | <100 | ✅ |
|
||||
| StateManager | 0.30 | <100 | ✅ |
|
||||
| FileSearchService | 8.42 | <100 | ✅ |
|
||||
|
||||
## Lazy Loading
|
||||
|
||||
| Operation | Time (ms) |
|
||||
|-----------|----------|
|
||||
| PraisonAI lazy load | 3886.47 |
|
||||
|
||||
## How to Reproduce
|
||||
|
||||
```bash
|
||||
cd praisonai
|
||||
python benchmarks/simple_benchmark.py
|
||||
```
|
||||
@@ -0,0 +1,231 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
PraisonAI CLI - Performance Benchmark
|
||||
|
||||
Measures import times and module loading performance.
|
||||
|
||||
Usage:
|
||||
python benchmarks/simple_benchmark.py
|
||||
python benchmarks/simple_benchmark.py --no-save
|
||||
"""
|
||||
|
||||
import time
|
||||
import sys
|
||||
import os
|
||||
import argparse
|
||||
from datetime import datetime
|
||||
|
||||
|
||||
ITERATIONS = 100
|
||||
|
||||
|
||||
def measure_import(module_name: str) -> float:
|
||||
"""Measure import time in milliseconds."""
|
||||
# Clear from cache if present
|
||||
modules_to_remove = [k for k in sys.modules.keys() if k.startswith(module_name)]
|
||||
for mod in modules_to_remove:
|
||||
del sys.modules[mod]
|
||||
|
||||
start = time.perf_counter()
|
||||
__import__(module_name)
|
||||
return (time.perf_counter() - start) * 1000
|
||||
|
||||
|
||||
def measure_instantiation(create_fn, iterations=ITERATIONS) -> float:
|
||||
"""Measure average instantiation time in microseconds."""
|
||||
times = []
|
||||
for _ in range(iterations):
|
||||
start = time.perf_counter()
|
||||
create_fn()
|
||||
times.append((time.perf_counter() - start) * 1_000_000)
|
||||
return sum(times) / len(times)
|
||||
|
||||
|
||||
def run_benchmark():
|
||||
"""Run the benchmark."""
|
||||
results = {}
|
||||
|
||||
print('=' * 60)
|
||||
print('PraisonAI CLI - Performance Benchmark')
|
||||
print('=' * 60)
|
||||
print(f'\nIterations: {ITERATIONS}')
|
||||
print('Metrics: Import time (ms), Instantiation time (μs)\n')
|
||||
|
||||
# 1. Package import time
|
||||
print("1. Testing package imports...")
|
||||
|
||||
# Clear cache
|
||||
modules_to_remove = [k for k in sys.modules.keys() if 'praisonai' in k]
|
||||
for mod in modules_to_remove:
|
||||
del sys.modules[mod]
|
||||
|
||||
start = time.perf_counter()
|
||||
import praisonai
|
||||
results['praisonai_import_ms'] = (time.perf_counter() - start) * 1000
|
||||
print(f" import praisonai: {results['praisonai_import_ms']:.2f}ms")
|
||||
|
||||
# 2. CLI features import
|
||||
print("\n2. Testing CLI features imports...")
|
||||
|
||||
start = time.perf_counter()
|
||||
from praisonai.cli.features.message_queue import MessageQueue
|
||||
results['message_queue_import_ms'] = (time.perf_counter() - start) * 1000
|
||||
print(f" message_queue: {results['message_queue_import_ms']:.2f}ms")
|
||||
|
||||
start = time.perf_counter()
|
||||
from praisonai.cli.features.at_mentions import CombinedCompleter
|
||||
results['at_mentions_import_ms'] = (time.perf_counter() - start) * 1000
|
||||
print(f" at_mentions: {results['at_mentions_import_ms']:.2f}ms")
|
||||
|
||||
# 3. Profiler import (should be fast, standalone)
|
||||
print("\n3. Testing profiler import...")
|
||||
start = time.perf_counter()
|
||||
from praisonai.profiler import Profiler
|
||||
results['profiler_import_ms'] = (time.perf_counter() - start) * 1000
|
||||
print(f" profiler: {results['profiler_import_ms']:.2f}ms")
|
||||
|
||||
# 4. Instantiation benchmarks
|
||||
print(f"\n4. Testing instantiation ({ITERATIONS} iterations)...")
|
||||
|
||||
from praisonai.cli.features.message_queue import MessageQueue, StateManager
|
||||
from praisonai.cli.features.at_mentions import FileSearchService
|
||||
|
||||
results['MessageQueue_us'] = measure_instantiation(lambda: MessageQueue())
|
||||
print(f" MessageQueue: {results['MessageQueue_us']:.2f}μs")
|
||||
|
||||
results['StateManager_us'] = measure_instantiation(lambda: StateManager())
|
||||
print(f" StateManager: {results['StateManager_us']:.2f}μs")
|
||||
|
||||
results['FileSearchService_us'] = measure_instantiation(
|
||||
lambda: FileSearchService(root_dir=".")
|
||||
)
|
||||
print(f" FileSearchService: {results['FileSearchService_us']:.2f}μs")
|
||||
|
||||
# 5. PraisonAI class (lazy load)
|
||||
print("\n5. Testing PraisonAI class access (lazy load)...")
|
||||
start = time.perf_counter()
|
||||
PraisonAI = praisonai.PraisonAI
|
||||
results['PraisonAI_lazy_ms'] = (time.perf_counter() - start) * 1000
|
||||
print(f" PraisonAI lazy load: {results['PraisonAI_lazy_ms']:.2f}ms")
|
||||
|
||||
return results
|
||||
|
||||
|
||||
def print_summary(results: dict):
|
||||
"""Print benchmark summary."""
|
||||
print('\n' + '=' * 60)
|
||||
print('SUMMARY')
|
||||
print('=' * 60)
|
||||
|
||||
print('\nImport Times (target: <50ms for features, <10ms for package):')
|
||||
print(f" {'Module':<25} {'Time':>10} {'Status':>10}")
|
||||
print(f" {'-'*25} {'-'*10} {'-'*10}")
|
||||
|
||||
import_checks = [
|
||||
('praisonai', results['praisonai_import_ms'], 10),
|
||||
('message_queue', results['message_queue_import_ms'], 50),
|
||||
('at_mentions', results['at_mentions_import_ms'], 50),
|
||||
('profiler', results['profiler_import_ms'], 10),
|
||||
]
|
||||
|
||||
all_pass = True
|
||||
for name, time_ms, threshold in import_checks:
|
||||
status = '✅ PASS' if time_ms < threshold else '❌ SLOW'
|
||||
if time_ms >= threshold:
|
||||
all_pass = False
|
||||
print(f" {name:<25} {time_ms:>8.2f}ms {status:>10}")
|
||||
|
||||
print('\nInstantiation Times (target: <100μs):')
|
||||
print(f" {'Class':<25} {'Time':>10} {'Status':>10}")
|
||||
print(f" {'-'*25} {'-'*10} {'-'*10}")
|
||||
|
||||
inst_checks = [
|
||||
('MessageQueue', results['MessageQueue_us'], 100),
|
||||
('StateManager', results['StateManager_us'], 100),
|
||||
('FileSearchService', results['FileSearchService_us'], 100),
|
||||
]
|
||||
|
||||
for name, time_us, threshold in inst_checks:
|
||||
status = '✅ PASS' if time_us < threshold else '❌ SLOW'
|
||||
if time_us >= threshold:
|
||||
all_pass = False
|
||||
print(f" {name:<25} {time_us:>8.2f}μs {status:>10}")
|
||||
|
||||
print('\n' + '=' * 60)
|
||||
if all_pass:
|
||||
print('✅ All benchmarks PASSED')
|
||||
else:
|
||||
print('❌ Some benchmarks FAILED')
|
||||
print('=' * 60)
|
||||
|
||||
return all_pass
|
||||
|
||||
|
||||
def save_results(results: dict):
|
||||
"""Save benchmark results to markdown file."""
|
||||
filepath = os.path.join(os.path.dirname(__file__), 'BENCHMARK_RESULTS.md')
|
||||
|
||||
with open(filepath, 'w') as f:
|
||||
f.write('# PraisonAI CLI - Benchmark Results\n\n')
|
||||
f.write(f'**Generated:** {datetime.now().strftime("%Y-%m-%d %H:%M:%S")}\n\n')
|
||||
|
||||
f.write('## Import Times\n\n')
|
||||
f.write('| Module | Time (ms) | Target | Status |\n')
|
||||
f.write('|--------|-----------|--------|--------|\n')
|
||||
|
||||
import_data = [
|
||||
('praisonai', results['praisonai_import_ms'], 10),
|
||||
('message_queue', results['message_queue_import_ms'], 50),
|
||||
('at_mentions', results['at_mentions_import_ms'], 50),
|
||||
('profiler', results['profiler_import_ms'], 10),
|
||||
]
|
||||
|
||||
for name, time_ms, threshold in import_data:
|
||||
status = '✅' if time_ms < threshold else '❌'
|
||||
f.write(f'| {name} | {time_ms:.2f} | <{threshold} | {status} |\n')
|
||||
|
||||
f.write('\n## Instantiation Times\n\n')
|
||||
f.write('| Class | Time (μs) | Target | Status |\n')
|
||||
f.write('|-------|-----------|--------|--------|\n')
|
||||
|
||||
inst_data = [
|
||||
('MessageQueue', results['MessageQueue_us'], 100),
|
||||
('StateManager', results['StateManager_us'], 100),
|
||||
('FileSearchService', results['FileSearchService_us'], 100),
|
||||
]
|
||||
|
||||
for name, time_us, threshold in inst_data:
|
||||
status = '✅' if time_us < threshold else '❌'
|
||||
f.write(f'| {name} | {time_us:.2f} | <{threshold} | {status} |\n')
|
||||
|
||||
f.write('\n## Lazy Loading\n\n')
|
||||
f.write(f'| Operation | Time (ms) |\n')
|
||||
f.write(f'|-----------|----------|\n')
|
||||
f.write(f'| PraisonAI lazy load | {results["PraisonAI_lazy_ms"]:.2f} |\n')
|
||||
|
||||
f.write('\n## How to Reproduce\n\n')
|
||||
f.write('```bash\n')
|
||||
f.write('cd praisonai\n')
|
||||
f.write('python benchmarks/simple_benchmark.py\n')
|
||||
f.write('```\n')
|
||||
|
||||
print(f'\nResults saved to: {filepath}')
|
||||
return filepath
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description='PraisonAI CLI Performance Benchmark')
|
||||
parser.add_argument('--no-save', action='store_true', help='Do not save results to file')
|
||||
args = parser.parse_args()
|
||||
|
||||
results = run_benchmark()
|
||||
all_pass = print_summary(results)
|
||||
|
||||
if not args.no_save:
|
||||
save_results(results)
|
||||
|
||||
return 0 if all_pass else 1
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
@@ -3,8 +3,8 @@ class Praisonai < Formula
|
||||
|
||||
desc "AI tools for various AI applications"
|
||||
homepage "https://github.com/MervinPraison/PraisonAI"
|
||||
url "https://github.com/MervinPraison/PraisonAI/archive/refs/tags/v2.3.40.tar.gz"
|
||||
sha256 `curl -sL https://github.com/MervinPraison/PraisonAI/archive/refs/tags/v2.3.40.tar.gz | shasum -a 256`.split.first
|
||||
url "https://github.com/MervinPraison/PraisonAI/archive/refs/tags/v2.3.87.tar.gz"
|
||||
sha256 `curl -sL https://github.com/MervinPraison/PraisonAI/archive/refs/tags/v2.3.87.tar.gz | shasum -a 256`.split.first
|
||||
license "MIT"
|
||||
|
||||
depends_on "[email protected]"
|
||||
|
||||
@@ -1,31 +1,34 @@
|
||||
# Suppress crewai.cli.config logger BEFORE any imports to prevent INFO log
|
||||
import logging
|
||||
logging.getLogger('crewai.cli.config').setLevel(logging.ERROR)
|
||||
|
||||
# Disable OpenTelemetry SDK
|
||||
import os
|
||||
os.environ["OTEL_SDK_DISABLED"] = "true"
|
||||
os.environ["EC_TELEMETRY"] = "false"
|
||||
from .cli import PraisonAI
|
||||
|
||||
# Version is lightweight, import directly
|
||||
from .version import __version__
|
||||
|
||||
# Re-export all classes from praisonaiagents to enable:
|
||||
# from praisonai import Agent, Task, PraisonAIAgents
|
||||
try:
|
||||
import praisonaiagents
|
||||
# Import all symbols from praisonaiagents using * import
|
||||
from praisonaiagents import *
|
||||
except ImportError:
|
||||
# If praisonaiagents is not available, these imports will fail gracefully
|
||||
pass
|
||||
|
||||
# Define __all__ to include both PraisonAI core classes and praisonaiagents exports
|
||||
# Define __all__ for lazy loading
|
||||
__all__ = [
|
||||
# Core PraisonAI classes
|
||||
'PraisonAI',
|
||||
'__version__',
|
||||
]
|
||||
|
||||
# Dynamically extend __all__ with praisonaiagents exports
|
||||
try:
|
||||
import praisonaiagents
|
||||
__all__.extend(praisonaiagents.__all__)
|
||||
except (ImportError, AttributeError):
|
||||
# If praisonaiagents is not available or doesn't have __all__, fail gracefully
|
||||
pass
|
||||
# Lazy loading for heavy imports
|
||||
def __getattr__(name):
|
||||
"""Lazy load heavy modules to improve import time."""
|
||||
if name == 'PraisonAI':
|
||||
from .cli import PraisonAI
|
||||
return PraisonAI
|
||||
|
||||
# Try praisonaiagents exports
|
||||
try:
|
||||
import praisonaiagents
|
||||
if hasattr(praisonaiagents, name):
|
||||
return getattr(praisonaiagents, name)
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
||||
@@ -0,0 +1,337 @@
|
||||
"""
|
||||
Agent Scheduler for PraisonAI - Run agents periodically 24/7.
|
||||
|
||||
This module provides scheduling capabilities for running PraisonAI agents
|
||||
at regular intervals, enabling 24/7 autonomous agent operations.
|
||||
|
||||
Example:
|
||||
# Run news checker every hour
|
||||
from praisonai.agent_scheduler import AgentScheduler
|
||||
from praisonai_agents import Agent
|
||||
|
||||
agent = Agent(
|
||||
name="NewsChecker",
|
||||
instructions="Check latest AI news and summarize",
|
||||
tools=[search_tool]
|
||||
)
|
||||
|
||||
scheduler = AgentScheduler(agent, task="Check latest AI news")
|
||||
scheduler.start(schedule_expr="hourly")
|
||||
"""
|
||||
|
||||
import threading
|
||||
import time
|
||||
import logging
|
||||
from datetime import datetime
|
||||
from typing import Optional, Dict, Any, Callable
|
||||
from abc import ABC, abstractmethod
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ScheduleParser:
|
||||
"""Parse schedule expressions into intervals."""
|
||||
|
||||
@staticmethod
|
||||
def parse(schedule_expr: str) -> int:
|
||||
"""
|
||||
Parse schedule expression and return interval in seconds.
|
||||
|
||||
Supported formats:
|
||||
- "daily" -> 86400 seconds
|
||||
- "hourly" -> 3600 seconds
|
||||
- "*/30m" -> 1800 seconds (every 30 minutes)
|
||||
- "*/1h" -> 3600 seconds (every 1 hour)
|
||||
- "60" -> 60 seconds (plain number)
|
||||
|
||||
Args:
|
||||
schedule_expr: Schedule expression string
|
||||
|
||||
Returns:
|
||||
Interval in seconds
|
||||
|
||||
Raises:
|
||||
ValueError: If schedule format is not supported
|
||||
"""
|
||||
schedule_expr = schedule_expr.strip().lower()
|
||||
|
||||
if schedule_expr == "daily":
|
||||
return 86400
|
||||
elif schedule_expr == "hourly":
|
||||
return 3600
|
||||
elif schedule_expr.isdigit():
|
||||
return int(schedule_expr)
|
||||
elif schedule_expr.startswith("*/"):
|
||||
interval_part = schedule_expr[2:]
|
||||
if interval_part.endswith("m"):
|
||||
minutes = int(interval_part[:-1])
|
||||
return minutes * 60
|
||||
elif interval_part.endswith("h"):
|
||||
hours = int(interval_part[:-1])
|
||||
return hours * 3600
|
||||
elif interval_part.endswith("s"):
|
||||
return int(interval_part[:-1])
|
||||
else:
|
||||
return int(interval_part)
|
||||
else:
|
||||
raise ValueError(f"Unsupported schedule format: {schedule_expr}")
|
||||
|
||||
|
||||
class AgentExecutorInterface(ABC):
|
||||
"""Abstract interface for agent execution."""
|
||||
|
||||
@abstractmethod
|
||||
def execute(self, task: str) -> Any:
|
||||
"""Execute the agent with given task."""
|
||||
pass
|
||||
|
||||
|
||||
class PraisonAgentExecutor(AgentExecutorInterface):
|
||||
"""Executor for PraisonAI agents."""
|
||||
|
||||
def __init__(self, agent):
|
||||
"""
|
||||
Initialize executor with a PraisonAI agent.
|
||||
|
||||
Args:
|
||||
agent: PraisonAI Agent instance
|
||||
"""
|
||||
self.agent = agent
|
||||
|
||||
def execute(self, task: str) -> Any:
|
||||
"""
|
||||
Execute the agent with given task.
|
||||
|
||||
Args:
|
||||
task: Task description for the agent
|
||||
|
||||
Returns:
|
||||
Agent execution result
|
||||
"""
|
||||
try:
|
||||
result = self.agent.start(task)
|
||||
return result
|
||||
except Exception as e:
|
||||
logger.error(f"Agent execution failed: {e}")
|
||||
raise
|
||||
|
||||
|
||||
class AgentScheduler:
|
||||
"""
|
||||
Scheduler for running PraisonAI agents periodically.
|
||||
|
||||
Features:
|
||||
- Interval-based scheduling (hourly, daily, custom)
|
||||
- Thread-safe operation
|
||||
- Automatic retry on failure
|
||||
- Execution logging and monitoring
|
||||
- Graceful shutdown
|
||||
|
||||
Example:
|
||||
scheduler = AgentScheduler(agent, task="Check news")
|
||||
scheduler.start(schedule_expr="hourly", max_retries=3)
|
||||
# Agent runs every hour automatically
|
||||
scheduler.stop() # Stop when needed
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
agent,
|
||||
task: str,
|
||||
config: Optional[Dict[str, Any]] = None,
|
||||
on_success: Optional[Callable] = None,
|
||||
on_failure: Optional[Callable] = None
|
||||
):
|
||||
"""
|
||||
Initialize agent scheduler.
|
||||
|
||||
Args:
|
||||
agent: PraisonAI Agent instance
|
||||
task: Task description to execute
|
||||
config: Optional configuration dict
|
||||
on_success: Callback function on successful execution
|
||||
on_failure: Callback function on failed execution
|
||||
"""
|
||||
self.agent = agent
|
||||
self.task = task
|
||||
self.config = config or {}
|
||||
self.on_success = on_success
|
||||
self.on_failure = on_failure
|
||||
|
||||
self.is_running = False
|
||||
self._stop_event = threading.Event()
|
||||
self._thread = None
|
||||
self._executor = PraisonAgentExecutor(agent)
|
||||
self._execution_count = 0
|
||||
self._success_count = 0
|
||||
self._failure_count = 0
|
||||
|
||||
def start(
|
||||
self,
|
||||
schedule_expr: str,
|
||||
max_retries: int = 3,
|
||||
run_immediately: bool = False
|
||||
) -> bool:
|
||||
"""
|
||||
Start scheduled agent execution.
|
||||
|
||||
Args:
|
||||
schedule_expr: Schedule expression (e.g., "hourly", "*/1h", "3600")
|
||||
max_retries: Maximum retry attempts on failure
|
||||
run_immediately: If True, run agent immediately before starting schedule
|
||||
|
||||
Returns:
|
||||
True if scheduler started successfully
|
||||
"""
|
||||
if self.is_running:
|
||||
logger.warning("Scheduler is already running")
|
||||
return False
|
||||
|
||||
try:
|
||||
interval = ScheduleParser.parse(schedule_expr)
|
||||
self.is_running = True
|
||||
self._stop_event.clear()
|
||||
|
||||
logger.info(f"Starting agent scheduler: {self.agent.name if hasattr(self.agent, 'name') else 'Agent'}")
|
||||
logger.info(f"Task: {self.task}")
|
||||
logger.info(f"Schedule: {schedule_expr} ({interval}s interval)")
|
||||
logger.info(f"Max retries: {max_retries}")
|
||||
|
||||
# Run immediately if requested
|
||||
if run_immediately:
|
||||
logger.info("Running agent immediately before starting schedule...")
|
||||
self._execute_with_retry(max_retries)
|
||||
|
||||
self._thread = threading.Thread(
|
||||
target=self._run_schedule,
|
||||
args=(interval, max_retries),
|
||||
daemon=True
|
||||
)
|
||||
self._thread.start()
|
||||
|
||||
logger.info("Agent scheduler started successfully")
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to start scheduler: {e}")
|
||||
self.is_running = False
|
||||
return False
|
||||
|
||||
def stop(self) -> bool:
|
||||
"""
|
||||
Stop the scheduler gracefully.
|
||||
|
||||
Returns:
|
||||
True if stopped successfully
|
||||
"""
|
||||
if not self.is_running:
|
||||
logger.info("Scheduler is not running")
|
||||
return True
|
||||
|
||||
logger.info("Stopping agent scheduler...")
|
||||
self._stop_event.set()
|
||||
|
||||
if self._thread and self._thread.is_alive():
|
||||
self._thread.join(timeout=10)
|
||||
|
||||
self.is_running = False
|
||||
logger.info("Agent scheduler stopped")
|
||||
logger.info(f"Execution stats - Total: {self._execution_count}, Success: {self._success_count}, Failed: {self._failure_count}")
|
||||
return True
|
||||
|
||||
def get_stats(self) -> Dict[str, Any]:
|
||||
"""
|
||||
Get execution statistics.
|
||||
|
||||
Returns:
|
||||
Dictionary with execution stats
|
||||
"""
|
||||
return {
|
||||
"is_running": self.is_running,
|
||||
"total_executions": self._execution_count,
|
||||
"successful_executions": self._success_count,
|
||||
"failed_executions": self._failure_count,
|
||||
"success_rate": (self._success_count / self._execution_count * 100) if self._execution_count > 0 else 0
|
||||
}
|
||||
|
||||
def _run_schedule(self, interval: int, max_retries: int):
|
||||
"""Internal method to run scheduled agent executions."""
|
||||
while not self._stop_event.is_set():
|
||||
logger.info(f"[{datetime.now().strftime('%Y-%m-%d %H:%M:%S')}] Starting scheduled agent execution")
|
||||
|
||||
self._execute_with_retry(max_retries)
|
||||
|
||||
# Wait for next scheduled time
|
||||
logger.info(f"Next execution in {interval} seconds ({interval/3600:.1f} hours)")
|
||||
self._stop_event.wait(interval)
|
||||
|
||||
def _execute_with_retry(self, max_retries: int):
|
||||
"""Execute agent with retry logic."""
|
||||
self._execution_count += 1
|
||||
success = False
|
||||
|
||||
for attempt in range(max_retries):
|
||||
try:
|
||||
logger.info(f"Attempt {attempt + 1}/{max_retries}")
|
||||
result = self._executor.execute(self.task)
|
||||
|
||||
logger.info(f"Agent execution successful on attempt {attempt + 1}")
|
||||
logger.info(f"Result: {result}")
|
||||
|
||||
self._success_count += 1
|
||||
success = True
|
||||
|
||||
if self.on_success:
|
||||
self.on_success(result)
|
||||
|
||||
break
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Agent execution failed on attempt {attempt + 1}: {e}")
|
||||
|
||||
if attempt < max_retries - 1:
|
||||
wait_time = 30 * (attempt + 1) # Exponential backoff
|
||||
logger.info(f"Waiting {wait_time}s before retry...")
|
||||
time.sleep(wait_time)
|
||||
|
||||
if not success:
|
||||
self._failure_count += 1
|
||||
logger.error(f"Agent execution failed after {max_retries} attempts")
|
||||
|
||||
if self.on_failure:
|
||||
self.on_failure(f"Failed after {max_retries} attempts")
|
||||
|
||||
def execute_once(self) -> Any:
|
||||
"""
|
||||
Execute agent immediately (one-time execution).
|
||||
|
||||
Returns:
|
||||
Agent execution result
|
||||
"""
|
||||
logger.info("Executing agent once")
|
||||
try:
|
||||
result = self._executor.execute(self.task)
|
||||
logger.info(f"One-time execution successful: {result}")
|
||||
return result
|
||||
except Exception as e:
|
||||
logger.error(f"One-time execution failed: {e}")
|
||||
raise
|
||||
|
||||
|
||||
def create_agent_scheduler(
|
||||
agent,
|
||||
task: str,
|
||||
config: Optional[Dict[str, Any]] = None
|
||||
) -> AgentScheduler:
|
||||
"""
|
||||
Factory function to create agent scheduler.
|
||||
|
||||
Args:
|
||||
agent: PraisonAI Agent instance
|
||||
task: Task description
|
||||
config: Optional configuration
|
||||
|
||||
Returns:
|
||||
Configured AgentScheduler instance
|
||||
"""
|
||||
return AgentScheduler(agent, task, config)
|
||||
+742
-105
@@ -51,14 +51,340 @@ try:
|
||||
except ImportError:
|
||||
PRAISONAI_TOOLS_AVAILABLE = False
|
||||
|
||||
# LiteLLM availability check for multi-provider support
|
||||
LITELLM_AVAILABLE = False
|
||||
try:
|
||||
import litellm # noqa: F401 - imported for availability check
|
||||
LITELLM_AVAILABLE = True
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
logging.basicConfig(level=os.environ.get('LOGLEVEL', 'INFO').upper(), format='%(asctime)s - %(levelname)s - %(message)s')
|
||||
|
||||
# Define Pydantic models outside of the generate method
|
||||
# =============================================================================
|
||||
# Available Tools List (shared between generators) - Legacy for praisonai_tools
|
||||
# =============================================================================
|
||||
AVAILABLE_TOOLS = [
|
||||
"CodeDocsSearchTool", "CSVSearchTool", "DirectorySearchTool", "DOCXSearchTool",
|
||||
"DirectoryReadTool", "FileReadTool", "TXTSearchTool", "JSONSearchTool",
|
||||
"MDXSearchTool", "PDFSearchTool", "RagTool", "ScrapeElementFromWebsiteTool",
|
||||
"ScrapeWebsiteTool", "WebsiteSearchTool", "XMLSearchTool",
|
||||
"YoutubeChannelSearchTool", "YoutubeVideoSearchTool"
|
||||
]
|
||||
|
||||
# =============================================================================
|
||||
# Enhanced Tool Discovery from praisonaiagents.tools
|
||||
# =============================================================================
|
||||
|
||||
# Tool categories with their tools from praisonaiagents.tools
|
||||
TOOL_CATEGORIES = {
|
||||
'web_search': [
|
||||
'internet_search', 'duckduckgo', 'tavily_search', 'exa_search',
|
||||
'search_web', 'ydc_search', 'searxng_search'
|
||||
],
|
||||
'web_scraping': [
|
||||
'scrape_page', 'extract_links', 'crawl', 'extract_text',
|
||||
'crawl4ai', 'crawl4ai_extract', 'get_article'
|
||||
],
|
||||
'file_operations': [
|
||||
'read_file', 'write_file', 'list_files', 'get_file_info',
|
||||
'copy_file', 'move_file', 'delete_file'
|
||||
],
|
||||
'code_execution': [
|
||||
'execute_command', 'execute_code', 'analyze_code', 'format_code'
|
||||
],
|
||||
'data_processing': [
|
||||
'read_csv', 'write_csv', 'analyze_csv', 'read_json', 'write_json',
|
||||
'read_excel', 'write_excel', 'read_yaml', 'write_yaml', 'read_xml'
|
||||
],
|
||||
'research': [
|
||||
'search_arxiv', 'get_arxiv_paper', 'wiki_search', 'wiki_summary',
|
||||
'get_news_sources', 'get_trending_topics'
|
||||
],
|
||||
'finance': [
|
||||
'get_stock_price', 'get_stock_info', 'get_historical_data'
|
||||
],
|
||||
'math': [
|
||||
'evaluate', 'solve_equation', 'convert_units', 'calculate_statistics'
|
||||
],
|
||||
'database': [
|
||||
'query', 'create_table', 'load_data', 'find_documents', 'vector_search'
|
||||
]
|
||||
}
|
||||
|
||||
# Keywords that map to tool categories
|
||||
TASK_KEYWORD_TO_TOOLS = {
|
||||
# Web search keywords
|
||||
'search': 'web_search',
|
||||
'find': 'web_search',
|
||||
'look up': 'web_search',
|
||||
'google': 'web_search',
|
||||
'internet': 'web_search',
|
||||
'online': 'web_search',
|
||||
'web': 'web_search',
|
||||
|
||||
# Web scraping keywords
|
||||
'scrape': 'web_scraping',
|
||||
'crawl': 'web_scraping',
|
||||
'extract from website': 'web_scraping',
|
||||
'get from url': 'web_scraping',
|
||||
'fetch page': 'web_scraping',
|
||||
|
||||
# File operation keywords
|
||||
'read file': 'file_operations',
|
||||
'write file': 'file_operations',
|
||||
'save': 'file_operations',
|
||||
'load': 'file_operations',
|
||||
'open file': 'file_operations',
|
||||
'create file': 'file_operations',
|
||||
|
||||
# Code execution keywords
|
||||
'execute': 'code_execution',
|
||||
'run code': 'code_execution',
|
||||
'python': 'code_execution',
|
||||
'script': 'code_execution',
|
||||
'command': 'code_execution',
|
||||
'shell': 'code_execution',
|
||||
|
||||
# Data processing keywords
|
||||
'csv': 'data_processing',
|
||||
'excel': 'data_processing',
|
||||
'json': 'data_processing',
|
||||
'yaml': 'data_processing',
|
||||
'xml': 'data_processing',
|
||||
'data': 'data_processing',
|
||||
'spreadsheet': 'data_processing',
|
||||
|
||||
# Research keywords
|
||||
'research': 'research',
|
||||
'paper': 'research',
|
||||
'arxiv': 'research',
|
||||
'wikipedia': 'research',
|
||||
'academic': 'research',
|
||||
'news': 'research',
|
||||
|
||||
# Finance keywords
|
||||
'stock': 'finance',
|
||||
'price': 'finance',
|
||||
'market': 'finance',
|
||||
'financial': 'finance',
|
||||
'trading': 'finance',
|
||||
|
||||
# Math keywords
|
||||
'calculate': 'math',
|
||||
'math': 'math',
|
||||
'equation': 'math',
|
||||
'compute': 'math',
|
||||
'statistics': 'math',
|
||||
|
||||
# Database keywords
|
||||
'database': 'database',
|
||||
'sql': 'database',
|
||||
'query': 'database',
|
||||
'mongodb': 'database',
|
||||
'vector': 'database'
|
||||
}
|
||||
|
||||
|
||||
def get_all_available_tools() -> Dict[str, List[str]]:
|
||||
"""
|
||||
Get all available tools organized by category.
|
||||
|
||||
Returns:
|
||||
Dict mapping category names to lists of tool names
|
||||
"""
|
||||
return TOOL_CATEGORIES.copy()
|
||||
|
||||
|
||||
def get_tools_for_task(task_description: str) -> List[str]:
|
||||
"""
|
||||
Analyze a task description and return appropriate tools.
|
||||
|
||||
Args:
|
||||
task_description: The task to analyze
|
||||
|
||||
Returns:
|
||||
List of tool names appropriate for the task
|
||||
"""
|
||||
task_lower = task_description.lower()
|
||||
matched_categories = set()
|
||||
|
||||
# Match keywords to categories
|
||||
for keyword, category in TASK_KEYWORD_TO_TOOLS.items():
|
||||
if keyword in task_lower:
|
||||
matched_categories.add(category)
|
||||
|
||||
# Collect tools from matched categories
|
||||
tools = []
|
||||
for category in matched_categories:
|
||||
if category in TOOL_CATEGORIES:
|
||||
tools.extend(TOOL_CATEGORIES[category])
|
||||
|
||||
# Always include core tools for flexibility
|
||||
core_tools = ['read_file', 'write_file', 'execute_command']
|
||||
for tool in core_tools:
|
||||
if tool not in tools:
|
||||
tools.append(tool)
|
||||
|
||||
# Remove duplicates while preserving order
|
||||
seen = set()
|
||||
unique_tools = []
|
||||
for tool in tools:
|
||||
if tool not in seen:
|
||||
seen.add(tool)
|
||||
unique_tools.append(tool)
|
||||
|
||||
return unique_tools
|
||||
|
||||
|
||||
def recommend_agent_count(task_description: str) -> int:
|
||||
"""
|
||||
Recommend the optimal number of agents based on task complexity.
|
||||
|
||||
Args:
|
||||
task_description: The task to analyze
|
||||
|
||||
Returns:
|
||||
Recommended number of agents (1-4)
|
||||
"""
|
||||
complexity = BaseAutoGenerator.analyze_complexity(task_description)
|
||||
|
||||
if complexity == 'simple':
|
||||
return 1
|
||||
elif complexity == 'moderate':
|
||||
return 2
|
||||
else: # complex
|
||||
# Count distinct aspects of the task
|
||||
task_lower = task_description.lower()
|
||||
aspects = 0
|
||||
|
||||
aspect_keywords = [
|
||||
['research', 'search', 'find', 'gather'],
|
||||
['analyze', 'evaluate', 'assess', 'review'],
|
||||
['write', 'create', 'generate', 'produce'],
|
||||
['edit', 'refine', 'improve', 'polish'],
|
||||
['coordinate', 'manage', 'orchestrate', 'delegate']
|
||||
]
|
||||
|
||||
for keyword_group in aspect_keywords:
|
||||
if any(kw in task_lower for kw in keyword_group):
|
||||
aspects += 1
|
||||
|
||||
return min(max(aspects, 2), 4) # Between 2 and 4 agents
|
||||
|
||||
# =============================================================================
|
||||
# Base Generator Class (DRY - shared functionality)
|
||||
# =============================================================================
|
||||
class BaseAutoGenerator:
|
||||
"""
|
||||
Base class for auto-generators with shared functionality.
|
||||
|
||||
Provides:
|
||||
- Lazy-loaded instructor client (LiteLLM or OpenAI fallback)
|
||||
- Environment variable handling for model/API configuration
|
||||
- Config list management
|
||||
"""
|
||||
|
||||
def __init__(self, config_list: Optional[List[Dict]] = None):
|
||||
"""
|
||||
Initialize base generator with LLM configuration.
|
||||
|
||||
Args:
|
||||
config_list: Optional LLM configuration list
|
||||
"""
|
||||
# Support multiple environment variable patterns for better compatibility
|
||||
model_name = os.environ.get("MODEL_NAME") or os.environ.get("OPENAI_MODEL_NAME", "gpt-4o-mini")
|
||||
base_url = (
|
||||
os.environ.get("OPENAI_BASE_URL") or
|
||||
os.environ.get("OPENAI_API_BASE") or
|
||||
os.environ.get("OLLAMA_API_BASE", "https://api.openai.com/v1")
|
||||
)
|
||||
|
||||
self.config_list = config_list or [
|
||||
{
|
||||
'model': model_name,
|
||||
'base_url': base_url,
|
||||
'api_key': os.environ.get("OPENAI_API_KEY")
|
||||
}
|
||||
]
|
||||
self._client = None # Lazy loading for performance
|
||||
|
||||
@property
|
||||
def client(self):
|
||||
"""Lazy load the instructor client to avoid performance impact.
|
||||
|
||||
Uses LiteLLM via instructor.from_provider if available for multi-provider support,
|
||||
otherwise falls back to direct OpenAI SDK.
|
||||
"""
|
||||
if self._client is None:
|
||||
if LITELLM_AVAILABLE:
|
||||
# Use LiteLLM for multi-provider support (100+ LLMs)
|
||||
model_name = self.config_list[0]['model']
|
||||
self._client = instructor.from_provider(
|
||||
f"litellm/{model_name}",
|
||||
mode=instructor.Mode.JSON
|
||||
)
|
||||
else:
|
||||
# Fallback to direct OpenAI SDK
|
||||
self._client = instructor.patch(
|
||||
OpenAI(
|
||||
base_url=self.config_list[0]['base_url'],
|
||||
api_key=self.config_list[0]['api_key'],
|
||||
),
|
||||
mode=instructor.Mode.JSON,
|
||||
)
|
||||
return self._client
|
||||
|
||||
@staticmethod
|
||||
def get_available_tools() -> List[str]:
|
||||
"""Return list of available tools for agent assignment."""
|
||||
return AVAILABLE_TOOLS.copy()
|
||||
|
||||
@staticmethod
|
||||
def analyze_complexity(topic: str) -> str:
|
||||
"""
|
||||
Analyze task complexity based on keywords.
|
||||
|
||||
Args:
|
||||
topic: The task description
|
||||
|
||||
Returns:
|
||||
str: Complexity level - 'simple', 'moderate', or 'complex'
|
||||
"""
|
||||
topic_lower = topic.lower()
|
||||
|
||||
# Complex task indicators
|
||||
complex_keywords = [
|
||||
'comprehensive', 'multi-step', 'analyze and', 'research and write',
|
||||
'multiple', 'coordinate', 'complex', 'detailed analysis',
|
||||
'full report', 'in-depth', 'thorough'
|
||||
]
|
||||
|
||||
# Simple task indicators
|
||||
simple_keywords = [
|
||||
'write a', 'create a', 'simple', 'quick', 'brief',
|
||||
'haiku', 'poem', 'summary', 'list', 'single'
|
||||
]
|
||||
|
||||
if any(kw in topic_lower for kw in complex_keywords):
|
||||
return 'complex'
|
||||
elif any(kw in topic_lower for kw in simple_keywords):
|
||||
return 'simple'
|
||||
else:
|
||||
return 'moderate'
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Pydantic Models for Structured Output
|
||||
# =============================================================================
|
||||
|
||||
class TaskDetails(BaseModel):
|
||||
"""Details for a single task."""
|
||||
description: str
|
||||
expected_output: str
|
||||
|
||||
class RoleDetails(BaseModel):
|
||||
"""Details for a single role/agent."""
|
||||
role: str
|
||||
goal: str
|
||||
backstory: str
|
||||
@@ -66,14 +392,62 @@ class RoleDetails(BaseModel):
|
||||
tools: List[str]
|
||||
|
||||
class TeamStructure(BaseModel):
|
||||
"""Structure for multi-agent team."""
|
||||
roles: Dict[str, RoleDetails]
|
||||
|
||||
class AutoGenerator:
|
||||
def __init__(self, topic="Movie Story writing about AI", agent_file="test.yaml", framework="crewai", config_list: Optional[List[Dict]] = None):
|
||||
class SingleAgentStructure(BaseModel):
|
||||
"""Structure for single-agent generation (Anthropic's 'start simple' principle)."""
|
||||
name: str
|
||||
role: str
|
||||
goal: str
|
||||
backstory: str
|
||||
instructions: str
|
||||
tools: List[str] = []
|
||||
task_description: str
|
||||
expected_output: str
|
||||
|
||||
class PatternRecommendation(BaseModel):
|
||||
"""LLM-based pattern recommendation with reasoning."""
|
||||
pattern: str # sequential, parallel, routing, orchestrator-workers, evaluator-optimizer
|
||||
reasoning: str # Why this pattern was chosen
|
||||
confidence: float # 0.0 to 1.0 confidence score
|
||||
|
||||
class ValidationGate(BaseModel):
|
||||
"""Validation gate for prompt chaining workflows."""
|
||||
criteria: str # What to validate
|
||||
pass_action: str # Action if validation passes (e.g., "continue", "next_step")
|
||||
fail_action: str # Action if validation fails (e.g., "retry", "escalate", "abort")
|
||||
|
||||
class AutoGenerator(BaseAutoGenerator):
|
||||
"""
|
||||
Auto-generates agents.yaml files from a topic description.
|
||||
|
||||
Inherits from BaseAutoGenerator for shared LLM client functionality.
|
||||
|
||||
Usage:
|
||||
generator = AutoGenerator(framework="crewai", topic="Create a movie script")
|
||||
path = generator.generate()
|
||||
"""
|
||||
|
||||
def __init__(self, topic="Movie Story writing about AI", agent_file="test.yaml",
|
||||
framework="crewai", config_list: Optional[List[Dict]] = None,
|
||||
pattern: str = "sequential", single_agent: bool = False):
|
||||
"""
|
||||
Initialize the AutoGenerator class with the specified topic, agent file, and framework.
|
||||
|
||||
Args:
|
||||
topic: The task/topic for agent generation
|
||||
agent_file: Output YAML file name
|
||||
framework: Framework to use (crewai, autogen, praisonai)
|
||||
config_list: Optional LLM configuration
|
||||
pattern: Workflow pattern (sequential, parallel, routing, orchestrator-workers, evaluator-optimizer)
|
||||
single_agent: If True, generate a single agent instead of a team
|
||||
|
||||
Note: autogen framework is different from this AutoGenerator class.
|
||||
"""
|
||||
# Initialize base class first (handles config_list and client)
|
||||
super().__init__(config_list=config_list)
|
||||
|
||||
# Validate framework availability and show framework-specific messages
|
||||
if framework == "crewai" and not CREWAI_AVAILABLE:
|
||||
raise ImportError("""
|
||||
@@ -106,35 +480,42 @@ Tools are not available for {framework}. To use tools, install:
|
||||
pip install "praisonai[{framework}]"
|
||||
""")
|
||||
|
||||
# Support multiple environment variable patterns for better compatibility
|
||||
# Priority order: MODEL_NAME > OPENAI_MODEL_NAME for model selection
|
||||
model_name = os.environ.get("MODEL_NAME") or os.environ.get("OPENAI_MODEL_NAME", "gpt-5-nano")
|
||||
|
||||
# Priority order for base_url: OPENAI_BASE_URL > OPENAI_API_BASE > OLLAMA_API_BASE
|
||||
# OPENAI_BASE_URL is the standard OpenAI SDK environment variable
|
||||
base_url = (
|
||||
os.environ.get("OPENAI_BASE_URL") or
|
||||
os.environ.get("OPENAI_API_BASE") or
|
||||
os.environ.get("OLLAMA_API_BASE", "https://api.openai.com/v1")
|
||||
)
|
||||
|
||||
self.config_list = config_list or [
|
||||
{
|
||||
'model': model_name,
|
||||
'base_url': base_url,
|
||||
'api_key': os.environ.get("OPENAI_API_KEY")
|
||||
}
|
||||
]
|
||||
self.topic = topic
|
||||
self.agent_file = agent_file
|
||||
self.framework = framework or "praisonai"
|
||||
self.client = instructor.patch(
|
||||
OpenAI(
|
||||
base_url=self.config_list[0]['base_url'],
|
||||
api_key=self.config_list[0]['api_key'],
|
||||
),
|
||||
mode=instructor.Mode.JSON,
|
||||
)
|
||||
self.pattern = pattern
|
||||
self.single_agent = single_agent
|
||||
|
||||
def recommend_pattern(self, topic: str = None) -> str:
|
||||
"""
|
||||
Recommend the best workflow pattern based on task characteristics.
|
||||
|
||||
Args:
|
||||
topic: The task description (uses self.topic if not provided)
|
||||
|
||||
Returns:
|
||||
str: Recommended pattern name
|
||||
"""
|
||||
task = topic or self.topic
|
||||
task_lower = task.lower()
|
||||
|
||||
# Keywords that suggest specific patterns
|
||||
parallel_keywords = ['multiple', 'concurrent', 'parallel', 'simultaneously', 'different sources', 'compare', 'various']
|
||||
routing_keywords = ['classify', 'categorize', 'route', 'different types', 'depending on', 'if...then']
|
||||
orchestrator_keywords = ['complex', 'comprehensive', 'multi-step', 'coordinate', 'delegate', 'break down', 'analyze and']
|
||||
evaluator_keywords = ['refine', 'improve', 'iterate', 'quality', 'review', 'feedback', 'polish', 'optimize']
|
||||
|
||||
# Check for pattern indicators
|
||||
if any(kw in task_lower for kw in evaluator_keywords):
|
||||
return "evaluator-optimizer"
|
||||
elif any(kw in task_lower for kw in orchestrator_keywords):
|
||||
return "orchestrator-workers"
|
||||
elif any(kw in task_lower for kw in routing_keywords):
|
||||
return "routing"
|
||||
elif any(kw in task_lower for kw in parallel_keywords):
|
||||
return "parallel"
|
||||
else:
|
||||
return "sequential"
|
||||
|
||||
def generate(self, merge=False):
|
||||
"""
|
||||
@@ -157,7 +538,8 @@ Tools are not available for {framework}. To use tools, install:
|
||||
response = self.client.chat.completions.create(
|
||||
model=self.config_list[0]['model'],
|
||||
response_model=TeamStructure,
|
||||
max_retries=10,
|
||||
max_retries=5,
|
||||
timeout=120.0, # 2 minute timeout for complex generations
|
||||
messages=[
|
||||
{"role": "system", "content": "You are a helpful assistant designed to output complex team structures."},
|
||||
{"role": "user", "content": self.get_user_content()}
|
||||
@@ -194,8 +576,7 @@ Tools are not available for {framework}. To use tools, install:
|
||||
"goal": role_details['goal'],
|
||||
"role": role_details['role'],
|
||||
"tasks": {},
|
||||
# "tools": role_details.get('tools', []),
|
||||
"tools": ['']
|
||||
"tools": role_details.get('tools', [])
|
||||
}
|
||||
|
||||
for task_id, task_details in role_details['tasks'].items():
|
||||
@@ -265,7 +646,7 @@ Tools are not available for {framework}. To use tools, install:
|
||||
"goal": role_details['goal'],
|
||||
"role": role_details['role'],
|
||||
"tasks": {},
|
||||
"tools": ['']
|
||||
"tools": role_details.get('tools', [])
|
||||
}
|
||||
|
||||
# Add tasks for this role
|
||||
@@ -277,9 +658,19 @@ Tools are not available for {framework}. To use tools, install:
|
||||
|
||||
return merged_data
|
||||
|
||||
def discover_tools_for_topic(self) -> List[str]:
|
||||
"""
|
||||
Discover appropriate tools for the topic using intelligent matching.
|
||||
|
||||
Returns:
|
||||
List of tool names appropriate for this topic
|
||||
"""
|
||||
return get_tools_for_task(self.topic)
|
||||
|
||||
def get_user_content(self):
|
||||
"""
|
||||
Generates a prompt for the OpenAI API to generate a team structure.
|
||||
Uses intelligent tool discovery based on task analysis.
|
||||
|
||||
Args:
|
||||
None
|
||||
@@ -292,45 +683,104 @@ Tools are not available for {framework}. To use tools, install:
|
||||
prompt = generator.get_user_content()
|
||||
print(prompt)
|
||||
"""
|
||||
user_content = """Generate a team structure for \"""" + self.topic + """\" task.
|
||||
No Input data will be provided to the team.
|
||||
The team will work in sequence. First role will pass the output to the next role, and so on.
|
||||
The last role will generate the final output.
|
||||
Think step by step.
|
||||
With maximum 3 roles, each with 1 task. Include role goals, backstories, task descriptions, and expected outputs.
|
||||
List of Available Tools: CodeDocsSearchTool, CSVSearchTool, DirectorySearchTool, DOCXSearchTool, DirectoryReadTool, FileReadTool, TXTSearchTool, JSONSearchTool, MDXSearchTool, PDFSearchTool, RagTool, ScrapeElementFromWebsiteTool, ScrapeWebsiteTool, WebsiteSearchTool, XMLSearchTool, YoutubeChannelSearchTool, YoutubeVideoSearchTool.
|
||||
Only use Available Tools. Do Not use any other tools.
|
||||
Example Below:
|
||||
Use below example to understand the structure of the output.
|
||||
The final role you create should satisfy the provided task: """ + self.topic + """.
|
||||
{
|
||||
"roles": {
|
||||
"narrative_designer": {
|
||||
"role": "Narrative Designer",
|
||||
"goal": "Create AI storylines",
|
||||
"backstory": "Skilled in narrative development for AI, with a focus on story resonance.",
|
||||
"tools": ["ScrapeWebsiteTool"],
|
||||
"tasks": {
|
||||
"story_concept_development": {
|
||||
"description": "Craft a unique AI story concept with depth and engagement using concept from this page the content https://www.asthebirdfliesblog.com/posts/how-to-write-book-story-development .",
|
||||
"expected_output": "Document with narrative arcs, character bios, and settings."
|
||||
}
|
||||
}
|
||||
},
|
||||
"scriptwriter": {
|
||||
"role": "Scriptwriter",
|
||||
"goal": "Write scripts from AI concepts",
|
||||
"backstory": "Expert in dialogue and script structure, translating concepts into scripts.",
|
||||
"tasks": {
|
||||
"scriptwriting_task": {
|
||||
"description": "Turn narrative concepts into scripts, including dialogue and scenes.",
|
||||
"expected_output": "Production-ready script with dialogue and scene details."
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
"""
|
||||
# Pattern-specific guidance
|
||||
pattern_guidance = {
|
||||
"sequential": "The team will work in sequence. Each role passes output to the next.",
|
||||
"parallel": "The team will work in parallel on independent subtasks, then combine results.",
|
||||
"routing": "A classifier agent will route requests to specialized agents based on input type.",
|
||||
"orchestrator-workers": "A central orchestrator will dynamically delegate tasks to specialized workers.",
|
||||
"evaluator-optimizer": "One agent generates content, another evaluates it in a loop until quality criteria are met."
|
||||
}
|
||||
|
||||
workflow_guidance = pattern_guidance.get(self.pattern, pattern_guidance["sequential"])
|
||||
|
||||
# Get recommended tools based on task analysis
|
||||
recommended_tools = self.discover_tools_for_topic()
|
||||
recommended_agent_count = recommend_agent_count(self.topic)
|
||||
complexity = self.analyze_complexity(self.topic)
|
||||
|
||||
# Build comprehensive tool list with categories
|
||||
all_tools_by_category = []
|
||||
for category, tools in TOOL_CATEGORIES.items():
|
||||
all_tools_by_category.append(f" {category}: {', '.join(tools)}")
|
||||
tools_reference = "\n".join(all_tools_by_category)
|
||||
|
||||
# Also include legacy tools for backward compatibility
|
||||
legacy_tools = ", ".join(AVAILABLE_TOOLS)
|
||||
|
||||
user_content = f"""Analyze and generate a team structure for: "{self.topic}"
|
||||
|
||||
TASK COMPLEXITY ANALYSIS (Pre-computed):
|
||||
- Complexity: {complexity}
|
||||
- Recommended agents: {recommended_agent_count}
|
||||
- Recommended tools based on task keywords: {', '.join(recommended_tools)}
|
||||
|
||||
STEP 1: VALIDATE TASK ANALYSIS
|
||||
Review the pre-computed analysis above. Adjust if needed based on your understanding.
|
||||
|
||||
STEP 2: DETERMINE OPTIMAL TEAM SIZE
|
||||
Based on complexity analysis:
|
||||
- Simple tasks: 1-2 agents (single focused agent or simple pair)
|
||||
- Moderate tasks: 2-3 agents (researcher + executor pattern)
|
||||
- Complex tasks: 3-4 agents (specialized team)
|
||||
|
||||
Recommended for this task: {recommended_agent_count} agent(s)
|
||||
|
||||
IMPORTANT: Avoid unnecessary complexity. Only add agents if there is meaningful specialization.
|
||||
Each agent must have a distinct, non-overlapping responsibility.
|
||||
|
||||
STEP 3: DESIGN THE TEAM (Pattern: {self.pattern})
|
||||
{workflow_guidance}
|
||||
|
||||
Each agent should have:
|
||||
- A clear, distinct role with meaningful specialization
|
||||
- A specific goal
|
||||
- Relevant backstory
|
||||
- 1 focused task with clear description and expected output
|
||||
- Appropriate tools from the recommended list
|
||||
|
||||
AVAILABLE TOOLS BY CATEGORY:
|
||||
{tools_reference}
|
||||
|
||||
LEGACY TOOLS (for backward compatibility):
|
||||
{legacy_tools}
|
||||
|
||||
RECOMMENDED TOOLS FOR THIS TASK: {', '.join(recommended_tools)}
|
||||
Prioritize using the recommended tools. Only add others if specifically needed.
|
||||
|
||||
Example structure (2 agents for a research + writing task):
|
||||
{{
|
||||
"roles": {{
|
||||
"researcher": {{
|
||||
"role": "Research Analyst",
|
||||
"goal": "Gather comprehensive information on the topic",
|
||||
"backstory": "Expert researcher skilled at finding and synthesizing information.",
|
||||
"tools": ["internet_search", "read_file"],
|
||||
"tasks": {{
|
||||
"research_task": {{
|
||||
"description": "Research key information about the topic and compile findings.",
|
||||
"expected_output": "Comprehensive research notes with key facts and insights."
|
||||
}}
|
||||
}}
|
||||
}},
|
||||
"writer": {{
|
||||
"role": "Content Writer",
|
||||
"goal": "Create polished final content",
|
||||
"backstory": "Skilled writer who transforms research into engaging content.",
|
||||
"tools": ["write_file"],
|
||||
"tasks": {{
|
||||
"writing_task": {{
|
||||
"description": "Write the final content based on research findings.",
|
||||
"expected_output": "Polished, well-structured final document."
|
||||
}}
|
||||
}}
|
||||
}}
|
||||
}}
|
||||
}}
|
||||
|
||||
Now generate the optimal team structure for: {self.topic}
|
||||
Use the recommended tools: {', '.join(recommended_tools)}
|
||||
"""
|
||||
return user_content
|
||||
|
||||
|
||||
@@ -372,13 +822,14 @@ class WorkflowStructure(BaseModel):
|
||||
description: str
|
||||
agents: Dict[str, WorkflowAgentDetails]
|
||||
steps: List[Dict] # Can be agent steps, route, parallel, etc.
|
||||
gates: Optional[List[ValidationGate]] = None # Optional validation gates
|
||||
|
||||
|
||||
class WorkflowAutoGenerator:
|
||||
class WorkflowAutoGenerator(BaseAutoGenerator):
|
||||
"""
|
||||
Auto-generates workflow.yaml files from a topic description.
|
||||
|
||||
Uses lazy loading to avoid performance impact when not used.
|
||||
Inherits from BaseAutoGenerator for shared LLM client functionality.
|
||||
|
||||
Usage:
|
||||
generator = WorkflowAutoGenerator(topic="Research AI trends and write a report")
|
||||
@@ -387,7 +838,9 @@ class WorkflowAutoGenerator:
|
||||
|
||||
def __init__(self, topic: str = "Research and write about AI",
|
||||
workflow_file: str = "workflow.yaml",
|
||||
config_list: Optional[List[Dict]] = None):
|
||||
config_list: Optional[List[Dict]] = None,
|
||||
framework: str = "praisonai",
|
||||
single_agent: bool = False):
|
||||
"""
|
||||
Initialize the WorkflowAutoGenerator.
|
||||
|
||||
@@ -395,45 +848,105 @@ class WorkflowAutoGenerator:
|
||||
topic: The task/topic for the workflow
|
||||
workflow_file: Output file name
|
||||
config_list: Optional LLM configuration
|
||||
framework: Framework to use (praisonai, crewai, autogen)
|
||||
single_agent: If True, generate a single agent workflow
|
||||
"""
|
||||
# Support multiple environment variable patterns
|
||||
model_name = os.environ.get("MODEL_NAME") or os.environ.get("OPENAI_MODEL_NAME", "gpt-4o-mini")
|
||||
base_url = (
|
||||
os.environ.get("OPENAI_BASE_URL") or
|
||||
os.environ.get("OPENAI_API_BASE") or
|
||||
os.environ.get("OLLAMA_API_BASE", "https://api.openai.com/v1")
|
||||
)
|
||||
# Initialize base class (handles config_list and client)
|
||||
super().__init__(config_list=config_list)
|
||||
|
||||
self.config_list = config_list or [
|
||||
{
|
||||
'model': model_name,
|
||||
'base_url': base_url,
|
||||
'api_key': os.environ.get("OPENAI_API_KEY")
|
||||
}
|
||||
]
|
||||
self.topic = topic
|
||||
self.workflow_file = workflow_file
|
||||
self._client = None # Lazy loading
|
||||
self.framework = framework
|
||||
self.single_agent = single_agent
|
||||
|
||||
@property
|
||||
def client(self):
|
||||
"""Lazy load the OpenAI client to avoid performance impact."""
|
||||
if self._client is None:
|
||||
self._client = instructor.patch(
|
||||
OpenAI(
|
||||
base_url=self.config_list[0]['base_url'],
|
||||
api_key=self.config_list[0]['api_key'],
|
||||
),
|
||||
mode=instructor.Mode.JSON,
|
||||
)
|
||||
return self._client
|
||||
def recommend_pattern(self, topic: str = None) -> str:
|
||||
"""
|
||||
Recommend the best workflow pattern based on task characteristics.
|
||||
|
||||
Args:
|
||||
topic: The task description (uses self.topic if not provided)
|
||||
|
||||
Returns:
|
||||
str: Recommended pattern name
|
||||
|
||||
Pattern recommendations based on Anthropic's best practices:
|
||||
- sequential: Clear step-by-step dependencies
|
||||
- parallel: Independent subtasks that can run concurrently
|
||||
- routing: Different input types need different handling
|
||||
- orchestrator-workers: Complex tasks needing dynamic decomposition
|
||||
- evaluator-optimizer: Tasks requiring iterative refinement
|
||||
"""
|
||||
task = topic or self.topic
|
||||
task_lower = task.lower()
|
||||
|
||||
# Keywords that suggest specific patterns
|
||||
parallel_keywords = ['multiple', 'concurrent', 'parallel', 'simultaneously', 'different sources', 'compare', 'various']
|
||||
routing_keywords = ['classify', 'categorize', 'route', 'different types', 'depending on', 'if...then']
|
||||
orchestrator_keywords = ['complex', 'comprehensive', 'multi-step', 'coordinate', 'delegate', 'break down', 'analyze and']
|
||||
evaluator_keywords = ['refine', 'improve', 'iterate', 'quality', 'review', 'feedback', 'polish', 'optimize']
|
||||
|
||||
# Check for pattern indicators
|
||||
if any(kw in task_lower for kw in evaluator_keywords):
|
||||
return "evaluator-optimizer"
|
||||
elif any(kw in task_lower for kw in orchestrator_keywords):
|
||||
return "orchestrator-workers"
|
||||
elif any(kw in task_lower for kw in routing_keywords):
|
||||
return "routing"
|
||||
elif any(kw in task_lower for kw in parallel_keywords):
|
||||
return "parallel"
|
||||
else:
|
||||
return "sequential"
|
||||
|
||||
def generate(self, pattern: str = "sequential") -> str:
|
||||
def recommend_pattern_llm(self, topic: str = None) -> PatternRecommendation:
|
||||
"""
|
||||
Use LLM to recommend the best workflow pattern with reasoning.
|
||||
|
||||
Args:
|
||||
topic: The task description (uses self.topic if not provided)
|
||||
|
||||
Returns:
|
||||
PatternRecommendation: Pattern with reasoning and confidence score
|
||||
"""
|
||||
task = topic or self.topic
|
||||
|
||||
prompt = f"""Analyze this task and recommend the best workflow pattern:
|
||||
|
||||
Task: "{task}"
|
||||
|
||||
Available patterns:
|
||||
1. sequential - Agents work one after another, passing output to the next
|
||||
2. parallel - Multiple agents work concurrently on independent subtasks
|
||||
3. routing - A classifier routes requests to specialized agents based on input type
|
||||
4. orchestrator-workers - Central orchestrator dynamically delegates to specialized workers
|
||||
5. evaluator-optimizer - Generator creates content, evaluator reviews in a loop until quality met
|
||||
|
||||
Respond with:
|
||||
- pattern: The recommended pattern name
|
||||
- reasoning: Why this pattern is best for this task
|
||||
- confidence: Your confidence score (0.0 to 1.0)
|
||||
"""
|
||||
|
||||
response = self.client.chat.completions.create(
|
||||
model=self.config_list[0]['model'],
|
||||
response_model=PatternRecommendation,
|
||||
max_retries=3,
|
||||
timeout=60.0,
|
||||
messages=[
|
||||
{"role": "system", "content": "You are an expert at designing AI agent workflows."},
|
||||
{"role": "user", "content": prompt}
|
||||
]
|
||||
)
|
||||
|
||||
return response
|
||||
|
||||
def generate(self, pattern: str = "sequential", merge: bool = False) -> str:
|
||||
"""
|
||||
Generate a workflow YAML file.
|
||||
|
||||
Args:
|
||||
pattern: Workflow pattern - "sequential", "routing", "parallel", "loop"
|
||||
pattern: Workflow pattern - "sequential", "routing", "parallel", "loop",
|
||||
"orchestrator-workers", "evaluator-optimizer"
|
||||
merge: If True, merge with existing workflow file instead of overwriting
|
||||
|
||||
Returns:
|
||||
Path to the generated workflow file
|
||||
@@ -442,6 +955,7 @@ class WorkflowAutoGenerator:
|
||||
model=self.config_list[0]['model'],
|
||||
response_model=WorkflowStructure,
|
||||
max_retries=5,
|
||||
timeout=120.0, # 2 minute timeout for complex generations
|
||||
messages=[
|
||||
{"role": "system", "content": "You are a helpful assistant that designs workflow structures."},
|
||||
{"role": "user", "content": self._get_prompt(pattern)}
|
||||
@@ -449,16 +963,86 @@ class WorkflowAutoGenerator:
|
||||
)
|
||||
|
||||
json_data = json.loads(response.model_dump_json())
|
||||
|
||||
if merge and os.path.exists(self.workflow_file):
|
||||
return self._save_workflow(self.merge_with_existing_workflow(json_data), pattern)
|
||||
return self._save_workflow(json_data, pattern)
|
||||
|
||||
def merge_with_existing_workflow(self, new_data: Dict) -> Dict:
|
||||
"""
|
||||
Merge new workflow data with existing workflow file.
|
||||
|
||||
Args:
|
||||
new_data: The new workflow data to merge
|
||||
|
||||
Returns:
|
||||
Dict: Merged workflow data
|
||||
"""
|
||||
try:
|
||||
with open(self.workflow_file, 'r') as f:
|
||||
existing_data = yaml.safe_load(f)
|
||||
|
||||
if not existing_data:
|
||||
return new_data
|
||||
except (yaml.YAMLError, FileNotFoundError) as e:
|
||||
logging.warning(f"Could not load existing workflow file {self.workflow_file}: {e}")
|
||||
return new_data
|
||||
|
||||
# Merge agents (avoid duplicates)
|
||||
merged_agents = existing_data.get('agents', {}).copy()
|
||||
for agent_id, agent_data in new_data.get('agents', {}).items():
|
||||
# Rename if conflict
|
||||
final_id = agent_id
|
||||
counter = 1
|
||||
while final_id in merged_agents:
|
||||
final_id = f"{agent_id}_auto_{counter}"
|
||||
counter += 1
|
||||
merged_agents[final_id] = agent_data
|
||||
|
||||
# Merge steps (append new steps)
|
||||
merged_steps = existing_data.get('steps', []) + new_data.get('steps', [])
|
||||
|
||||
# Create merged structure
|
||||
merged = {
|
||||
'name': existing_data.get('name', new_data.get('name', 'Merged Workflow')),
|
||||
'description': f"{existing_data.get('description', '')} + {new_data.get('description', '')}",
|
||||
'agents': merged_agents,
|
||||
'steps': merged_steps
|
||||
}
|
||||
|
||||
return merged
|
||||
|
||||
def _get_prompt(self, pattern: str) -> str:
|
||||
"""Generate the prompt based on the workflow pattern."""
|
||||
# Analyze complexity to determine agent count
|
||||
complexity = self.analyze_complexity(self.topic)
|
||||
if complexity == 'simple':
|
||||
agent_guidance = "Create 1-2 agents (simple task detected)."
|
||||
elif complexity == 'complex':
|
||||
agent_guidance = "Create 3-4 agents (complex task detected)."
|
||||
else:
|
||||
agent_guidance = "Create 2-3 agents (moderate task detected)."
|
||||
|
||||
# Get available tools
|
||||
tools_list = ", ".join(self.get_available_tools())
|
||||
|
||||
base_prompt = f"""Generate a workflow structure for: "{self.topic}"
|
||||
|
||||
STEP 1: ANALYZE TASK COMPLEXITY
|
||||
- Is this a simple task (1-2 agents)?
|
||||
- Does it require multiple specialists (2-3 agents)?
|
||||
- Is it complex with many dependencies (3-4 agents)?
|
||||
|
||||
STEP 2: DESIGN WORKFLOW
|
||||
The workflow should use the "{pattern}" pattern.
|
||||
Create 2-4 agents with clear roles and instructions.
|
||||
{agent_guidance}
|
||||
Each agent should have clear roles and instructions.
|
||||
Each step should have a clear action.
|
||||
|
||||
STEP 3: ASSIGN TOOLS (if needed)
|
||||
Available Tools: {tools_list}
|
||||
Only assign tools if the task requires them. Use empty list or null if no tools needed.
|
||||
|
||||
"""
|
||||
|
||||
if pattern == "routing":
|
||||
@@ -501,6 +1085,59 @@ Example structure:
|
||||
{"agent": "aggregator", "action": "Combine all findings"}
|
||||
]
|
||||
}
|
||||
"""
|
||||
elif pattern == "orchestrator-workers":
|
||||
base_prompt += """
|
||||
Create an orchestrator-workers workflow where a central orchestrator dynamically delegates tasks to specialized workers.
|
||||
The orchestrator analyzes the input, decides which workers are needed, and synthesizes results.
|
||||
|
||||
Example structure:
|
||||
{
|
||||
"name": "Orchestrator-Workers Workflow",
|
||||
"description": "Central orchestrator delegates to specialized workers",
|
||||
"agents": {
|
||||
"orchestrator": {"name": "Orchestrator", "role": "Task Coordinator", "goal": "Analyze tasks and delegate to appropriate workers", "instructions": "Break down the task, identify required specialists, and coordinate their work. Output a JSON with 'subtasks' array listing which workers to invoke."},
|
||||
"researcher": {"name": "Researcher", "role": "Research Specialist", "goal": "Gather information", "instructions": "Research and provide factual information"},
|
||||
"analyst": {"name": "Analyst", "role": "Data Analyst", "goal": "Analyze data and patterns", "instructions": "Analyze information and identify insights"},
|
||||
"writer": {"name": "Writer", "role": "Content Writer", "goal": "Create written content", "instructions": "Write clear, engaging content"},
|
||||
"synthesizer": {"name": "Synthesizer", "role": "Results Synthesizer", "goal": "Combine all worker outputs", "instructions": "Synthesize all worker outputs into a coherent final result"}
|
||||
},
|
||||
"steps": [
|
||||
{"agent": "orchestrator", "action": "Analyze task and determine required workers: {{input}}"},
|
||||
{"name": "worker_dispatch", "parallel": [
|
||||
{"agent": "researcher", "action": "Research: {{input}}"},
|
||||
{"agent": "analyst", "action": "Analyze: {{input}}"},
|
||||
{"agent": "writer", "action": "Draft content for: {{input}}"}
|
||||
]},
|
||||
{"agent": "synthesizer", "action": "Combine all worker outputs into final result"}
|
||||
]
|
||||
}
|
||||
"""
|
||||
elif pattern == "evaluator-optimizer":
|
||||
base_prompt += """
|
||||
Create an evaluator-optimizer workflow where one agent generates content and another evaluates it in a loop.
|
||||
The generator improves based on evaluator feedback until quality criteria are met.
|
||||
|
||||
Example structure:
|
||||
{
|
||||
"name": "Evaluator-Optimizer Workflow",
|
||||
"description": "Iterative refinement through generation and evaluation",
|
||||
"agents": {
|
||||
"generator": {"name": "Generator", "role": "Content Generator", "goal": "Generate high-quality content", "instructions": "Create content based on the input. If feedback is provided, improve the content accordingly."},
|
||||
"evaluator": {"name": "Evaluator", "role": "Quality Evaluator", "goal": "Evaluate content quality", "instructions": "Evaluate the content on: clarity, accuracy, completeness, and relevance. Score 1-10 for each. If average score < 7, provide specific improvement feedback. If score >= 7, respond with 'APPROVED'."}
|
||||
},
|
||||
"steps": [
|
||||
{"agent": "generator", "action": "Generate initial content for: {{input}}"},
|
||||
{"name": "evaluation_loop", "loop": {
|
||||
"agent": "evaluator",
|
||||
"action": "Evaluate the generated content",
|
||||
"condition": "output does not contain 'APPROVED'",
|
||||
"max_iterations": 3,
|
||||
"feedback_to": "generator"
|
||||
}},
|
||||
{"agent": "generator", "action": "Finalize content based on all feedback"}
|
||||
]
|
||||
}
|
||||
"""
|
||||
else: # sequential
|
||||
base_prompt += """
|
||||
|
||||
@@ -8,12 +8,19 @@ Structure:
|
||||
- features/: Feature handlers for CLI flags and commands
|
||||
"""
|
||||
|
||||
from .main import PraisonAI
|
||||
|
||||
__all__ = ["PraisonAI"]
|
||||
|
||||
|
||||
def __getattr__(name):
|
||||
"""Lazy load PraisonAI to avoid slow imports."""
|
||||
if name == "PraisonAI":
|
||||
from .main import PraisonAI
|
||||
return PraisonAI
|
||||
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
||||
|
||||
|
||||
def main():
|
||||
"""CLI entry point function."""
|
||||
from .main import PraisonAI
|
||||
praison_ai = PraisonAI()
|
||||
praison_ai.main()
|
||||
@@ -21,6 +21,14 @@ Features:
|
||||
- flow_display: Visual workflow tracking
|
||||
- workflow: YAML workflow management
|
||||
- n8n: Export workflows to n8n visual editor
|
||||
- external_agents: External AI CLI tool integrations (Claude, Gemini, Codex, Cursor)
|
||||
- slash_commands: Interactive slash commands (/help, /cost, /plan, etc.)
|
||||
- autonomy_mode: Autonomy levels (suggest, auto-edit, full-auto)
|
||||
- cost_tracker: Real-time cost and token tracking
|
||||
- message_queue: Message queuing while agent is processing
|
||||
- at_mentions: @ mention autocomplete for files and directories
|
||||
- compare: CLI command comparison feature
|
||||
- eval: Agent evaluation framework (accuracy, performance, reliability, criteria)
|
||||
"""
|
||||
|
||||
# Type hints available for IDE support
|
||||
@@ -43,6 +51,26 @@ __all__ = [
|
||||
'FlowDisplayHandler',
|
||||
'WorkflowHandler',
|
||||
'N8nHandler',
|
||||
'ExternalAgentsHandler',
|
||||
# New CLI enhancement features
|
||||
'SlashCommandHandler',
|
||||
'AutonomyModeHandler',
|
||||
'CostTrackerHandler',
|
||||
'RepoMapHandler',
|
||||
'InteractiveTUIHandler',
|
||||
'GitIntegrationHandler',
|
||||
'SandboxExecutorHandler',
|
||||
'MessageQueueHandler',
|
||||
# @ mention autocomplete
|
||||
'AtMentionCompleter',
|
||||
'CombinedCompleter',
|
||||
'FileSearchService',
|
||||
# Evaluation
|
||||
'EvalHandler',
|
||||
# Compare feature
|
||||
'CompareHandler',
|
||||
# Agent Skills
|
||||
'SkillsHandler',
|
||||
]
|
||||
|
||||
def __getattr__(name):
|
||||
@@ -95,4 +123,51 @@ def __getattr__(name):
|
||||
elif name == 'N8nHandler':
|
||||
from .n8n import N8nHandler
|
||||
return N8nHandler
|
||||
elif name == 'ExternalAgentsHandler':
|
||||
from .external_agents import ExternalAgentsHandler
|
||||
return ExternalAgentsHandler
|
||||
# New CLI enhancement features
|
||||
elif name == 'SlashCommandHandler':
|
||||
from .slash_commands import SlashCommandHandler
|
||||
return SlashCommandHandler
|
||||
elif name == 'AutonomyModeHandler':
|
||||
from .autonomy_mode import AutonomyModeHandler
|
||||
return AutonomyModeHandler
|
||||
elif name == 'CostTrackerHandler':
|
||||
from .cost_tracker import CostTrackerHandler
|
||||
return CostTrackerHandler
|
||||
elif name == 'RepoMapHandler':
|
||||
from .repo_map import RepoMapHandler
|
||||
return RepoMapHandler
|
||||
elif name == 'InteractiveTUIHandler':
|
||||
from .interactive_tui import InteractiveTUIHandler
|
||||
return InteractiveTUIHandler
|
||||
elif name == 'GitIntegrationHandler':
|
||||
from .git_integration import GitIntegrationHandler
|
||||
return GitIntegrationHandler
|
||||
elif name == 'SandboxExecutorHandler':
|
||||
from .sandbox_executor import SandboxExecutorHandler
|
||||
return SandboxExecutorHandler
|
||||
elif name == 'MessageQueueHandler':
|
||||
from .message_queue import MessageQueueHandler
|
||||
return MessageQueueHandler
|
||||
# @ mention autocomplete
|
||||
elif name == 'AtMentionCompleter':
|
||||
from .at_mentions import AtMentionCompleter
|
||||
return AtMentionCompleter
|
||||
elif name == 'CombinedCompleter':
|
||||
from .at_mentions import CombinedCompleter
|
||||
return CombinedCompleter
|
||||
elif name == 'FileSearchService':
|
||||
from .at_mentions import FileSearchService
|
||||
return FileSearchService
|
||||
elif name == 'CompareHandler':
|
||||
from .compare import CompareHandler
|
||||
return CompareHandler
|
||||
elif name == 'EvalHandler':
|
||||
from .eval import EvalHandler
|
||||
return EvalHandler
|
||||
elif name == 'SkillsHandler':
|
||||
from .skills import SkillsHandler
|
||||
return SkillsHandler
|
||||
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
||||
@@ -0,0 +1,734 @@
|
||||
"""
|
||||
Agent Scheduler CLI Feature.
|
||||
|
||||
Provides CLI commands for scheduling agents to run 24/7 at regular intervals.
|
||||
"""
|
||||
|
||||
import logging
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class AgentSchedulerHandler:
|
||||
"""Handler for agent scheduler CLI commands."""
|
||||
|
||||
@staticmethod
|
||||
def handle_daemon_command(subcommand: str, args, unknown_args=None) -> int:
|
||||
"""
|
||||
Handle daemon management commands (start, list, stop, logs, restart).
|
||||
|
||||
Args:
|
||||
subcommand: Command to execute (start, list, stop, logs, restart)
|
||||
args: Parsed command-line arguments
|
||||
unknown_args: Additional arguments
|
||||
|
||||
Returns:
|
||||
Exit code
|
||||
"""
|
||||
from praisonai.scheduler.state_manager import SchedulerStateManager
|
||||
from praisonai.scheduler.daemon_manager import DaemonManager
|
||||
|
||||
state_manager = SchedulerStateManager()
|
||||
daemon_manager = DaemonManager()
|
||||
|
||||
if subcommand == "start":
|
||||
return AgentSchedulerHandler._handle_start(args, unknown_args, state_manager, daemon_manager)
|
||||
elif subcommand == "list":
|
||||
return AgentSchedulerHandler._handle_list(state_manager)
|
||||
elif subcommand == "stop":
|
||||
return AgentSchedulerHandler._handle_stop(unknown_args, state_manager, daemon_manager)
|
||||
elif subcommand == "logs":
|
||||
return AgentSchedulerHandler._handle_logs(unknown_args, daemon_manager)
|
||||
elif subcommand == "restart":
|
||||
return AgentSchedulerHandler._handle_restart(unknown_args, state_manager, daemon_manager)
|
||||
elif subcommand == "delete":
|
||||
return AgentSchedulerHandler._handle_delete(unknown_args, state_manager)
|
||||
elif subcommand == "describe":
|
||||
return AgentSchedulerHandler._handle_describe(unknown_args, state_manager, daemon_manager)
|
||||
elif subcommand == "save":
|
||||
return AgentSchedulerHandler._handle_save(unknown_args, state_manager)
|
||||
elif subcommand == "stop-all":
|
||||
return AgentSchedulerHandler._handle_stop_all(state_manager, daemon_manager)
|
||||
elif subcommand == "stats":
|
||||
return AgentSchedulerHandler._handle_stats(state_manager, unknown_args, daemon_manager)
|
||||
else:
|
||||
print(f"❌ Unknown subcommand: {subcommand}")
|
||||
print("\nAvailable commands:")
|
||||
print(" start, list, stop, logs, restart, delete, describe, save, stop-all, stats")
|
||||
return 1
|
||||
|
||||
@staticmethod
|
||||
def _handle_start(args, unknown_args, state_manager, daemon_manager) -> int:
|
||||
"""Handle 'schedule start' command."""
|
||||
from datetime import datetime
|
||||
|
||||
if not unknown_args:
|
||||
print("❌ Error: Please provide scheduler name and task")
|
||||
print("\nUsage:")
|
||||
print(' praisonai schedule start <name> "Your task" --interval hourly')
|
||||
print("\nExample:")
|
||||
print(' praisonai schedule start news-checker "Check AI news" --interval hourly')
|
||||
return 1
|
||||
|
||||
# Parse arguments
|
||||
name = unknown_args[0]
|
||||
task = unknown_args[1] if len(unknown_args) > 1 else None
|
||||
|
||||
if not task:
|
||||
print("❌ Error: Please provide a task")
|
||||
return 1
|
||||
|
||||
# Get options
|
||||
interval = getattr(args, 'schedule_interval', None) or 'hourly'
|
||||
max_retries = getattr(args, 'schedule_max_retries', None) or 3
|
||||
timeout = getattr(args, 'timeout', None)
|
||||
max_cost = getattr(args, 'max_cost', None)
|
||||
|
||||
# Check if name already exists
|
||||
existing = state_manager.load_state(name)
|
||||
if existing and daemon_manager.get_status(existing.get('pid', 0))['is_alive']:
|
||||
print(f"❌ Error: Scheduler '{name}' is already running (PID: {existing['pid']})")
|
||||
print(f" Use 'praisonai schedule stop {name}' to stop it first")
|
||||
return 1
|
||||
|
||||
# Start daemon
|
||||
print(f"🚀 Starting scheduler '{name}'...")
|
||||
print(f" Task: {task}")
|
||||
print(f" Interval: {interval}")
|
||||
if timeout:
|
||||
print(f" Timeout: {timeout}s")
|
||||
if max_cost:
|
||||
print(f" Budget: ${max_cost}")
|
||||
|
||||
pid = daemon_manager.start_scheduler_daemon(
|
||||
name=name,
|
||||
task=task,
|
||||
interval=interval,
|
||||
max_cost=max_cost,
|
||||
timeout=timeout,
|
||||
max_retries=max_retries
|
||||
)
|
||||
|
||||
# Save state
|
||||
state = {
|
||||
"name": name,
|
||||
"pid": pid,
|
||||
"task": task,
|
||||
"interval": interval,
|
||||
"timeout": timeout,
|
||||
"max_cost": max_cost,
|
||||
"max_retries": max_retries,
|
||||
"status": "running",
|
||||
"started_at": datetime.now().isoformat(),
|
||||
"executions": 0,
|
||||
"cost": 0.0
|
||||
}
|
||||
state_manager.save_state(name, state)
|
||||
|
||||
print(f"\n✅ Scheduler '{name}' started successfully!")
|
||||
print(f" PID: {pid}")
|
||||
print(f" Logs: ~/.praisonai/logs/{name}.log")
|
||||
print(f"\nManage:")
|
||||
print(f" praisonai schedule list")
|
||||
print(f" praisonai schedule describe {name}")
|
||||
print(f" praisonai schedule logs {name} --follow")
|
||||
print(f" praisonai schedule stop {name}")
|
||||
|
||||
return 0
|
||||
|
||||
@staticmethod
|
||||
def _handle_list(state_manager) -> int:
|
||||
"""Handle 'schedule list' command."""
|
||||
states = state_manager.list_all()
|
||||
|
||||
if not states:
|
||||
print("No schedulers running.")
|
||||
print("\nStart one with:")
|
||||
print(' praisonai schedule start <name> "Your task" --interval hourly')
|
||||
return 0
|
||||
|
||||
# Clean up dead processes
|
||||
state_manager.cleanup_dead_processes()
|
||||
states = state_manager.list_all()
|
||||
|
||||
# Display table
|
||||
print(f"\n{'Name':<20} {'Status':<10} {'PID':<8} {'Interval':<12} {'Task':<40}")
|
||||
print("=" * 90)
|
||||
|
||||
for state in states:
|
||||
name = state.get('name', 'unknown')[:20]
|
||||
pid = state.get('pid', 0)
|
||||
status = "running" if state_manager.is_process_alive(pid) else "stopped"
|
||||
interval = state.get('interval', 'unknown')[:12]
|
||||
task = state.get('task', '')[:40]
|
||||
|
||||
status_icon = "🟢" if status == "running" else "🔴"
|
||||
print(f"{name:<20} {status_icon} {status:<8} {pid:<8} {interval:<12} {task:<40}")
|
||||
|
||||
print(f"\nTotal: {len(states)} scheduler(s)")
|
||||
return 0
|
||||
|
||||
@staticmethod
|
||||
def _handle_stats(state_manager, unknown_args=None, daemon_manager=None) -> int:
|
||||
"""Handle 'schedule stats' command - show aggregate or individual statistics."""
|
||||
# If a name is provided, show individual stats (alias for describe)
|
||||
if unknown_args and len(unknown_args) > 0:
|
||||
name = unknown_args[0]
|
||||
return AgentSchedulerHandler._handle_describe([name], state_manager, daemon_manager)
|
||||
|
||||
# Otherwise show aggregate stats
|
||||
states = state_manager.list_all()
|
||||
|
||||
if not states:
|
||||
print("No schedulers found.")
|
||||
print("\nStart one with:")
|
||||
print(' praisonai schedule start <name> "Your task" --interval hourly')
|
||||
return 0
|
||||
|
||||
# Calculate aggregate stats
|
||||
total_schedulers = len(states)
|
||||
running = sum(1 for s in states if s.get('status') == 'running')
|
||||
stopped = total_schedulers - running
|
||||
total_executions = sum(s.get('executions', 0) for s in states)
|
||||
total_cost = sum(s.get('cost', 0.0) for s in states)
|
||||
|
||||
print(f"\n{'='*60}")
|
||||
print(f"📊 Aggregate Scheduler Statistics")
|
||||
print(f"{'='*60}")
|
||||
print(f"Total Schedulers: {total_schedulers}")
|
||||
print(f" 🟢 Running: {running}")
|
||||
print(f" 🔴 Stopped: {stopped}")
|
||||
print(f"\nTotal Executions: {total_executions}")
|
||||
print(f"Total Cost: ${total_cost:.4f}")
|
||||
|
||||
if total_executions > 0:
|
||||
avg_cost = total_cost / total_executions
|
||||
print(f"Avg Cost/Execution: ${avg_cost:.4f}")
|
||||
|
||||
print(f"{'='*60}\n")
|
||||
|
||||
# Show per-scheduler breakdown
|
||||
print("Per-Scheduler Breakdown:")
|
||||
print(f"{'Name':<20} {'Status':<10} {'Executions':<12} {'Cost':<12}")
|
||||
print("="*60)
|
||||
|
||||
for state in sorted(states, key=lambda x: x.get('executions', 0), reverse=True):
|
||||
name = state['name'][:20]
|
||||
status = state.get('status', 'unknown')
|
||||
executions = state.get('executions', 0)
|
||||
cost = state.get('cost', 0.0)
|
||||
|
||||
status_icon = "🟢" if status == "running" else "🔴"
|
||||
print(f"{name:<20} {status_icon} {status:<8} {executions:<12} ${cost:<11.4f}")
|
||||
|
||||
return 0
|
||||
|
||||
@staticmethod
|
||||
def _handle_stop_all(state_manager, daemon_manager) -> int:
|
||||
"""Handle 'schedule stop-all' command."""
|
||||
states = state_manager.list_all()
|
||||
|
||||
if not states:
|
||||
print("No schedulers running.")
|
||||
return 0
|
||||
|
||||
print(f"\n🛑 Stopping all schedulers ({len(states)} total)...\n")
|
||||
|
||||
stopped = 0
|
||||
failed = 0
|
||||
|
||||
for state in states:
|
||||
name = state['name']
|
||||
pid = state['pid']
|
||||
|
||||
try:
|
||||
if daemon_manager.stop_daemon(pid):
|
||||
state_manager.delete_state(name)
|
||||
print(f"✅ Stopped '{name}' (PID: {pid})")
|
||||
stopped += 1
|
||||
else:
|
||||
print(f"❌ Failed to stop '{name}' (PID: {pid})")
|
||||
failed += 1
|
||||
except Exception as e:
|
||||
print(f"❌ Error stopping '{name}': {e}")
|
||||
failed += 1
|
||||
|
||||
print(f"\n📊 Summary: {stopped} stopped, {failed} failed")
|
||||
return 0 if failed == 0 else 1
|
||||
|
||||
@staticmethod
|
||||
def _handle_stop(unknown_args, state_manager, daemon_manager) -> int:
|
||||
"""Handle 'schedule stop' command."""
|
||||
if not unknown_args:
|
||||
print("❌ Error: Please provide scheduler name")
|
||||
print("\nUsage: praisonai schedule stop <name>")
|
||||
return 1
|
||||
|
||||
name = unknown_args[0]
|
||||
state = state_manager.load_state(name)
|
||||
|
||||
if not state:
|
||||
print(f"❌ Error: Scheduler '{name}' not found")
|
||||
print("\nList schedulers with: praisonai schedule list")
|
||||
return 1
|
||||
|
||||
pid = state.get('pid')
|
||||
if not pid:
|
||||
print(f"❌ Error: No PID found for scheduler '{name}'")
|
||||
return 1
|
||||
|
||||
print(f"🛑 Stopping scheduler '{name}' (PID: {pid})...")
|
||||
|
||||
success = daemon_manager.stop_daemon(pid)
|
||||
|
||||
if success:
|
||||
state['status'] = 'stopped'
|
||||
state_manager.save_state(name, state)
|
||||
print(f"✅ Scheduler '{name}' stopped successfully")
|
||||
return 0
|
||||
else:
|
||||
print(f"❌ Failed to stop scheduler '{name}'")
|
||||
return 1
|
||||
|
||||
@staticmethod
|
||||
def _handle_logs(unknown_args, daemon_manager) -> int:
|
||||
"""Handle 'schedule logs' command."""
|
||||
if not unknown_args:
|
||||
print("❌ Error: Please provide scheduler name")
|
||||
print("\nUsage: praisonai schedule logs <name> [-f]")
|
||||
return 1
|
||||
|
||||
name = unknown_args[0]
|
||||
follow = '-f' in unknown_args or '--follow' in unknown_args
|
||||
|
||||
if follow:
|
||||
print(f"📋 Following logs for '{name}' (Ctrl+C to stop)...")
|
||||
import subprocess
|
||||
log_file = daemon_manager.log_dir / f"{name}.log"
|
||||
if not log_file.exists():
|
||||
print(f"❌ Error: Log file not found for '{name}'")
|
||||
return 1
|
||||
|
||||
try:
|
||||
subprocess.run(['tail', '-f', str(log_file)])
|
||||
except KeyboardInterrupt:
|
||||
print("\n✅ Stopped following logs")
|
||||
return 0
|
||||
else:
|
||||
logs = daemon_manager.read_logs(name, lines=50)
|
||||
if logs:
|
||||
print(f"📋 Last 50 lines of logs for '{name}':\n")
|
||||
print(logs)
|
||||
return 0
|
||||
else:
|
||||
print(f"❌ Error: No logs found for '{name}'")
|
||||
return 1
|
||||
|
||||
@staticmethod
|
||||
def _handle_restart(unknown_args, state_manager, daemon_manager) -> int:
|
||||
"""Handle 'schedule restart' command."""
|
||||
from datetime import datetime
|
||||
import time
|
||||
|
||||
if not unknown_args:
|
||||
print("❌ Error: Please provide scheduler name")
|
||||
print("\nUsage: praisonai schedule restart <name>")
|
||||
return 1
|
||||
|
||||
name = unknown_args[0]
|
||||
state = state_manager.load_state(name)
|
||||
|
||||
if not state:
|
||||
print(f"❌ Error: Scheduler '{name}' not found")
|
||||
return 1
|
||||
|
||||
print(f"🔄 Restarting scheduler '{name}'...")
|
||||
|
||||
# Stop if running
|
||||
pid = state.get('pid')
|
||||
if pid and state_manager.is_process_alive(pid):
|
||||
daemon_manager.stop_daemon(pid)
|
||||
time.sleep(1)
|
||||
|
||||
# Start again
|
||||
new_pid = daemon_manager.start_scheduler_daemon(
|
||||
name=name,
|
||||
task=state['task'],
|
||||
interval=state['interval'],
|
||||
max_cost=state.get('max_cost'),
|
||||
timeout=state.get('timeout'),
|
||||
max_retries=state.get('max_retries', 3)
|
||||
)
|
||||
|
||||
state['pid'] = new_pid
|
||||
state['status'] = 'running'
|
||||
state['started_at'] = datetime.now().isoformat()
|
||||
state_manager.save_state(name, state)
|
||||
|
||||
print(f"✅ Scheduler '{name}' restarted (PID: {new_pid})")
|
||||
return 0
|
||||
|
||||
@staticmethod
|
||||
def _handle_delete(unknown_args, state_manager) -> int:
|
||||
"""Handle 'schedule delete' command."""
|
||||
if not unknown_args:
|
||||
print("❌ Error: Please provide scheduler name")
|
||||
print("\nUsage: praisonai schedule delete <name>")
|
||||
return 1
|
||||
|
||||
name = unknown_args[0]
|
||||
|
||||
if state_manager.delete_state(name):
|
||||
print(f"✅ Scheduler '{name}' deleted from list")
|
||||
return 0
|
||||
else:
|
||||
print(f"❌ Error: Scheduler '{name}' not found")
|
||||
return 1
|
||||
|
||||
@staticmethod
|
||||
def _handle_describe(unknown_args, state_manager, daemon_manager) -> int:
|
||||
"""Handle 'schedule describe' command."""
|
||||
from datetime import datetime
|
||||
|
||||
if not unknown_args:
|
||||
print("❌ Error: Please provide scheduler name")
|
||||
print("\nUsage: praisonai schedule describe <name>")
|
||||
return 1
|
||||
|
||||
name = unknown_args[0]
|
||||
state = state_manager.load_state(name)
|
||||
|
||||
if not state:
|
||||
print(f"❌ Error: Scheduler '{name}' not found")
|
||||
print("\nList schedulers with: praisonai schedule list")
|
||||
return 1
|
||||
|
||||
# Get process status
|
||||
pid = state.get('pid', 0)
|
||||
is_alive = state_manager.is_process_alive(pid)
|
||||
status = "🟢 running" if is_alive else "🔴 stopped"
|
||||
|
||||
# Calculate uptime
|
||||
started_at = state.get('started_at')
|
||||
uptime = "N/A"
|
||||
if started_at:
|
||||
try:
|
||||
start_time = datetime.fromisoformat(started_at)
|
||||
uptime_delta = datetime.now() - start_time
|
||||
hours = int(uptime_delta.total_seconds() // 3600)
|
||||
minutes = int((uptime_delta.total_seconds() % 3600) // 60)
|
||||
uptime = f"{hours}h {minutes}m"
|
||||
except:
|
||||
pass
|
||||
|
||||
# Display detailed info
|
||||
print(f"\n{'='*60}")
|
||||
print(f"📋 Scheduler Details: {name}")
|
||||
print(f"{'='*60}")
|
||||
print(f"Status: {status}")
|
||||
print(f"PID: {pid}")
|
||||
print(f"Uptime: {uptime}")
|
||||
print(f"Task: {state.get('task', 'N/A')}")
|
||||
print(f"Interval: {state.get('interval', 'N/A')}")
|
||||
print(f"Max Retries: {state.get('max_retries', 'N/A')}")
|
||||
|
||||
if state.get('timeout'):
|
||||
print(f"Timeout: {state['timeout']}s")
|
||||
|
||||
if state.get('max_cost'):
|
||||
print(f"Budget: ${state['max_cost']}")
|
||||
|
||||
print(f"Executions: {state.get('executions', 0)}")
|
||||
print(f"Total Cost: ${state.get('cost', 0.0):.4f}")
|
||||
print(f"Started: {started_at or 'N/A'}")
|
||||
|
||||
# Log file location
|
||||
log_file = daemon_manager.log_dir / f"{name}.log"
|
||||
print(f"Logs: {log_file}")
|
||||
|
||||
print(f"{'='*60}\n")
|
||||
|
||||
return 0
|
||||
|
||||
@staticmethod
|
||||
def _handle_save(unknown_args, state_manager) -> int:
|
||||
"""Handle 'schedule save' command."""
|
||||
import yaml
|
||||
|
||||
if not unknown_args:
|
||||
print("❌ Error: Please provide scheduler name")
|
||||
print("\nUsage: praisonai schedule save <name> [output.yaml]")
|
||||
return 1
|
||||
|
||||
name = unknown_args[0]
|
||||
output_file = unknown_args[1] if len(unknown_args) > 1 else f"{name}.yaml"
|
||||
|
||||
state = state_manager.load_state(name)
|
||||
|
||||
if not state:
|
||||
print(f"❌ Error: Scheduler '{name}' not found")
|
||||
return 1
|
||||
|
||||
# Create YAML config from state
|
||||
yaml_config = {
|
||||
'framework': 'praisonai',
|
||||
'agents': [{
|
||||
'name': name,
|
||||
'role': 'Task Executor',
|
||||
'goal': state.get('task', ''),
|
||||
'instructions': state.get('task', ''),
|
||||
'verbose': True
|
||||
}],
|
||||
'task': state.get('task', ''),
|
||||
'schedule': {
|
||||
'interval': state.get('interval', 'hourly'),
|
||||
'max_retries': state.get('max_retries', 3),
|
||||
'run_immediately': True
|
||||
}
|
||||
}
|
||||
|
||||
# Add optional fields
|
||||
if state.get('timeout'):
|
||||
yaml_config['schedule']['timeout'] = state['timeout']
|
||||
|
||||
if state.get('max_cost'):
|
||||
yaml_config['schedule']['max_cost'] = state['max_cost']
|
||||
|
||||
# Write to file
|
||||
try:
|
||||
with open(output_file, 'w') as f:
|
||||
yaml.dump(yaml_config, f, default_flow_style=False, sort_keys=False)
|
||||
|
||||
print(f"✅ Configuration saved to: {output_file}")
|
||||
print(f"\nRun with:")
|
||||
print(f" praisonai schedule {output_file}")
|
||||
return 0
|
||||
except Exception as e:
|
||||
print(f"❌ Error saving configuration: {e}")
|
||||
return 1
|
||||
|
||||
@staticmethod
|
||||
def handle_schedule_command(args, unknown_args, daemon_mode=False) -> int:
|
||||
"""
|
||||
Handle the schedule command for running agents periodically.
|
||||
|
||||
Supports two modes:
|
||||
1. YAML mode: praisonai schedule agents.yaml
|
||||
2. Prompt mode: praisonai schedule "Your task here" --interval hourly
|
||||
|
||||
Args:
|
||||
args: Parsed command-line arguments
|
||||
unknown_args: Additional arguments after the command
|
||||
|
||||
Returns:
|
||||
Exit code (0 for success, 1 for error)
|
||||
"""
|
||||
try:
|
||||
from praisonai.scheduler import AgentScheduler
|
||||
from praisonaiagents import Agent
|
||||
except ImportError:
|
||||
print("Error: praisonai.scheduler or praisonaiagents module not found")
|
||||
print("Please ensure PraisonAI is properly installed")
|
||||
print("pip install praisonai praisonaiagents")
|
||||
return 1
|
||||
|
||||
# Check if first arg is a YAML file or a prompt
|
||||
first_arg = unknown_args[0] if unknown_args else None
|
||||
is_yaml_mode = first_arg and (first_arg.endswith('.yaml') or first_arg.endswith('.yml'))
|
||||
|
||||
# Get overrides from CLI
|
||||
interval_override = getattr(args, 'schedule_interval', None) or 'hourly'
|
||||
max_retries_override = getattr(args, 'schedule_max_retries', None) or 3
|
||||
timeout_override = getattr(args, 'timeout', None)
|
||||
max_cost_override = getattr(args, 'max_cost', None)
|
||||
verbose = getattr(args, 'verbose', False)
|
||||
|
||||
# Set up logging - only show logs if verbose
|
||||
if verbose:
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format='[%(asctime)s] %(levelname)s - %(message)s',
|
||||
datefmt='%Y-%m-%d %H:%M:%S'
|
||||
)
|
||||
else:
|
||||
# Suppress scheduler logs in non-verbose mode
|
||||
logging.getLogger('praisonai.scheduler').setLevel(logging.WARNING)
|
||||
|
||||
try:
|
||||
if is_yaml_mode:
|
||||
# YAML mode: Load from agents.yaml
|
||||
yaml_path = first_arg
|
||||
print(f"🤖 Loading agent configuration from: {yaml_path}")
|
||||
scheduler = AgentScheduler.from_yaml(
|
||||
yaml_path=yaml_path,
|
||||
interval_override=interval_override,
|
||||
max_retries_override=max_retries_override,
|
||||
timeout_override=timeout_override,
|
||||
max_cost_override=max_cost_override
|
||||
)
|
||||
else:
|
||||
# Prompt mode: Create agent from direct prompt
|
||||
if not first_arg:
|
||||
print("❌ Error: Please provide either a YAML file or a task prompt")
|
||||
print("\nExamples:")
|
||||
print(" praisonai schedule agents.yaml")
|
||||
print(' praisonai schedule "Check news every hour" --interval hourly')
|
||||
return 1
|
||||
|
||||
task = first_arg
|
||||
|
||||
# Create agent
|
||||
agent = Agent(
|
||||
name="Scheduled Agent",
|
||||
role="Task Executor",
|
||||
goal=task,
|
||||
instructions=task,
|
||||
verbose=True # Enable verbose to see output in logs
|
||||
)
|
||||
|
||||
# Create scheduler
|
||||
scheduler = AgentScheduler(
|
||||
agent=agent,
|
||||
task=task,
|
||||
timeout=timeout_override,
|
||||
max_cost=max_cost_override
|
||||
)
|
||||
|
||||
# Get configuration
|
||||
interval = interval_override
|
||||
max_retries = max_retries_override
|
||||
|
||||
if is_yaml_mode:
|
||||
schedule_config = scheduler._yaml_schedule_config
|
||||
interval = interval_override or schedule_config.get('interval', 'hourly')
|
||||
max_retries = max_retries_override or schedule_config.get('max_retries', 3)
|
||||
|
||||
print(f"Schedule: {interval}")
|
||||
print(f"Max Retries: {max_retries}")
|
||||
if scheduler.timeout:
|
||||
print(f"Timeout: {scheduler.timeout}s")
|
||||
if scheduler.max_cost:
|
||||
print(f"Budget: ${scheduler.max_cost}")
|
||||
print(f"{'='*60}\n")
|
||||
|
||||
# Start scheduler
|
||||
if is_yaml_mode:
|
||||
scheduler.start_from_yaml_config()
|
||||
else:
|
||||
scheduler.start(schedule_expr=interval, max_retries=max_retries, run_immediately=True)
|
||||
|
||||
# If daemon mode, block to keep process alive
|
||||
if daemon_mode:
|
||||
import time
|
||||
try:
|
||||
while scheduler.is_running:
|
||||
time.sleep(1)
|
||||
except KeyboardInterrupt:
|
||||
scheduler.stop()
|
||||
else:
|
||||
# Foreground mode - wait for Ctrl+C
|
||||
print("⏰ Starting scheduler... (Press Ctrl+C to stop)\n")
|
||||
import time
|
||||
try:
|
||||
while scheduler.is_running:
|
||||
time.sleep(1)
|
||||
except KeyboardInterrupt:
|
||||
print("\n\n🛑 Stopping scheduler...")
|
||||
scheduler.stop()
|
||||
|
||||
# Display final stats
|
||||
stats = scheduler.get_stats()
|
||||
print("\n📊 Final Statistics:")
|
||||
print(f" Total Executions: {stats['total_executions']}")
|
||||
print(f" Successful: {stats['successful_executions']}")
|
||||
print(f" Failed: {stats['failed_executions']}")
|
||||
print(f" Success Rate: {stats['success_rate']:.1f}%")
|
||||
print("\n✅ Agent stopped successfully\n")
|
||||
|
||||
return 0
|
||||
|
||||
except FileNotFoundError as e:
|
||||
print(f"❌ Error: {e}")
|
||||
print(f"\nMake sure {yaml_path} exists in the current directory.")
|
||||
print("\nExample agents.yaml structure:")
|
||||
print("""
|
||||
framework: praisonai
|
||||
|
||||
agents:
|
||||
- name: "AI News Monitor"
|
||||
role: "News Analyst"
|
||||
instructions: "Search and summarize AI news"
|
||||
tools:
|
||||
- search_tool
|
||||
|
||||
task: "Search for latest AI news"
|
||||
|
||||
schedule:
|
||||
interval: "hourly"
|
||||
max_retries: 3
|
||||
run_immediately: true
|
||||
""")
|
||||
return 1
|
||||
|
||||
except ValueError as e:
|
||||
print(f"❌ Configuration Error: {e}")
|
||||
return 1
|
||||
|
||||
except Exception as e:
|
||||
print(f"❌ Error: {e}")
|
||||
if verbose:
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
return 1
|
||||
|
||||
@staticmethod
|
||||
def add_schedule_arguments(subparsers):
|
||||
"""
|
||||
Add schedule subcommand and arguments to parser.
|
||||
|
||||
Args:
|
||||
subparsers: Subparsers object from argparse
|
||||
"""
|
||||
schedule_parser = subparsers.add_parser(
|
||||
'schedule',
|
||||
help='Schedule an agent to run continuously at regular intervals'
|
||||
)
|
||||
|
||||
schedule_parser.add_argument(
|
||||
'schedule_yaml',
|
||||
nargs='?',
|
||||
default='agents.yaml',
|
||||
help='Path to agents.yaml file (default: agents.yaml)'
|
||||
)
|
||||
|
||||
schedule_parser.add_argument(
|
||||
'--interval',
|
||||
dest='schedule_interval',
|
||||
type=str,
|
||||
help='Override schedule interval (e.g., "hourly", "*/30m", "daily")'
|
||||
)
|
||||
|
||||
schedule_parser.add_argument(
|
||||
'--max-retries',
|
||||
dest='schedule_max_retries',
|
||||
type=int,
|
||||
help='Override maximum retry attempts (default: from YAML or 3)'
|
||||
)
|
||||
|
||||
schedule_parser.add_argument(
|
||||
'--verbose', '-v',
|
||||
action='store_true',
|
||||
help='Enable verbose logging'
|
||||
)
|
||||
|
||||
schedule_parser.set_defaults(func=AgentSchedulerHandler.handle_schedule_command)
|
||||
|
||||
|
||||
def setup_scheduler_cli(subparsers):
|
||||
"""
|
||||
Setup scheduler CLI commands.
|
||||
|
||||
Args:
|
||||
subparsers: Subparsers object from argparse
|
||||
"""
|
||||
AgentSchedulerHandler.add_schedule_arguments(subparsers)
|
||||
@@ -0,0 +1,729 @@
|
||||
"""
|
||||
Agent Scheduler CLI Feature.
|
||||
|
||||
Provides CLI commands for scheduling agents to run 24/7 at regular intervals.
|
||||
"""
|
||||
|
||||
import logging
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class AgentSchedulerHandler:
|
||||
"""Handler for agent scheduler CLI commands."""
|
||||
|
||||
@staticmethod
|
||||
def handle_daemon_command(subcommand: str, args, unknown_args=None) -> int:
|
||||
"""
|
||||
Handle daemon management commands (start, list, stop, logs, restart).
|
||||
|
||||
Args:
|
||||
subcommand: Command to execute (start, list, stop, logs, restart)
|
||||
args: Parsed command-line arguments
|
||||
unknown_args: Additional arguments
|
||||
|
||||
Returns:
|
||||
Exit code
|
||||
"""
|
||||
from praisonai.scheduler.state_manager import SchedulerStateManager
|
||||
from praisonai.scheduler.daemon_manager import DaemonManager
|
||||
|
||||
state_manager = SchedulerStateManager()
|
||||
daemon_manager = DaemonManager()
|
||||
|
||||
if subcommand == "start":
|
||||
return AgentSchedulerHandler._handle_start(args, unknown_args, state_manager, daemon_manager)
|
||||
elif subcommand == "list":
|
||||
return AgentSchedulerHandler._handle_list(state_manager)
|
||||
elif subcommand == "stop":
|
||||
return AgentSchedulerHandler._handle_stop(unknown_args, state_manager, daemon_manager)
|
||||
elif subcommand == "logs":
|
||||
return AgentSchedulerHandler._handle_logs(unknown_args, daemon_manager)
|
||||
elif subcommand == "restart":
|
||||
return AgentSchedulerHandler._handle_restart(unknown_args, state_manager, daemon_manager)
|
||||
elif subcommand == "delete":
|
||||
return AgentSchedulerHandler._handle_delete(unknown_args, state_manager)
|
||||
elif subcommand == "describe":
|
||||
return AgentSchedulerHandler._handle_describe(unknown_args, state_manager, daemon_manager)
|
||||
elif subcommand == "save":
|
||||
return AgentSchedulerHandler._handle_save(unknown_args, state_manager)
|
||||
elif subcommand == "stop-all":
|
||||
return AgentSchedulerHandler._handle_stop_all(state_manager, daemon_manager)
|
||||
elif subcommand == "stats":
|
||||
return AgentSchedulerHandler._handle_stats(state_manager, unknown_args, daemon_manager)
|
||||
else:
|
||||
print(f"❌ Unknown subcommand: {subcommand}")
|
||||
print("\nAvailable commands:")
|
||||
print(" start, list, stop, logs, restart, delete, describe, save, stop-all, stats")
|
||||
return 1
|
||||
|
||||
@staticmethod
|
||||
def _handle_start(args, unknown_args, state_manager, daemon_manager) -> int:
|
||||
"""Handle 'schedule start' command."""
|
||||
from datetime import datetime
|
||||
|
||||
if not unknown_args:
|
||||
print("❌ Error: Please provide scheduler name and task")
|
||||
print("\nUsage:")
|
||||
print(' praisonai schedule start <name> "Your task" --interval hourly')
|
||||
print("\nExample:")
|
||||
print(' praisonai schedule start news-checker "Check AI news" --interval hourly')
|
||||
return 1
|
||||
|
||||
# Parse arguments
|
||||
name = unknown_args[0]
|
||||
task = unknown_args[1] if len(unknown_args) > 1 else None
|
||||
|
||||
if not task:
|
||||
print("❌ Error: Please provide a task")
|
||||
return 1
|
||||
|
||||
# Get options
|
||||
interval = getattr(args, 'schedule_interval', None) or 'hourly'
|
||||
max_retries = getattr(args, 'schedule_max_retries', None) or 3
|
||||
timeout = getattr(args, 'timeout', None)
|
||||
max_cost = getattr(args, 'max_cost', None)
|
||||
|
||||
# Check if name already exists
|
||||
existing = state_manager.load_state(name)
|
||||
if existing and daemon_manager.get_status(existing.get('pid', 0))['is_alive']:
|
||||
print(f"❌ Error: Scheduler '{name}' is already running (PID: {existing['pid']})")
|
||||
print(f" Use 'praisonai schedule stop {name}' to stop it first")
|
||||
return 1
|
||||
|
||||
# Start daemon
|
||||
print(f"🚀 Starting scheduler '{name}'...")
|
||||
print(f" Task: {task}")
|
||||
print(f" Interval: {interval}")
|
||||
if timeout:
|
||||
print(f" Timeout: {timeout}s")
|
||||
if max_cost:
|
||||
print(f" Budget: ${max_cost}")
|
||||
|
||||
pid = daemon_manager.start_scheduler_daemon(
|
||||
name=name,
|
||||
task=task,
|
||||
interval=interval,
|
||||
max_cost=max_cost,
|
||||
timeout=timeout,
|
||||
max_retries=max_retries
|
||||
)
|
||||
|
||||
# Save state
|
||||
state = {
|
||||
"name": name,
|
||||
"pid": pid,
|
||||
"task": task,
|
||||
"interval": interval,
|
||||
"timeout": timeout,
|
||||
"max_cost": max_cost,
|
||||
"max_retries": max_retries,
|
||||
"status": "running",
|
||||
"started_at": datetime.now().isoformat(),
|
||||
"executions": 0,
|
||||
"cost": 0.0
|
||||
}
|
||||
state_manager.save_state(name, state)
|
||||
|
||||
print(f"\n✅ Scheduler '{name}' started successfully!")
|
||||
print(f" PID: {pid}")
|
||||
print(f" Logs: ~/.praisonai/logs/{name}.log")
|
||||
print(f"\nManage:")
|
||||
print(f" praisonai schedule list")
|
||||
print(f" praisonai schedule describe {name}")
|
||||
print(f" praisonai schedule logs {name} --follow")
|
||||
print(f" praisonai schedule stop {name}")
|
||||
|
||||
return 0
|
||||
|
||||
@staticmethod
|
||||
def _handle_list(state_manager) -> int:
|
||||
"""Handle 'schedule list' command."""
|
||||
states = state_manager.list_all()
|
||||
|
||||
if not states:
|
||||
print("No schedulers running.")
|
||||
print("\nStart one with:")
|
||||
print(' praisonai schedule start <name> "Your task" --interval hourly')
|
||||
return 0
|
||||
|
||||
# Clean up dead processes
|
||||
state_manager.cleanup_dead_processes()
|
||||
states = state_manager.list_all()
|
||||
|
||||
# Display table
|
||||
print(f"\n{'Name':<20} {'Status':<10} {'PID':<8} {'Interval':<12} {'Task':<40}")
|
||||
print("=" * 90)
|
||||
|
||||
for state in states:
|
||||
name = state.get('name', 'unknown')[:20]
|
||||
pid = state.get('pid', 0)
|
||||
status = "running" if state_manager.is_process_alive(pid) else "stopped"
|
||||
interval = state.get('interval', 'unknown')[:12]
|
||||
task = state.get('task', '')[:40]
|
||||
|
||||
status_icon = "🟢" if status == "running" else "🔴"
|
||||
print(f"{name:<20} {status_icon} {status:<8} {pid:<8} {interval:<12} {task:<40}")
|
||||
|
||||
print(f"\nTotal: {len(states)} scheduler(s)")
|
||||
return 0
|
||||
|
||||
@staticmethod
|
||||
def _handle_stats(state_manager, unknown_args=None, daemon_manager=None) -> int:
|
||||
"""Handle 'schedule stats' command - show aggregate or individual statistics."""
|
||||
# If a name is provided, show individual stats (alias for describe)
|
||||
if unknown_args and len(unknown_args) > 0:
|
||||
name = unknown_args[0]
|
||||
return AgentSchedulerHandler._handle_describe([name], state_manager, None)
|
||||
|
||||
# Otherwise show aggregate stats
|
||||
states = state_manager.list_all()
|
||||
|
||||
if not states:
|
||||
print("No schedulers found.")
|
||||
print("\nStart one with:")
|
||||
print(' praisonai schedule start <name> "Your task" --interval hourly')
|
||||
return 0
|
||||
|
||||
# Calculate aggregate stats
|
||||
total_schedulers = len(states)
|
||||
running = sum(1 for s in states if s.get('status') == 'running')
|
||||
stopped = total_schedulers - running
|
||||
total_executions = sum(s.get('executions', 0) for s in states)
|
||||
total_cost = sum(s.get('cost', 0.0) for s in states)
|
||||
|
||||
print(f"\n{'='*60}")
|
||||
print(f"📊 Aggregate Scheduler Statistics")
|
||||
print(f"{'='*60}")
|
||||
print(f"Total Schedulers: {total_schedulers}")
|
||||
print(f" 🟢 Running: {running}")
|
||||
print(f" 🔴 Stopped: {stopped}")
|
||||
print(f"\nTotal Executions: {total_executions}")
|
||||
print(f"Total Cost: ${total_cost:.4f}")
|
||||
|
||||
if total_executions > 0:
|
||||
avg_cost = total_cost / total_executions
|
||||
print(f"Avg Cost/Execution: ${avg_cost:.4f}")
|
||||
|
||||
print(f"{'='*60}\n")
|
||||
|
||||
# Show per-scheduler breakdown
|
||||
print("Per-Scheduler Breakdown:")
|
||||
print(f"{'Name':<20} {'Status':<10} {'Executions':<12} {'Cost':<12}")
|
||||
print("="*60)
|
||||
|
||||
for state in sorted(states, key=lambda x: x.get('executions', 0), reverse=True):
|
||||
name = state['name'][:20]
|
||||
status = state.get('status', 'unknown')
|
||||
executions = state.get('executions', 0)
|
||||
cost = state.get('cost', 0.0)
|
||||
|
||||
status_icon = "🟢" if status == "running" else "🔴"
|
||||
print(f"{name:<20} {status_icon} {status:<8} {executions:<12} ${cost:<11.4f}")
|
||||
|
||||
return 0
|
||||
|
||||
@staticmethod
|
||||
def _handle_stop_all(state_manager, daemon_manager) -> int:
|
||||
"""Handle 'schedule stop-all' command."""
|
||||
states = state_manager.list_all()
|
||||
|
||||
if not states:
|
||||
print("No schedulers running.")
|
||||
return 0
|
||||
|
||||
print(f"\n🛑 Stopping all schedulers ({len(states)} total)...\n")
|
||||
|
||||
stopped = 0
|
||||
failed = 0
|
||||
|
||||
for state in states:
|
||||
name = state['name']
|
||||
pid = state['pid']
|
||||
|
||||
try:
|
||||
if daemon_manager.stop_daemon(pid):
|
||||
state_manager.delete_state(name)
|
||||
print(f"✅ Stopped '{name}' (PID: {pid})")
|
||||
stopped += 1
|
||||
else:
|
||||
print(f"❌ Failed to stop '{name}' (PID: {pid})")
|
||||
failed += 1
|
||||
except Exception as e:
|
||||
print(f"❌ Error stopping '{name}': {e}")
|
||||
failed += 1
|
||||
|
||||
print(f"\n📊 Summary: {stopped} stopped, {failed} failed")
|
||||
return 0 if failed == 0 else 1
|
||||
|
||||
@staticmethod
|
||||
def _handle_stop(unknown_args, state_manager, daemon_manager) -> int:
|
||||
"""Handle 'schedule stop' command."""
|
||||
if not unknown_args:
|
||||
print("❌ Error: Please provide scheduler name")
|
||||
print("\nUsage: praisonai schedule stop <name>")
|
||||
return 1
|
||||
|
||||
name = unknown_args[0]
|
||||
state = state_manager.load_state(name)
|
||||
|
||||
if not state:
|
||||
print(f"❌ Error: Scheduler '{name}' not found")
|
||||
print("\nList schedulers with: praisonai schedule list")
|
||||
return 1
|
||||
|
||||
pid = state.get('pid')
|
||||
if not pid:
|
||||
print(f"❌ Error: No PID found for scheduler '{name}'")
|
||||
return 1
|
||||
|
||||
print(f"🛑 Stopping scheduler '{name}' (PID: {pid})...")
|
||||
|
||||
success = daemon_manager.stop_daemon(pid)
|
||||
|
||||
if success:
|
||||
state['status'] = 'stopped'
|
||||
state_manager.save_state(name, state)
|
||||
print(f"✅ Scheduler '{name}' stopped successfully")
|
||||
return 0
|
||||
else:
|
||||
print(f"❌ Failed to stop scheduler '{name}'")
|
||||
return 1
|
||||
|
||||
@staticmethod
|
||||
def _handle_logs(unknown_args, daemon_manager) -> int:
|
||||
"""Handle 'schedule logs' command."""
|
||||
if not unknown_args:
|
||||
print("❌ Error: Please provide scheduler name")
|
||||
print("\nUsage: praisonai schedule logs <name> [-f]")
|
||||
return 1
|
||||
|
||||
name = unknown_args[0]
|
||||
follow = '-f' in unknown_args or '--follow' in unknown_args
|
||||
|
||||
if follow:
|
||||
print(f"📋 Following logs for '{name}' (Ctrl+C to stop)...")
|
||||
import subprocess
|
||||
log_file = daemon_manager.log_dir / f"{name}.log"
|
||||
if not log_file.exists():
|
||||
print(f"❌ Error: Log file not found for '{name}'")
|
||||
return 1
|
||||
|
||||
try:
|
||||
subprocess.run(['tail', '-f', str(log_file)])
|
||||
except KeyboardInterrupt:
|
||||
print("\n✅ Stopped following logs")
|
||||
return 0
|
||||
else:
|
||||
logs = daemon_manager.read_logs(name, lines=50)
|
||||
if logs:
|
||||
print(f"📋 Last 50 lines of logs for '{name}':\n")
|
||||
print(logs)
|
||||
return 0
|
||||
else:
|
||||
print(f"❌ Error: No logs found for '{name}'")
|
||||
return 1
|
||||
|
||||
@staticmethod
|
||||
def _handle_restart(unknown_args, state_manager, daemon_manager) -> int:
|
||||
"""Handle 'schedule restart' command."""
|
||||
from datetime import datetime
|
||||
import time
|
||||
|
||||
if not unknown_args:
|
||||
print("❌ Error: Please provide scheduler name")
|
||||
print("\nUsage: praisonai schedule restart <name>")
|
||||
return 1
|
||||
|
||||
name = unknown_args[0]
|
||||
state = state_manager.load_state(name)
|
||||
|
||||
if not state:
|
||||
print(f"❌ Error: Scheduler '{name}' not found")
|
||||
return 1
|
||||
|
||||
print(f"🔄 Restarting scheduler '{name}'...")
|
||||
|
||||
# Stop if running
|
||||
pid = state.get('pid')
|
||||
if pid and state_manager.is_process_alive(pid):
|
||||
daemon_manager.stop_daemon(pid)
|
||||
time.sleep(1)
|
||||
|
||||
# Start again
|
||||
new_pid = daemon_manager.start_scheduler_daemon(
|
||||
name=name,
|
||||
task=state['task'],
|
||||
interval=state['interval'],
|
||||
max_cost=state.get('max_cost'),
|
||||
timeout=state.get('timeout'),
|
||||
max_retries=state.get('max_retries', 3)
|
||||
)
|
||||
|
||||
state['pid'] = new_pid
|
||||
state['status'] = 'running'
|
||||
state['started_at'] = datetime.now().isoformat()
|
||||
state_manager.save_state(name, state)
|
||||
|
||||
print(f"✅ Scheduler '{name}' restarted (PID: {new_pid})")
|
||||
return 0
|
||||
|
||||
@staticmethod
|
||||
def _handle_delete(unknown_args, state_manager) -> int:
|
||||
"""Handle 'schedule delete' command."""
|
||||
if not unknown_args:
|
||||
print("❌ Error: Please provide scheduler name")
|
||||
print("\nUsage: praisonai schedule delete <name>")
|
||||
return 1
|
||||
|
||||
name = unknown_args[0]
|
||||
|
||||
if state_manager.delete_state(name):
|
||||
print(f"✅ Scheduler '{name}' deleted from list")
|
||||
return 0
|
||||
else:
|
||||
print(f"❌ Error: Scheduler '{name}' not found")
|
||||
return 1
|
||||
|
||||
@staticmethod
|
||||
def _handle_describe(unknown_args, state_manager, daemon_manager) -> int:
|
||||
"""Handle 'schedule describe' command."""
|
||||
from datetime import datetime
|
||||
|
||||
if not unknown_args:
|
||||
print("❌ Error: Please provide scheduler name")
|
||||
print("\nUsage: praisonai schedule describe <name>")
|
||||
return 1
|
||||
|
||||
name = unknown_args[0]
|
||||
state = state_manager.load_state(name)
|
||||
|
||||
if not state:
|
||||
print(f"❌ Error: Scheduler '{name}' not found")
|
||||
print("\nList schedulers with: praisonai schedule list")
|
||||
return 1
|
||||
|
||||
# Get process status
|
||||
pid = state.get('pid', 0)
|
||||
is_alive = state_manager.is_process_alive(pid)
|
||||
status = "🟢 running" if is_alive else "🔴 stopped"
|
||||
|
||||
# Calculate uptime
|
||||
started_at = state.get('started_at')
|
||||
uptime = "N/A"
|
||||
if started_at:
|
||||
try:
|
||||
start_time = datetime.fromisoformat(started_at)
|
||||
uptime_delta = datetime.now() - start_time
|
||||
hours = int(uptime_delta.total_seconds() // 3600)
|
||||
minutes = int((uptime_delta.total_seconds() % 3600) // 60)
|
||||
uptime = f"{hours}h {minutes}m"
|
||||
except:
|
||||
pass
|
||||
|
||||
# Display detailed info
|
||||
print(f"\n{'='*60}")
|
||||
print(f"📋 Scheduler Details: {name}")
|
||||
print(f"{'='*60}")
|
||||
print(f"Status: {status}")
|
||||
print(f"PID: {pid}")
|
||||
print(f"Uptime: {uptime}")
|
||||
print(f"Task: {state.get('task', 'N/A')}")
|
||||
print(f"Interval: {state.get('interval', 'N/A')}")
|
||||
print(f"Max Retries: {state.get('max_retries', 'N/A')}")
|
||||
|
||||
if state.get('timeout'):
|
||||
print(f"Timeout: {state['timeout']}s")
|
||||
|
||||
if state.get('max_cost'):
|
||||
print(f"Budget: ${state['max_cost']}")
|
||||
|
||||
print(f"Executions: {state.get('executions', 0)}")
|
||||
print(f"Total Cost: ${state.get('cost', 0.0):.4f}")
|
||||
print(f"Started: {started_at or 'N/A'}")
|
||||
|
||||
# Log file location
|
||||
log_file = daemon_manager.log_dir / f"{name}.log"
|
||||
print(f"Logs: {log_file}")
|
||||
|
||||
print(f"{'='*60}\n")
|
||||
|
||||
return 0
|
||||
|
||||
@staticmethod
|
||||
def _handle_save(unknown_args, state_manager) -> int:
|
||||
"""Handle 'schedule save' command."""
|
||||
import yaml
|
||||
|
||||
if not unknown_args:
|
||||
print("❌ Error: Please provide scheduler name")
|
||||
print("\nUsage: praisonai schedule save <name> [output.yaml]")
|
||||
return 1
|
||||
|
||||
name = unknown_args[0]
|
||||
output_file = unknown_args[1] if len(unknown_args) > 1 else f"{name}.yaml"
|
||||
|
||||
state = state_manager.load_state(name)
|
||||
|
||||
if not state:
|
||||
print(f"❌ Error: Scheduler '{name}' not found")
|
||||
return 1
|
||||
|
||||
# Create YAML config from state
|
||||
yaml_config = {
|
||||
'framework': 'praisonai',
|
||||
'agents': [{
|
||||
'name': name,
|
||||
'role': 'Task Executor',
|
||||
'goal': state.get('task', ''),
|
||||
'instructions': state.get('task', ''),
|
||||
'verbose': True
|
||||
}],
|
||||
'task': state.get('task', ''),
|
||||
'schedule': {
|
||||
'interval': state.get('interval', 'hourly'),
|
||||
'max_retries': state.get('max_retries', 3),
|
||||
'run_immediately': True
|
||||
}
|
||||
}
|
||||
|
||||
# Add optional fields
|
||||
if state.get('timeout'):
|
||||
yaml_config['schedule']['timeout'] = state['timeout']
|
||||
|
||||
if state.get('max_cost'):
|
||||
yaml_config['schedule']['max_cost'] = state['max_cost']
|
||||
|
||||
# Write to file
|
||||
try:
|
||||
with open(output_file, 'w') as f:
|
||||
yaml.dump(yaml_config, f, default_flow_style=False, sort_keys=False)
|
||||
|
||||
print(f"✅ Configuration saved to: {output_file}")
|
||||
print(f"\nRun with:")
|
||||
print(f" praisonai schedule {output_file}")
|
||||
return 0
|
||||
except Exception as e:
|
||||
print(f"❌ Error saving configuration: {e}")
|
||||
return 1
|
||||
|
||||
@staticmethod
|
||||
def handle_schedule_command(args, unknown_args=None) -> int:
|
||||
"""
|
||||
Handle the schedule command for running agents periodically.
|
||||
|
||||
Supports two modes:
|
||||
1. YAML mode: praisonai schedule agents.yaml
|
||||
2. Prompt mode: praisonai schedule "Your task here" --interval hourly
|
||||
|
||||
Args:
|
||||
args: Parsed command-line arguments
|
||||
unknown_args: Additional arguments after the command
|
||||
|
||||
Returns:
|
||||
Exit code (0 for success, 1 for error)
|
||||
"""
|
||||
try:
|
||||
from praisonai.scheduler import AgentScheduler
|
||||
from praisonaiagents import Agent
|
||||
except ImportError:
|
||||
print("Error: praisonai.scheduler or praisonaiagents module not found")
|
||||
print("Please ensure PraisonAI is properly installed")
|
||||
print("pip install praisonai praisonaiagents")
|
||||
return 1
|
||||
|
||||
# Check if first arg is a YAML file or a prompt
|
||||
first_arg = unknown_args[0] if unknown_args else None
|
||||
is_yaml_mode = first_arg and (first_arg.endswith('.yaml') or first_arg.endswith('.yml'))
|
||||
|
||||
# Get overrides from CLI
|
||||
interval_override = getattr(args, 'schedule_interval', None) or 'hourly'
|
||||
max_retries_override = getattr(args, 'schedule_max_retries', None) or 3
|
||||
timeout_override = getattr(args, 'timeout', None)
|
||||
max_cost_override = getattr(args, 'max_cost', None)
|
||||
verbose = getattr(args, 'verbose', False)
|
||||
|
||||
# Set up logging - only show logs if verbose
|
||||
if verbose:
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format='[%(asctime)s] %(levelname)s - %(message)s',
|
||||
datefmt='%Y-%m-%d %H:%M:%S'
|
||||
)
|
||||
else:
|
||||
# Suppress scheduler logs in non-verbose mode
|
||||
logging.getLogger('praisonai.scheduler').setLevel(logging.WARNING)
|
||||
|
||||
try:
|
||||
if is_yaml_mode:
|
||||
# YAML mode: Load from agents.yaml
|
||||
yaml_path = first_arg
|
||||
print(f"🤖 Loading agent configuration from: {yaml_path}")
|
||||
scheduler = AgentScheduler.from_yaml(
|
||||
yaml_path=yaml_path,
|
||||
interval_override=interval_override,
|
||||
max_retries_override=max_retries_override,
|
||||
timeout_override=timeout_override,
|
||||
max_cost_override=max_cost_override
|
||||
)
|
||||
else:
|
||||
# Prompt mode: Create agent from direct prompt
|
||||
if not first_arg:
|
||||
print("❌ Error: Please provide either a YAML file or a task prompt")
|
||||
print("\nExamples:")
|
||||
print(" praisonai schedule agents.yaml")
|
||||
print(' praisonai schedule "Check news every hour" --interval hourly')
|
||||
return 1
|
||||
|
||||
task = first_arg
|
||||
|
||||
# Create agent
|
||||
agent = Agent(
|
||||
name="Scheduled Agent",
|
||||
role="Task Executor",
|
||||
goal=task,
|
||||
instructions=task,
|
||||
verbose=False
|
||||
)
|
||||
|
||||
# Create scheduler
|
||||
scheduler = AgentScheduler(
|
||||
agent=agent,
|
||||
task=task,
|
||||
timeout=timeout_override,
|
||||
max_cost=max_cost_override
|
||||
)
|
||||
|
||||
# Get configuration
|
||||
interval = interval_override
|
||||
max_retries = max_retries_override
|
||||
|
||||
if is_yaml_mode:
|
||||
schedule_config = scheduler._yaml_schedule_config
|
||||
interval = interval_override or schedule_config.get('interval', 'hourly')
|
||||
max_retries = max_retries_override or schedule_config.get('max_retries', 3)
|
||||
|
||||
print(f"Schedule: {interval}")
|
||||
print(f"Max Retries: {max_retries}")
|
||||
if scheduler.timeout:
|
||||
print(f"Timeout: {scheduler.timeout}s")
|
||||
if scheduler.max_cost:
|
||||
print(f"Budget: ${scheduler.max_cost}")
|
||||
print(f"{'='*60}\n")
|
||||
|
||||
# Start scheduler
|
||||
print("⏰ Starting scheduler... (Press Ctrl+C to stop)\n")
|
||||
if is_yaml_mode:
|
||||
scheduler.start_from_yaml_config()
|
||||
else:
|
||||
scheduler.start(
|
||||
schedule_expr=interval,
|
||||
max_retries=max_retries,
|
||||
run_immediately=True
|
||||
)
|
||||
|
||||
# Keep running until interrupted
|
||||
try:
|
||||
while scheduler.is_running:
|
||||
import time
|
||||
time.sleep(1)
|
||||
except KeyboardInterrupt:
|
||||
print("\n\n🛑 Stopping scheduler...")
|
||||
scheduler.stop()
|
||||
|
||||
# Display final stats
|
||||
stats = scheduler.get_stats()
|
||||
print("\n📊 Final Statistics:")
|
||||
print(f" Total Executions: {stats['total_executions']}")
|
||||
print(f" Successful: {stats['successful_executions']}")
|
||||
print(f" Failed: {stats['failed_executions']}")
|
||||
print(f" Success Rate: {stats['success_rate']:.1f}%")
|
||||
print("\n✅ Agent stopped successfully\n")
|
||||
|
||||
return 0
|
||||
|
||||
except FileNotFoundError as e:
|
||||
print(f"❌ Error: {e}")
|
||||
print(f"\nMake sure {yaml_path} exists in the current directory.")
|
||||
print("\nExample agents.yaml structure:")
|
||||
print("""
|
||||
framework: praisonai
|
||||
|
||||
agents:
|
||||
- name: "AI News Monitor"
|
||||
role: "News Analyst"
|
||||
instructions: "Search and summarize AI news"
|
||||
tools:
|
||||
- search_tool
|
||||
|
||||
task: "Search for latest AI news"
|
||||
|
||||
schedule:
|
||||
interval: "hourly"
|
||||
max_retries: 3
|
||||
run_immediately: true
|
||||
""")
|
||||
return 1
|
||||
|
||||
except ValueError as e:
|
||||
print(f"❌ Configuration Error: {e}")
|
||||
return 1
|
||||
|
||||
except Exception as e:
|
||||
print(f"❌ Error: {e}")
|
||||
if verbose:
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
return 1
|
||||
|
||||
@staticmethod
|
||||
def add_schedule_arguments(subparsers):
|
||||
"""
|
||||
Add schedule subcommand and arguments to parser.
|
||||
|
||||
Args:
|
||||
subparsers: Subparsers object from argparse
|
||||
"""
|
||||
schedule_parser = subparsers.add_parser(
|
||||
'schedule',
|
||||
help='Schedule an agent to run continuously at regular intervals'
|
||||
)
|
||||
|
||||
schedule_parser.add_argument(
|
||||
'schedule_yaml',
|
||||
nargs='?',
|
||||
default='agents.yaml',
|
||||
help='Path to agents.yaml file (default: agents.yaml)'
|
||||
)
|
||||
|
||||
schedule_parser.add_argument(
|
||||
'--interval',
|
||||
dest='schedule_interval',
|
||||
type=str,
|
||||
help='Override schedule interval (e.g., "hourly", "*/30m", "daily")'
|
||||
)
|
||||
|
||||
schedule_parser.add_argument(
|
||||
'--max-retries',
|
||||
dest='schedule_max_retries',
|
||||
type=int,
|
||||
help='Override maximum retry attempts (default: from YAML or 3)'
|
||||
)
|
||||
|
||||
schedule_parser.add_argument(
|
||||
'--verbose', '-v',
|
||||
action='store_true',
|
||||
help='Enable verbose logging'
|
||||
)
|
||||
|
||||
schedule_parser.set_defaults(func=AgentSchedulerHandler.handle_schedule_command)
|
||||
|
||||
|
||||
def setup_scheduler_cli(subparsers):
|
||||
"""
|
||||
Setup scheduler CLI commands.
|
||||
|
||||
Args:
|
||||
subparsers: Subparsers object from argparse
|
||||
"""
|
||||
AgentSchedulerHandler.add_schedule_arguments(subparsers)
|
||||
@@ -0,0 +1,375 @@
|
||||
"""
|
||||
Multi-Agent CLI Feature Handler
|
||||
|
||||
Provides CLI commands for defining and running multiple agents:
|
||||
- praisonai agents run --agent "name:role:tools" --agent "..." --task "..."
|
||||
|
||||
Example:
|
||||
praisonai agents run \
|
||||
--agent "researcher:Research Analyst:internet_search" \
|
||||
--agent "writer:Content Writer:write_file" \
|
||||
--task "Research AI trends and write a report"
|
||||
"""
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import Optional, List, Dict, Any
|
||||
|
||||
|
||||
def parse_agent_definition(definition: str) -> Dict[str, Any]:
|
||||
"""Parse an agent definition string.
|
||||
|
||||
Format: name:role:tools:goal (tools and goal are optional)
|
||||
|
||||
Args:
|
||||
definition: Agent definition string
|
||||
|
||||
Returns:
|
||||
Dictionary with name, role, tools, and optionally goal
|
||||
"""
|
||||
parts = definition.split(':')
|
||||
|
||||
result = {
|
||||
'name': parts[0].strip() if len(parts) > 0 else 'agent',
|
||||
'role': parts[1].strip() if len(parts) > 1 else 'Assistant',
|
||||
'tools': [],
|
||||
'goal': None
|
||||
}
|
||||
|
||||
# Parse tools (comma-separated)
|
||||
if len(parts) > 2 and parts[2].strip():
|
||||
result['tools'] = [t.strip() for t in parts[2].split(',') if t.strip()]
|
||||
|
||||
# Parse goal if provided
|
||||
if len(parts) > 3:
|
||||
result['goal'] = parts[3].strip()
|
||||
|
||||
return result
|
||||
|
||||
|
||||
class MultiAgentHandler:
|
||||
"""Handler for multi-agent CLI commands."""
|
||||
|
||||
def __init__(self, verbose: bool = True):
|
||||
"""Initialize the multi-agent handler.
|
||||
|
||||
Args:
|
||||
verbose: Whether to print verbose output
|
||||
"""
|
||||
self.verbose = verbose
|
||||
|
||||
def create_agents_from_definitions(
|
||||
self,
|
||||
definitions: List[str],
|
||||
llm: Optional[str] = None,
|
||||
instructions: Optional[str] = None
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Create agent configurations from definition strings.
|
||||
|
||||
Args:
|
||||
definitions: List of agent definition strings
|
||||
llm: LLM model to use for all agents
|
||||
instructions: Additional instructions for all agents
|
||||
|
||||
Returns:
|
||||
List of agent configuration dictionaries
|
||||
"""
|
||||
agents = []
|
||||
|
||||
for definition in definitions:
|
||||
agent_config = parse_agent_definition(definition)
|
||||
|
||||
if llm:
|
||||
agent_config['llm'] = llm
|
||||
|
||||
if instructions:
|
||||
agent_config['instructions'] = instructions
|
||||
|
||||
agents.append(agent_config)
|
||||
|
||||
return agents
|
||||
|
||||
def prepare_execution_config(
|
||||
self,
|
||||
agent_definitions: List[str],
|
||||
task: str,
|
||||
process: str = "sequential",
|
||||
llm: Optional[str] = None,
|
||||
instructions: Optional[str] = None
|
||||
) -> Dict[str, Any]:
|
||||
"""Prepare configuration for agent execution.
|
||||
|
||||
Args:
|
||||
agent_definitions: List of agent definition strings
|
||||
task: Task description
|
||||
process: Execution process (sequential, parallel)
|
||||
llm: LLM model to use
|
||||
instructions: Additional instructions
|
||||
|
||||
Returns:
|
||||
Execution configuration dictionary
|
||||
"""
|
||||
agents = self.create_agents_from_definitions(
|
||||
agent_definitions, llm, instructions
|
||||
)
|
||||
|
||||
return {
|
||||
'agents': agents,
|
||||
'task': task,
|
||||
'process': process,
|
||||
'llm': llm
|
||||
}
|
||||
|
||||
def _load_tools(self, tool_names: List[str]) -> List:
|
||||
"""Load tool functions by name.
|
||||
|
||||
Args:
|
||||
tool_names: List of tool names to load
|
||||
|
||||
Returns:
|
||||
List of tool functions
|
||||
"""
|
||||
tools = []
|
||||
|
||||
try:
|
||||
from praisonaiagents.tools import (
|
||||
internet_search, read_file, write_file, list_files,
|
||||
execute_command, read_csv, write_csv, analyze_csv
|
||||
)
|
||||
|
||||
tool_map = {
|
||||
'internet_search': internet_search,
|
||||
'read_file': read_file,
|
||||
'write_file': write_file,
|
||||
'list_files': list_files,
|
||||
'execute_command': execute_command,
|
||||
'read_csv': read_csv,
|
||||
'write_csv': write_csv,
|
||||
'analyze_csv': analyze_csv,
|
||||
}
|
||||
|
||||
for name in tool_names:
|
||||
if name in tool_map:
|
||||
tools.append(tool_map[name])
|
||||
else:
|
||||
if self.verbose:
|
||||
print(f"⚠ Tool '{name}' not found, skipping")
|
||||
except ImportError as e:
|
||||
if self.verbose:
|
||||
print(f"⚠ Could not load tools: {e}")
|
||||
|
||||
return tools
|
||||
|
||||
def _execute_agents(
|
||||
self,
|
||||
agents_config: List[Dict[str, Any]],
|
||||
task: str,
|
||||
process: str = "sequential",
|
||||
llm: Optional[str] = None
|
||||
) -> str:
|
||||
"""Execute agents with the given configuration.
|
||||
|
||||
Args:
|
||||
agents_config: List of agent configurations
|
||||
task: Task to execute
|
||||
process: Execution process
|
||||
llm: LLM model to use
|
||||
|
||||
Returns:
|
||||
Execution result
|
||||
"""
|
||||
try:
|
||||
from praisonaiagents import Agent, Task, PraisonAIAgents
|
||||
|
||||
# Create Agent objects
|
||||
agents = []
|
||||
tasks = []
|
||||
|
||||
for i, config in enumerate(agents_config):
|
||||
# Load tools for this agent
|
||||
tools = self._load_tools(config.get('tools', []))
|
||||
|
||||
agent = Agent(
|
||||
name=config['name'],
|
||||
role=config['role'],
|
||||
goal=config.get('goal') or f"Complete tasks as {config['role']}",
|
||||
backstory=f"You are a skilled {config['role']}.",
|
||||
instructions=config.get('instructions', ''),
|
||||
tools=tools if tools else None,
|
||||
llm=config.get('llm') or llm or os.environ.get('OPENAI_MODEL_NAME', 'gpt-4o-mini'),
|
||||
verbose=self.verbose
|
||||
)
|
||||
agents.append(agent)
|
||||
|
||||
# Create task for this agent
|
||||
agent_task = Task(
|
||||
name=f"task_{config['name']}",
|
||||
description=task if i == 0 else f"Continue the work: {task}",
|
||||
expected_output="Completed task output",
|
||||
agent=agent
|
||||
)
|
||||
tasks.append(agent_task)
|
||||
|
||||
# Run agents
|
||||
if self.verbose:
|
||||
print(f"\n🚀 Running {len(agents)} agents ({process} mode)...")
|
||||
for agent in agents:
|
||||
print(f" - {agent.name}: {agent.role}")
|
||||
print()
|
||||
|
||||
praison_agents = PraisonAIAgents(
|
||||
agents=agents,
|
||||
tasks=tasks,
|
||||
process=process,
|
||||
verbose=self.verbose
|
||||
)
|
||||
|
||||
result = praison_agents.start()
|
||||
return result
|
||||
|
||||
except Exception as e:
|
||||
if self.verbose:
|
||||
print(f"❌ Execution failed: {e}")
|
||||
raise
|
||||
|
||||
def run(
|
||||
self,
|
||||
agent_definitions: List[str],
|
||||
task: str,
|
||||
process: str = "sequential",
|
||||
llm: Optional[str] = None,
|
||||
instructions: Optional[str] = None
|
||||
) -> str:
|
||||
"""Run multiple agents with the given task.
|
||||
|
||||
Args:
|
||||
agent_definitions: List of agent definition strings
|
||||
task: Task to execute
|
||||
process: Execution process (sequential, parallel)
|
||||
llm: LLM model to use
|
||||
instructions: Additional instructions
|
||||
|
||||
Returns:
|
||||
Execution result
|
||||
"""
|
||||
config = self.prepare_execution_config(
|
||||
agent_definitions, task, process, llm, instructions
|
||||
)
|
||||
|
||||
return self._execute_agents(
|
||||
config['agents'],
|
||||
config['task'],
|
||||
config['process'],
|
||||
config['llm']
|
||||
)
|
||||
|
||||
|
||||
def handle_agents_command(args) -> int:
|
||||
"""Handle agents subcommand from CLI.
|
||||
|
||||
Args:
|
||||
args: Parsed command line arguments
|
||||
|
||||
Returns:
|
||||
Exit code (0 for success, non-zero for error)
|
||||
"""
|
||||
handler = MultiAgentHandler(verbose=True)
|
||||
|
||||
try:
|
||||
if args.agents_command == "run":
|
||||
if not hasattr(args, 'agent') or not args.agent:
|
||||
print("Error: At least one --agent is required")
|
||||
return 1
|
||||
|
||||
if not hasattr(args, 'task') or not args.task:
|
||||
print("Error: --task is required")
|
||||
return 1
|
||||
|
||||
result = handler.run(
|
||||
agent_definitions=args.agent,
|
||||
task=args.task,
|
||||
process=getattr(args, 'process', 'sequential'),
|
||||
llm=getattr(args, 'llm', None),
|
||||
instructions=getattr(args, 'instructions', None)
|
||||
)
|
||||
|
||||
print("\n" + "="*50)
|
||||
print("RESULT:")
|
||||
print("="*50)
|
||||
print(result)
|
||||
|
||||
elif args.agents_command == "list":
|
||||
# List available tools
|
||||
print("\nAvailable tools for agents:")
|
||||
print("-" * 30)
|
||||
tools = [
|
||||
("internet_search", "Search the web"),
|
||||
("read_file", "Read file contents"),
|
||||
("write_file", "Write to a file"),
|
||||
("list_files", "List directory contents"),
|
||||
("execute_command", "Execute shell commands"),
|
||||
("read_csv", "Read CSV files"),
|
||||
("write_csv", "Write CSV files"),
|
||||
("analyze_csv", "Analyze CSV data"),
|
||||
]
|
||||
for name, desc in tools:
|
||||
print(f" {name}: {desc}")
|
||||
|
||||
else:
|
||||
print(f"Unknown agents command: {args.agents_command}")
|
||||
return 1
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error: {e}")
|
||||
return 1
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
def add_agents_parser(subparsers) -> None:
|
||||
"""Add agents subcommand to argument parser.
|
||||
|
||||
Args:
|
||||
subparsers: Subparsers object from argparse
|
||||
"""
|
||||
# run command
|
||||
run_parser = subparsers.add_parser(
|
||||
'run',
|
||||
help='Run multiple agents with a task'
|
||||
)
|
||||
run_parser.add_argument(
|
||||
'--agent', '-a',
|
||||
action='append',
|
||||
required=True,
|
||||
help='Agent definition (format: name:role:tools). Can be used multiple times.'
|
||||
)
|
||||
run_parser.add_argument(
|
||||
'--task', '-t',
|
||||
required=True,
|
||||
help='Task for the agents to complete'
|
||||
)
|
||||
run_parser.add_argument(
|
||||
'--process', '-p',
|
||||
choices=['sequential', 'parallel'],
|
||||
default='sequential',
|
||||
help='Execution process (default: sequential)'
|
||||
)
|
||||
run_parser.add_argument(
|
||||
'--llm', '-m',
|
||||
help='LLM model to use for all agents'
|
||||
)
|
||||
run_parser.add_argument(
|
||||
'--instructions', '-i',
|
||||
help='Additional instructions for all agents'
|
||||
)
|
||||
run_parser.add_argument(
|
||||
'--verbose', '-v',
|
||||
action='store_true',
|
||||
help='Enable verbose output'
|
||||
)
|
||||
|
||||
# list command - list available tools
|
||||
list_parser = subparsers.add_parser(
|
||||
'list',
|
||||
help='List available tools for agents'
|
||||
)
|
||||
@@ -0,0 +1,471 @@
|
||||
"""
|
||||
@ Mention Autocomplete for PraisonAI CLI.
|
||||
|
||||
Provides file/directory autocomplete when user types @.
|
||||
Uses only stdlib imports for zero performance impact.
|
||||
|
||||
Features:
|
||||
- Detect @ trigger in input
|
||||
- Fuzzy search files/directories
|
||||
- Cache results for performance
|
||||
- Integrate with prompt_toolkit
|
||||
"""
|
||||
|
||||
import os
|
||||
import time
|
||||
import fnmatch
|
||||
from dataclasses import dataclass
|
||||
from typing import List, Optional, Dict, Tuple
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Data Classes
|
||||
# ============================================================================
|
||||
|
||||
@dataclass
|
||||
class AtMentionContext:
|
||||
"""Context for an @ mention in user input."""
|
||||
is_active: bool
|
||||
query: str
|
||||
start_pos: int
|
||||
mention_type: str = "file" # file, directory, web, etc.
|
||||
|
||||
|
||||
@dataclass
|
||||
class FileSuggestion:
|
||||
"""A file or directory suggestion."""
|
||||
path: str
|
||||
file_type: str # "file" or "directory"
|
||||
score: int = 100
|
||||
display: str = ""
|
||||
|
||||
def __post_init__(self):
|
||||
if not self.display:
|
||||
self.display = self.path
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# @ Mention Detection
|
||||
# ============================================================================
|
||||
|
||||
def detect_at_mention(text: str, cursor_pos: int) -> Optional[AtMentionContext]:
|
||||
"""
|
||||
Detect if cursor is within an @ mention context.
|
||||
|
||||
Scans backward from cursor to find @ symbol.
|
||||
Returns None if not in @ context.
|
||||
|
||||
Args:
|
||||
text: Full input text
|
||||
cursor_pos: Current cursor position
|
||||
|
||||
Returns:
|
||||
AtMentionContext if in @ mention, None otherwise
|
||||
"""
|
||||
if cursor_pos <= 0 or cursor_pos > len(text):
|
||||
return None
|
||||
|
||||
# Scan backward from cursor to find @
|
||||
at_pos = -1
|
||||
for i in range(cursor_pos - 1, -1, -1):
|
||||
char = text[i]
|
||||
|
||||
# Found @
|
||||
if char == '@':
|
||||
at_pos = i
|
||||
break
|
||||
|
||||
# Space/newline breaks @ context
|
||||
if char in ' \t\n\r':
|
||||
return None
|
||||
|
||||
if at_pos == -1:
|
||||
return None
|
||||
|
||||
# Extract query between @ and cursor
|
||||
query = text[at_pos + 1:cursor_pos]
|
||||
|
||||
return AtMentionContext(
|
||||
is_active=True,
|
||||
query=query,
|
||||
start_pos=at_pos
|
||||
)
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# File Search Service
|
||||
# ============================================================================
|
||||
|
||||
class FileSearchService:
|
||||
"""
|
||||
Service for searching files in a directory.
|
||||
|
||||
Features:
|
||||
- Walk directory tree
|
||||
- Fuzzy match file paths
|
||||
- Cache results with TTL
|
||||
- Respect .gitignore (basic)
|
||||
- Support absolute paths (@/Users/...) and home paths (@~/...)
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
root_dir: str,
|
||||
cache_ttl: int = 30,
|
||||
max_depth: int = 5,
|
||||
ignore_patterns: Optional[List[str]] = None
|
||||
):
|
||||
self._root = os.path.abspath(root_dir)
|
||||
self._cache_ttl = cache_ttl
|
||||
self._max_depth = max_depth
|
||||
self._ignore_patterns = ignore_patterns or [
|
||||
'.git', '__pycache__', 'node_modules', '.venv', 'venv',
|
||||
'*.pyc', '*.pyo', '.DS_Store', '*.egg-info'
|
||||
]
|
||||
|
||||
# Cache: query -> (results, timestamp)
|
||||
self._cache: Dict[str, Tuple[List[FileSuggestion], float]] = {}
|
||||
|
||||
# File list cache per directory
|
||||
self._file_list: Optional[List[str]] = None
|
||||
self._file_list_time: float = 0
|
||||
self._file_list_dir: Optional[str] = None
|
||||
|
||||
def _should_ignore(self, name: str) -> bool:
|
||||
"""Check if file/dir should be ignored."""
|
||||
for pattern in self._ignore_patterns:
|
||||
if fnmatch.fnmatch(name, pattern):
|
||||
return True
|
||||
return False
|
||||
|
||||
def _get_file_list(self, search_dir: str = None) -> List[str]:
|
||||
"""Get list of all files (cached)."""
|
||||
now = time.time()
|
||||
target_dir = search_dir or self._root
|
||||
|
||||
# Return cached if fresh and same directory
|
||||
if (self._file_list is not None and
|
||||
self._file_list_dir == target_dir and
|
||||
(now - self._file_list_time) < self._cache_ttl):
|
||||
return self._file_list
|
||||
|
||||
files = []
|
||||
|
||||
try:
|
||||
for root, dirs, filenames in os.walk(target_dir):
|
||||
# Calculate depth
|
||||
rel_root = os.path.relpath(root, target_dir)
|
||||
depth = 0 if rel_root == '.' else rel_root.count(os.sep) + 1
|
||||
|
||||
if depth > self._max_depth:
|
||||
dirs.clear() # Don't descend further
|
||||
continue
|
||||
|
||||
# Filter ignored directories
|
||||
dirs[:] = [d for d in dirs if not self._should_ignore(d)]
|
||||
|
||||
# Add directories
|
||||
for d in dirs:
|
||||
rel_path = os.path.relpath(os.path.join(root, d), target_dir)
|
||||
files.append(rel_path + os.sep)
|
||||
|
||||
# Add files
|
||||
for f in filenames:
|
||||
if not self._should_ignore(f):
|
||||
rel_path = os.path.relpath(os.path.join(root, f), target_dir)
|
||||
files.append(rel_path)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
self._file_list = files
|
||||
self._file_list_time = now
|
||||
self._file_list_dir = target_dir
|
||||
return files
|
||||
|
||||
def _fuzzy_match(self, text: str, query: str) -> int:
|
||||
"""
|
||||
Simple fuzzy match scoring.
|
||||
|
||||
Returns score 0-100, higher is better.
|
||||
"""
|
||||
if not query:
|
||||
return 50 # Default score for empty query
|
||||
|
||||
text_lower = text.lower()
|
||||
query_lower = query.lower()
|
||||
|
||||
# Exact match
|
||||
if text_lower == query_lower:
|
||||
return 100
|
||||
|
||||
# Starts with
|
||||
if text_lower.startswith(query_lower):
|
||||
return 90
|
||||
|
||||
# Contains
|
||||
if query_lower in text_lower:
|
||||
return 80
|
||||
|
||||
# Fuzzy: all query chars appear in order
|
||||
text_idx = 0
|
||||
query_idx = 0
|
||||
matches = 0
|
||||
|
||||
while text_idx < len(text_lower) and query_idx < len(query_lower):
|
||||
if text_lower[text_idx] == query_lower[query_idx]:
|
||||
matches += 1
|
||||
query_idx += 1
|
||||
text_idx += 1
|
||||
|
||||
if query_idx == len(query_lower):
|
||||
# All query chars found
|
||||
return 50 + int(30 * matches / len(text))
|
||||
|
||||
return 0
|
||||
|
||||
def search(self, query: str, max_results: int = 20) -> List[FileSuggestion]:
|
||||
"""
|
||||
Search for files matching query.
|
||||
|
||||
Supports:
|
||||
- Relative paths: @file.txt, @src/main.py
|
||||
- Home paths: @~/Documents, @~/.config
|
||||
- Absolute paths: @/Users/name/file.txt, @/etc/hosts
|
||||
|
||||
Args:
|
||||
query: Search query (fuzzy matched)
|
||||
max_results: Maximum results to return
|
||||
|
||||
Returns:
|
||||
List of FileSuggestion sorted by score
|
||||
"""
|
||||
# Determine search directory based on query
|
||||
search_dir = self._root
|
||||
search_query = query
|
||||
prefix = ""
|
||||
|
||||
# Handle home directory paths (@~/)
|
||||
if query.startswith('~'):
|
||||
expanded = os.path.expanduser(query)
|
||||
# Find the directory part and filename part
|
||||
if os.path.isdir(expanded):
|
||||
search_dir = expanded
|
||||
search_query = ""
|
||||
prefix = query.rstrip('/') + '/'
|
||||
else:
|
||||
parent = os.path.dirname(expanded)
|
||||
if os.path.isdir(parent):
|
||||
search_dir = parent
|
||||
search_query = os.path.basename(expanded)
|
||||
prefix = os.path.dirname(query).rstrip('/') + '/'
|
||||
else:
|
||||
# Just expand ~ and search from home
|
||||
search_dir = os.path.expanduser('~')
|
||||
search_query = query[2:] if query.startswith('~/') else query[1:]
|
||||
prefix = "~/"
|
||||
|
||||
# Handle absolute paths (@/Users/..., @/etc/...)
|
||||
elif query.startswith('/'):
|
||||
if os.path.isdir(query):
|
||||
search_dir = query
|
||||
search_query = ""
|
||||
prefix = query.rstrip('/') + '/'
|
||||
else:
|
||||
parent = os.path.dirname(query)
|
||||
if os.path.isdir(parent):
|
||||
search_dir = parent
|
||||
search_query = os.path.basename(query)
|
||||
prefix = parent.rstrip('/') + '/'
|
||||
else:
|
||||
# Search from root with the query
|
||||
search_dir = '/'
|
||||
search_query = query[1:]
|
||||
prefix = "/"
|
||||
|
||||
# Check cache
|
||||
cache_key = f"{search_dir}:{search_query}:{max_results}"
|
||||
now = time.time()
|
||||
|
||||
if cache_key in self._cache:
|
||||
results, cached_time = self._cache[cache_key]
|
||||
if (now - cached_time) < self._cache_ttl:
|
||||
return results
|
||||
|
||||
# Get all files from the appropriate directory
|
||||
all_files = self._get_file_list(search_dir)
|
||||
|
||||
# Score and filter
|
||||
scored: List[Tuple[int, str]] = []
|
||||
for path in all_files:
|
||||
score = self._fuzzy_match(path, search_query)
|
||||
if score > 0:
|
||||
scored.append((score, path))
|
||||
|
||||
# Sort by score descending
|
||||
scored.sort(key=lambda x: (-x[0], x[1]))
|
||||
|
||||
# Build results with proper prefix
|
||||
results = []
|
||||
for score, path in scored[:max_results]:
|
||||
file_type = "directory" if path.endswith(os.sep) else "file"
|
||||
display_path = prefix + path if prefix else path
|
||||
results.append(FileSuggestion(
|
||||
path=display_path,
|
||||
file_type=file_type,
|
||||
score=score
|
||||
))
|
||||
|
||||
# Cache results
|
||||
self._cache[cache_key] = (results, now)
|
||||
|
||||
return results
|
||||
|
||||
def clear_cache(self) -> None:
|
||||
"""Clear all caches."""
|
||||
self._cache.clear()
|
||||
self._file_list = None
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Completers for prompt_toolkit
|
||||
# ============================================================================
|
||||
|
||||
# Lazy import to avoid performance impact
|
||||
_Completer = None
|
||||
_Completion = None
|
||||
|
||||
def _get_prompt_toolkit():
|
||||
"""Lazy import prompt_toolkit."""
|
||||
global _Completer, _Completion
|
||||
if _Completer is None:
|
||||
from prompt_toolkit.completion import Completer, Completion
|
||||
_Completer = Completer
|
||||
_Completion = Completion
|
||||
return _Completer, _Completion
|
||||
|
||||
|
||||
class AtMentionCompleter:
|
||||
"""
|
||||
Completer for @ mentions.
|
||||
|
||||
Shows file suggestions when user types @.
|
||||
Implements prompt_toolkit Completer interface.
|
||||
"""
|
||||
|
||||
def __init__(self, root_dir: str):
|
||||
self._file_service = FileSearchService(root_dir)
|
||||
|
||||
def get_completions(self, document, complete_event):
|
||||
"""Get completions for @ mentions."""
|
||||
_, Completion = _get_prompt_toolkit()
|
||||
|
||||
text = document.text_before_cursor
|
||||
cursor_pos = len(text)
|
||||
|
||||
# Detect @ context
|
||||
context = detect_at_mention(text, cursor_pos)
|
||||
if context is None or not context.is_active:
|
||||
return
|
||||
|
||||
# Search files
|
||||
results = self._file_service.search(context.query, max_results=15)
|
||||
|
||||
# Yield completions
|
||||
for suggestion in results:
|
||||
# Icon based on type
|
||||
icon = "📁 " if suggestion.file_type == "directory" else "📄 "
|
||||
display = f"{icon}{suggestion.path}"
|
||||
|
||||
# Calculate replacement position
|
||||
# Replace from @ to cursor
|
||||
start_position = -(cursor_pos - context.start_pos)
|
||||
|
||||
yield Completion(
|
||||
text=f"@{suggestion.path}",
|
||||
start_position=start_position,
|
||||
display=display
|
||||
)
|
||||
|
||||
|
||||
def create_combined_completer(commands: List[str], root_dir: str):
|
||||
"""
|
||||
Factory function to create a CombinedCompleter.
|
||||
|
||||
Returns a proper Completer subclass that handles both / and @.
|
||||
"""
|
||||
Completer, Completion = _get_prompt_toolkit()
|
||||
file_service = FileSearchService(root_dir)
|
||||
|
||||
class CombinedCompleter(Completer):
|
||||
"""
|
||||
Combined completer for both / commands and @ mentions.
|
||||
|
||||
Properly inherits from prompt_toolkit Completer.
|
||||
"""
|
||||
|
||||
def get_completions(self, document, complete_event):
|
||||
"""Get completions for / or @."""
|
||||
text = document.text_before_cursor.lstrip()
|
||||
|
||||
# Check for slash command
|
||||
if text.startswith('/'):
|
||||
cmd_text = text[1:].lower()
|
||||
for cmd in commands:
|
||||
if cmd.lower().startswith(cmd_text):
|
||||
yield Completion(
|
||||
f'/{cmd}',
|
||||
start_position=-len(text),
|
||||
display=f'/{cmd}'
|
||||
)
|
||||
return
|
||||
|
||||
# Check for @ mention
|
||||
full_text = document.text_before_cursor
|
||||
cursor_pos = len(full_text)
|
||||
|
||||
context = detect_at_mention(full_text, cursor_pos)
|
||||
if context is None or not context.is_active:
|
||||
return
|
||||
|
||||
# Search files
|
||||
results = file_service.search(context.query, max_results=15)
|
||||
|
||||
# Yield completions
|
||||
for suggestion in results:
|
||||
icon = "📁 " if suggestion.file_type == "directory" else "📄 "
|
||||
display = f"{icon}{suggestion.path}"
|
||||
start_position = -(cursor_pos - context.start_pos)
|
||||
|
||||
yield Completion(
|
||||
text=f"@{suggestion.path}",
|
||||
start_position=start_position,
|
||||
display=display
|
||||
)
|
||||
|
||||
return CombinedCompleter()
|
||||
|
||||
|
||||
# Backward compatible alias
|
||||
class CombinedCompleter:
|
||||
"""
|
||||
Wrapper class for backward compatibility.
|
||||
|
||||
Use create_combined_completer() for proper Completer subclass.
|
||||
"""
|
||||
|
||||
def __new__(cls, commands: List[str], root_dir: str):
|
||||
return create_combined_completer(commands, root_dir)
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Exports
|
||||
# ============================================================================
|
||||
|
||||
__all__ = [
|
||||
'AtMentionContext',
|
||||
'FileSuggestion',
|
||||
'detect_at_mention',
|
||||
'FileSearchService',
|
||||
'AtMentionCompleter',
|
||||
'CombinedCompleter',
|
||||
]
|
||||
Loaded 100 of 184 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user