mirror of
https://github.com/MervinPraison/PraisonAI.git
synced 2026-11-07 17:33:33 -10:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
335e8d7e47 | ||
|
|
d65840e092 | ||
|
|
5a13571fdd | ||
|
|
e3b48c2526 | ||
|
|
8d32238b4c | ||
|
|
c17b1298bb | ||
|
|
ec6c150315 | ||
|
|
3d26cedd5d | ||
|
|
8c735a60b2 | ||
|
|
2f5f2f1c13 | ||
|
|
0253073c68 | ||
|
|
7f0e334e8c | ||
|
|
cacf1097ef | ||
|
|
09f01919d2 | ||
|
|
92397615f8 | ||
|
|
4b21943a53 | ||
|
|
af56a753bd | ||
|
|
15c2559a3c | ||
|
|
3215bdc0a6 | ||
|
|
0ff0d83744 | ||
|
|
8f956f91dd | ||
|
|
954f6ac33d | ||
|
|
264d11d8d9 | ||
|
|
88da2bdd7b | ||
|
|
b84e6c2460 | ||
|
|
0434b23ada | ||
|
|
a9951e0694 | ||
|
|
073062cd96 | ||
|
|
936b422876 | ||
|
|
99ab421ce2 | ||
|
|
c1f334905e |
No files matched your search
@@ -171,12 +171,13 @@ npm install praisonai
|
||||
|
||||
| Feature | Code | Docs |
|
||||
|---------|:----:|:----:|
|
||||
| 🚀 Single Agent | [Example](examples/python/agents/single-agent.py) | [📖](https://docs.praison.ai/agents/single) |
|
||||
| 🤝 Multi Agents | [Example](examples/python/general/mini_agents_example.py) | [📖](https://docs.praison.ai/concepts/agents) |
|
||||
| 🤖 Auto Agents | [Example](examples/python/general/auto_agents_example.py) | [📖](https://docs.praison.ai/features/autoagents) |
|
||||
| 🔄 Self Reflection AI Agents | [Example](examples/python/concepts/self-reflection-details.py) | [📖](https://docs.praison.ai/features/selfreflection) |
|
||||
| 🧠 Reasoning AI Agents | [Example](examples/python/concepts/reasoning-extraction.py) | [📖](https://docs.praison.ai/features/reasoning) |
|
||||
| 👁️ Multi Modal AI Agents | [Example](examples/python/general/multimodal.py) | [📖](https://docs.praison.ai/features/multimodal) |
|
||||
| **🤖 Core Agents** | | |
|
||||
| ↳ Single Agent | [Example](examples/python/agents/single-agent.py) | [📖](https://docs.praison.ai/agents/single) |
|
||||
| ↳ Multi Agents | [Example](examples/python/general/mini_agents_example.py) | [📖](https://docs.praison.ai/concepts/agents) |
|
||||
| ↳ Auto Agents | [Example](examples/python/general/auto_agents_example.py) | [📖](https://docs.praison.ai/features/autoagents) |
|
||||
| ↳ Self Reflection AI Agents | [Example](examples/python/concepts/self-reflection-details.py) | [📖](https://docs.praison.ai/features/selfreflection) |
|
||||
| ↳ Reasoning AI Agents | [Example](examples/python/concepts/reasoning-extraction.py) | [📖](https://docs.praison.ai/features/reasoning) |
|
||||
| ↳ Multi Modal AI Agents | [Example](examples/python/general/multimodal.py) | [📖](https://docs.praison.ai/features/multimodal) |
|
||||
| **🔄 Workflows** | | |
|
||||
| ↳ Simple Workflow | [Example](examples/python/workflows/simple_workflow.py) | [📖](https://docs.praison.ai/features/workflows) |
|
||||
| ↳ Workflow with Agents | [Example](examples/python/workflows/workflow_with_agents.py) | [📖](https://docs.praison.ai/features/workflows) |
|
||||
@@ -188,10 +189,7 @@ npm install praisonai
|
||||
| ↳ Workflow Branching | [Example](examples/python/workflows/workflow_branching.py) | [📖](https://docs.praison.ai/features/workflows) |
|
||||
| ↳ Workflow Early Stop | [Example](examples/python/workflows/workflow_early_stop.py) | [📖](https://docs.praison.ai/features/workflows) |
|
||||
| ↳ Workflow Checkpoints | [Example](examples/python/workflows/workflow_checkpoints.py) | [📖](https://docs.praison.ai/features/workflows) |
|
||||
| 📚 Add Custom Knowledge | [Example](examples/python/concepts/knowledge-agents.py) | [📖](https://docs.praison.ai/features/knowledge) |
|
||||
| 🧠 Memory (Short & Long Term) | [Example](examples/python/general/memory_example.py) | [📖](https://docs.praison.ai/concepts/memory) |
|
||||
| 📄 Chat with PDF Agents | [Example](examples/python/concepts/chat-with-pdf.py) | [📖](https://docs.praison.ai/features/chat-with-pdf) |
|
||||
| **💻 Code** | | |
|
||||
| **💻 Code & Development** | | |
|
||||
| ↳ Code Interpreter Agents | [Example](examples/python/agents/code-agent.py) | [📖](https://docs.praison.ai/features/codeagent) |
|
||||
| ↳ AI Code Editing Tools | [Example](examples/python/code/code_editing_example.py) | [📖](https://docs.praison.ai/code/editing) |
|
||||
| ↳ External Agents (All) | [Example](examples/python/code/external_agents_example.py) | [📖](https://docs.praison.ai/code/external-agents) |
|
||||
@@ -199,65 +197,78 @@ npm install praisonai
|
||||
| ↳ Gemini CLI | [Example](examples/python/code/gemini_cli_example.py) | [📖](https://docs.praison.ai/code/gemini-cli) |
|
||||
| ↳ Codex CLI | [Example](examples/python/code/codex_cli_example.py) | [📖](https://docs.praison.ai/code/codex-cli) |
|
||||
| ↳ Cursor CLI | [Example](examples/python/code/cursor_cli_example.py) | [📖](https://docs.praison.ai/code/cursor-cli) |
|
||||
| 📚 RAG Agents | [Example](examples/python/concepts/rag-agents.py) | [📖](https://docs.praison.ai/features/rag) |
|
||||
| 🤔 Async & Parallel Processing | [Example](examples/python/general/async_example.py) | [📖](https://docs.praison.ai/features/async) |
|
||||
| 🔢 Math Agents | [Example](examples/python/agents/math-agent.py) | [📖](https://docs.praison.ai/features/mathagent) |
|
||||
| 🎯 Structured Output Agents | [Example](examples/python/general/structured_agents_example.py) | [📖](https://docs.praison.ai/features/structured) |
|
||||
| 🔗 LangChain Integrated Agents | [Example](examples/python/general/langchain_example.py) | [📖](https://docs.praison.ai/features/langchain) |
|
||||
| 📞 Callback Agents | [Example](examples/python/general/advanced-callback-systems.py) | [📖](https://docs.praison.ai/features/callbacks) |
|
||||
| 🛠️ 100+ Custom Tools | [Example](examples/python/general/tools_example.py) | [📖](https://docs.praison.ai/tools/tools) |
|
||||
| 📄 YAML Configuration | [Example](examples/cookbooks/yaml/secondary_market_research_agents.yaml) | [📖](https://docs.praison.ai/developers/agents-playbook) |
|
||||
| 💯 100+ LLM Support | [Example](examples/python/providers/openai/openai_gpt4_example.py) | [📖](https://docs.praison.ai/models) |
|
||||
| 🔬 Deep Research Agents | [Example](examples/python/agents/research-agent.py) | [📖](https://docs.praison.ai/agents/deep-research) |
|
||||
| 🔄 Query Rewriter Agent | [Example](#5-query-rewriter-agent) | [📖](https://docs.praison.ai/agents/query-rewriter) |
|
||||
| 🌐 Native Web Search | [Example](examples/python/agents/websearch-agent.py) | [📖](https://docs.praison.ai/agents/websearch) |
|
||||
| 📥 Web Fetch (Anthropic) | [Example](#web-search-web-fetch--prompt-caching) | [📖](https://docs.praison.ai/features/model-capabilities) |
|
||||
| 💾 Prompt Caching | [Example](#web-search-web-fetch--prompt-caching) | [📖](https://docs.praison.ai/features/model-capabilities) |
|
||||
| 🧠 Claude Memory Tool | [Example](#claude-memory-tool-cli) | [📖](https://docs.praison.ai/features/claude-memory-tool) |
|
||||
| 💾 File-Based Memory | [Example](examples/python/general/memory_example.py) | [📖](https://docs.praison.ai/concepts/memory) |
|
||||
| 🔍 Built-in Search Tools | [Example](examples/python/agents/websearch-agent.py) | [📖](https://docs.praison.ai/tools/tavily) |
|
||||
| 🔎 Unified Web Search | [Example](src/praisonai-agents/examples/web_search_example.py) | [📖](https://docs.praison.ai/tools/web-search) |
|
||||
| 📋 Planning Mode | [Example](examples/python/agents/planning-agent.py) | [📖](https://docs.praison.ai/features/planning-mode) |
|
||||
| 🔧 Planning Tools | [Example](#3-agent-with-planning-mode) | [📖](https://docs.praison.ai/features/planning-mode) |
|
||||
| 🧠 Planning Reasoning | [Example](#3-agent-with-planning-mode) | [📖](https://docs.praison.ai/features/planning-mode) |
|
||||
| 🔌 MCP Transports | [Example](examples/python/mcp/mcp-transports-overview.py) | [📖](https://docs.praison.ai/mcp/transports) |
|
||||
| 🌐 WebSocket MCP | [Example](examples/python/mcp/websocket-mcp.py) | [📖](https://docs.praison.ai/mcp/sse-transport) |
|
||||
| 🔐 MCP Security | [Example](examples/python/mcp/mcp-security.py) | [📖](https://docs.praison.ai/mcp/transports) |
|
||||
| 🔄 MCP Resumability | [Example](examples/python/mcp/mcp-resumability.py) | [📖](https://docs.praison.ai/mcp/sse-transport) |
|
||||
| ⚡ Fast Context | [Example](examples/context/00_agent_fast_context_basic.py) | [📖](https://docs.praison.ai/features/fast-context) |
|
||||
| 🖼️ Image Generation Agent | [Example](examples/python/image/image-agent.py) | [📖](https://docs.praison.ai/features/image-generation) |
|
||||
| 📷 Image to Text Agent | [Example](examples/python/agents/image-to-text-agent.py) | [📖](https://docs.praison.ai/agents/image-to-text) |
|
||||
| 🎬 Video Agent | [Example](examples/python/agents/video-agent.py) | [📖](https://docs.praison.ai/agents/video) |
|
||||
| 📊 Data Analyst Agent | [Example](examples/python/agents/data-analyst-agent.py) | [📖](https://docs.praison.ai/agents/data-analyst) |
|
||||
| 💰 Finance Agent | [Example](examples/python/agents/finance-agent.py) | [📖](https://docs.praison.ai/agents/finance) |
|
||||
| 🛒 Shopping Agent | [Example](examples/python/agents/shopping-agent.py) | [📖](https://docs.praison.ai/agents/shopping) |
|
||||
| ⭐ Recommendation Agent | [Example](examples/python/agents/recommendation-agent.py) | [📖](https://docs.praison.ai/agents/recommendation) |
|
||||
| 📖 Wikipedia Agent | [Example](examples/python/agents/wikipedia-agent.py) | [📖](https://docs.praison.ai/agents/wikipedia) |
|
||||
| 💻 Programming Agent | [Example](examples/python/agents/programming-agent.py) | [📖](https://docs.praison.ai/agents/programming) |
|
||||
| 📝 Markdown Agent | [Example](examples/python/agents/markdown-agent.py) | [📖](https://docs.praison.ai/agents/markdown) |
|
||||
| 📝 Prompt Expander Agent | [Example](#prompt-expansion) | [📖](https://docs.praison.ai/agents/prompt-expander) |
|
||||
| 🔀 Model Router | [Example](examples/python/agents/router-agent-cost-optimization.py) | [📖](https://docs.praison.ai/features/model-router) |
|
||||
| ⛓️ Prompt Chaining | [Example](examples/python/general/prompt_chaining.py) | [📖](https://docs.praison.ai/features/promptchaining) |
|
||||
| 🔍 Evaluator Optimiser | [Example](examples/python/general/evaluator-optimiser.py) | [📖](https://docs.praison.ai/features/evaluator-optimiser) |
|
||||
| 👷 Orchestrator Workers | [Example](examples/python/general/orchestrator-workers.py) | [📖](https://docs.praison.ai/features/orchestrator-worker) |
|
||||
| ⚡ Parallelisation | [Example](examples/python/general/parallelisation.py) | [📖](https://docs.praison.ai/features/parallelisation) |
|
||||
| 🔁 Repetitive Agents | [Example](examples/python/concepts/repetitive-agents.py) | [📖](https://docs.praison.ai/features/repetitive) |
|
||||
| 🤝 Agent Handoffs | [Example](examples/python/handoff/handoff_basic.py) | [📖](https://docs.praison.ai/features/handoffs) |
|
||||
| 🛡️ Guardrails | [Example](examples/python/guardrails/comprehensive-guardrails-example.py) | [📖](https://docs.praison.ai/features/guardrails) |
|
||||
| 💬 Sessions Management | [Example](examples/python/sessions/comprehensive-session-management.py) | [📖](https://docs.praison.ai/features/sessions) |
|
||||
| ✅ Human Approval | [Example](examples/python/general/human_approval_example.py) | [📖](https://docs.praison.ai/features/approval) |
|
||||
| 🔄 Stateful Agents | [Example](examples/python/stateful/workflow-state-example.py) | [📖](https://docs.praison.ai/features/stateful-agents) |
|
||||
| 🤖 Autonomous Workflow | [Example](examples/python/general/autonomous-agent.py) | [📖](https://docs.praison.ai/features/autonomous-workflow) |
|
||||
| 📜 Rules & Instructions | [Example](#6-rules--instructions) | [📖](https://docs.praison.ai/features/rules) |
|
||||
| 🪝 Hooks | [Example](#9-hooks) | [📖](https://docs.praison.ai/features/hooks) |
|
||||
| 📈 Telemetry | [Example](examples/python/telemetry/production-telemetry-example.py) | [📖](https://docs.praison.ai/features/telemetry) |
|
||||
| 📹 Camera Integration | [Example](examples/python/camera/) | [📖](https://docs.praison.ai/features/camera-integration) |
|
||||
| 📄 Project Docs (.praison/docs/) | [Example](#docs-cli) | [📖](https://docs.praison.ai/docs/cli/docs) |
|
||||
| 🔌 MCP Config Management | [Example](#mcp-config-cli) | [📖](https://docs.praison.ai/docs/cli/mcp) |
|
||||
| 💬 AI Commit Messages | [Example](#ai-commit-cli) | [📖](https://docs.praison.ai/docs/cli/commit) |
|
||||
| @ @Mentions in Prompts | [Example](#mentions-in-prompts) | [📖](https://docs.praison.ai/docs/cli/mentions) |
|
||||
| 💾 Auto-Save Sessions | [Example](#session-management-python) | [📖](https://docs.praison.ai/docs/cli/session) |
|
||||
| 📜 History in Context | [Example](#session-management-python) | [📖](https://docs.praison.ai/docs/cli/session) |
|
||||
| **🧠 Memory & Knowledge** | | |
|
||||
| ↳ Memory (Short & Long Term) | [Example](examples/python/general/memory_example.py) | [📖](https://docs.praison.ai/concepts/memory) |
|
||||
| ↳ File-Based Memory | [Example](examples/python/general/memory_example.py) | [📖](https://docs.praison.ai/concepts/memory) |
|
||||
| ↳ Claude Memory Tool | [Example](#claude-memory-tool-cli) | [📖](https://docs.praison.ai/features/claude-memory-tool) |
|
||||
| ↳ Add Custom Knowledge | [Example](examples/python/concepts/knowledge-agents.py) | [📖](https://docs.praison.ai/features/knowledge) |
|
||||
| ↳ RAG Agents | [Example](examples/python/concepts/rag-agents.py) | [📖](https://docs.praison.ai/features/rag) |
|
||||
| ↳ Chat with PDF Agents | [Example](examples/python/concepts/chat-with-pdf.py) | [📖](https://docs.praison.ai/features/chat-with-pdf) |
|
||||
| **🔬 Research & Intelligence** | | |
|
||||
| ↳ Deep Research Agents | [Example](examples/python/agents/research-agent.py) | [📖](https://docs.praison.ai/agents/deep-research) |
|
||||
| ↳ Query Rewriter Agent | [Example](#5-query-rewriter-agent) | [📖](https://docs.praison.ai/agents/query-rewriter) |
|
||||
| ↳ Native Web Search | [Example](examples/python/agents/websearch-agent.py) | [📖](https://docs.praison.ai/agents/websearch) |
|
||||
| ↳ Built-in Search Tools | [Example](examples/python/agents/websearch-agent.py) | [📖](https://docs.praison.ai/tools/tavily) |
|
||||
| ↳ Unified Web Search | [Example](src/praisonai-agents/examples/web_search_example.py) | [📖](https://docs.praison.ai/tools/web-search) |
|
||||
| ↳ Web Fetch (Anthropic) | [Example](#web-search-web-fetch--prompt-caching) | [📖](https://docs.praison.ai/features/model-capabilities) |
|
||||
| **📋 Planning & Execution** | | |
|
||||
| ↳ Planning Mode | [Example](examples/python/agents/planning-agent.py) | [📖](https://docs.praison.ai/features/planning-mode) |
|
||||
| ↳ Planning Tools | [Example](#3-agent-with-planning-mode) | [📖](https://docs.praison.ai/features/planning-mode) |
|
||||
| ↳ Planning Reasoning | [Example](#3-agent-with-planning-mode) | [📖](https://docs.praison.ai/features/planning-mode) |
|
||||
| ↳ Prompt Chaining | [Example](examples/python/general/prompt_chaining.py) | [📖](https://docs.praison.ai/features/promptchaining) |
|
||||
| ↳ Evaluator Optimiser | [Example](examples/python/general/evaluator-optimiser.py) | [📖](https://docs.praison.ai/features/evaluator-optimiser) |
|
||||
| ↳ Orchestrator Workers | [Example](examples/python/general/orchestrator-workers.py) | [📖](https://docs.praison.ai/features/orchestrator-worker) |
|
||||
| **👥 Specialized Agents** | | |
|
||||
| ↳ Data Analyst Agent | [Example](examples/python/agents/data-analyst-agent.py) | [📖](https://docs.praison.ai/agents/data-analyst) |
|
||||
| ↳ Finance Agent | [Example](examples/python/agents/finance-agent.py) | [📖](https://docs.praison.ai/agents/finance) |
|
||||
| ↳ Shopping Agent | [Example](examples/python/agents/shopping-agent.py) | [📖](https://docs.praison.ai/agents/shopping) |
|
||||
| ↳ Recommendation Agent | [Example](examples/python/agents/recommendation-agent.py) | [📖](https://docs.praison.ai/agents/recommendation) |
|
||||
| ↳ Wikipedia Agent | [Example](examples/python/agents/wikipedia-agent.py) | [📖](https://docs.praison.ai/agents/wikipedia) |
|
||||
| ↳ Programming Agent | [Example](examples/python/agents/programming-agent.py) | [📖](https://docs.praison.ai/agents/programming) |
|
||||
| ↳ Math Agents | [Example](examples/python/agents/math-agent.py) | [📖](https://docs.praison.ai/features/mathagent) |
|
||||
| ↳ Markdown Agent | [Example](examples/python/agents/markdown-agent.py) | [📖](https://docs.praison.ai/agents/markdown) |
|
||||
| ↳ Prompt Expander Agent | [Example](#prompt-expansion) | [📖](https://docs.praison.ai/agents/prompt-expander) |
|
||||
| **🎨 Media & Multimodal** | | |
|
||||
| ↳ Image Generation Agent | [Example](examples/python/image/image-agent.py) | [📖](https://docs.praison.ai/features/image-generation) |
|
||||
| ↳ Image to Text Agent | [Example](examples/python/agents/image-to-text-agent.py) | [📖](https://docs.praison.ai/agents/image-to-text) |
|
||||
| ↳ Video Agent | [Example](examples/python/agents/video-agent.py) | [📖](https://docs.praison.ai/agents/video) |
|
||||
| ↳ Camera Integration | [Example](examples/python/camera/) | [📖](https://docs.praison.ai/features/camera-integration) |
|
||||
| **🔌 Protocols & Integration** | | |
|
||||
| ↳ MCP Transports | [Example](examples/python/mcp/mcp-transports-overview.py) | [📖](https://docs.praison.ai/mcp/transports) |
|
||||
| ↳ WebSocket MCP | [Example](examples/python/mcp/websocket-mcp.py) | [📖](https://docs.praison.ai/mcp/sse-transport) |
|
||||
| ↳ MCP Security | [Example](examples/python/mcp/mcp-security.py) | [📖](https://docs.praison.ai/mcp/transports) |
|
||||
| ↳ MCP Resumability | [Example](examples/python/mcp/mcp-resumability.py) | [📖](https://docs.praison.ai/mcp/sse-transport) |
|
||||
| ↳ MCP Config Management | [Example](#mcp-config-cli) | [📖](https://docs.praison.ai/docs/cli/mcp) |
|
||||
| ↳ LangChain Integrated Agents | [Example](examples/python/general/langchain_example.py) | [📖](https://docs.praison.ai/features/langchain) |
|
||||
| **🛡️ Safety & Control** | | |
|
||||
| ↳ Guardrails | [Example](examples/python/guardrails/comprehensive-guardrails-example.py) | [📖](https://docs.praison.ai/features/guardrails) |
|
||||
| ↳ Human Approval | [Example](examples/python/general/human_approval_example.py) | [📖](https://docs.praison.ai/features/approval) |
|
||||
| ↳ Rules & Instructions | [Example](#6-rules--instructions) | [📖](https://docs.praison.ai/features/rules) |
|
||||
| **⚙️ Advanced Features** | | |
|
||||
| ↳ Async & Parallel Processing | [Example](examples/python/general/async_example.py) | [📖](https://docs.praison.ai/features/async) |
|
||||
| ↳ Parallelisation | [Example](examples/python/general/parallelisation.py) | [📖](https://docs.praison.ai/features/parallelisation) |
|
||||
| ↳ Repetitive Agents | [Example](examples/python/concepts/repetitive-agents.py) | [📖](https://docs.praison.ai/features/repetitive) |
|
||||
| ↳ Agent Handoffs | [Example](examples/python/handoff/handoff_basic.py) | [📖](https://docs.praison.ai/features/handoffs) |
|
||||
| ↳ Stateful Agents | [Example](examples/python/stateful/workflow-state-example.py) | [📖](https://docs.praison.ai/features/stateful-agents) |
|
||||
| ↳ Autonomous Workflow | [Example](examples/python/general/autonomous-agent.py) | [📖](https://docs.praison.ai/features/autonomous-workflow) |
|
||||
| ↳ Structured Output Agents | [Example](examples/python/general/structured_agents_example.py) | [📖](https://docs.praison.ai/features/structured) |
|
||||
| ↳ Model Router | [Example](examples/python/agents/router-agent-cost-optimization.py) | [📖](https://docs.praison.ai/features/model-router) |
|
||||
| ↳ Prompt Caching | [Example](#web-search-web-fetch--prompt-caching) | [📖](https://docs.praison.ai/features/model-capabilities) |
|
||||
| ↳ Fast Context | [Example](examples/context/00_agent_fast_context_basic.py) | [📖](https://docs.praison.ai/features/fast-context) |
|
||||
| **🛠️ Tools & Configuration** | | |
|
||||
| ↳ 100+ Custom Tools | [Example](examples/python/general/tools_example.py) | [📖](https://docs.praison.ai/tools/tools) |
|
||||
| ↳ YAML Configuration | [Example](examples/cookbooks/yaml/secondary_market_research_agents.yaml) | [📖](https://docs.praison.ai/developers/agents-playbook) |
|
||||
| ↳ 100+ LLM Support | [Example](examples/python/providers/openai/openai_gpt4_example.py) | [📖](https://docs.praison.ai/models) |
|
||||
| ↳ Callback Agents | [Example](examples/python/general/advanced-callback-systems.py) | [📖](https://docs.praison.ai/features/callbacks) |
|
||||
| ↳ Hooks | [Example](#9-hooks) | [📖](https://docs.praison.ai/features/hooks) |
|
||||
| **📊 Monitoring & Management** | | |
|
||||
| ↳ Sessions Management | [Example](examples/python/sessions/comprehensive-session-management.py) | [📖](https://docs.praison.ai/features/sessions) |
|
||||
| ↳ Auto-Save Sessions | [Example](#session-management-python) | [📖](https://docs.praison.ai/docs/cli/session) |
|
||||
| ↳ History in Context | [Example](#session-management-python) | [📖](https://docs.praison.ai/docs/cli/session) |
|
||||
| ↳ Telemetry | [Example](examples/python/telemetry/production-telemetry-example.py) | [📖](https://docs.praison.ai/features/telemetry) |
|
||||
| ↳ Project Docs (.praison/docs/) | [Example](#docs-cli) | [📖](https://docs.praison.ai/docs/cli/docs) |
|
||||
| ↳ AI Commit Messages | [Example](#ai-commit-cli) | [📖](https://docs.praison.ai/docs/cli/commit) |
|
||||
| ↳ @Mentions in Prompts | [Example](#mentions-in-prompts) | [📖](https://docs.praison.ai/docs/cli/mentions) |
|
||||
| **🖥️ CLI Features** | | |
|
||||
| ↳ Slash Commands | [Example](examples/python/cli/slash_commands_example.py) | [📖](https://docs.praison.ai/docs/cli/slash-commands) |
|
||||
| ↳ Autonomy Modes | [Example](examples/python/cli/autonomy_modes_example.py) | [📖](https://docs.praison.ai/docs/cli/autonomy-modes) |
|
||||
@@ -266,6 +277,17 @@ npm install praisonai
|
||||
| ↳ Interactive TUI | [Example](examples/python/cli/interactive_tui_example.py) | [📖](https://docs.praison.ai/docs/cli/interactive-tui) |
|
||||
| ↳ Git Integration | [Example](examples/python/cli/git_integration_example.py) | [📖](https://docs.praison.ai/docs/cli/git-integration) |
|
||||
| ↳ Sandbox Execution | [Example](examples/python/cli/sandbox_execution_example.py) | [📖](https://docs.praison.ai/docs/cli/sandbox-execution) |
|
||||
| ↳ CLI Compare | [Example](examples/compare/cli_compare_basic.py) | [📖](https://docs.praison.ai/docs/cli/compare) |
|
||||
| **🧪 Evaluation** | | |
|
||||
| ↳ Accuracy Evaluation | [Example](examples/eval/accuracy_example.py) | [📖](https://docs.praison.ai/docs/cli/eval) |
|
||||
| ↳ Performance Evaluation | [Example](examples/eval/performance_example.py) | [📖](https://docs.praison.ai/docs/cli/eval) |
|
||||
| ↳ Reliability Evaluation | [Example](examples/eval/reliability_example.py) | [📖](https://docs.praison.ai/docs/cli/eval) |
|
||||
| ↳ Criteria Evaluation | [Example](examples/eval/criteria_example.py) | [📖](https://docs.praison.ai/docs/cli/eval) |
|
||||
| **🎯 Agent Skills** | | |
|
||||
| ↳ Skills Management | [Example](examples/skills/basic_skill_usage.py) | [📖](https://docs.praison.ai/features/skills) |
|
||||
| ↳ Custom Skills | [Example](examples/skills/custom_skill_example.py) | [📖](https://docs.praison.ai/features/skills) |
|
||||
| **⏰ 24/7 Scheduling** | | |
|
||||
| ↳ Agent Scheduler | [Example](examples/python/scheduled_agents/news_checker_live.py) | [📖](https://docs.praison.ai/docs/cli/scheduler) |
|
||||
|
||||
---
|
||||
|
||||
@@ -1058,6 +1080,20 @@ praisonai "Complex analysis task" --planning --planning-reasoning
|
||||
praisonai "Task" --planning --auto-approve-plan
|
||||
```
|
||||
|
||||
### Tool Approval CLI:
|
||||
```bash
|
||||
# Auto-approve ALL tool executions (use with caution!)
|
||||
praisonai "run ls command" --trust
|
||||
|
||||
# Auto-approve tools up to a risk level (prompt for higher)
|
||||
# Levels: low, medium, high, critical
|
||||
praisonai "write to file" --approve-level high # Prompts for critical tools only
|
||||
praisonai "task" --approve-level medium # Prompts for high and critical
|
||||
|
||||
# Default behavior (no flags): prompts for all dangerous tools
|
||||
praisonai "run shell command" # Will prompt for approval
|
||||
```
|
||||
|
||||
### Memory CLI:
|
||||
```bash
|
||||
# Enable memory for agent (persists across sessions)
|
||||
@@ -1228,6 +1264,21 @@ praisonai "Analyze this data" --metrics
|
||||
praisonai "Complex task" --metrics --planning
|
||||
```
|
||||
|
||||
### Scheduler CLI:
|
||||
|
||||
```bash
|
||||
praisonai schedule start <name> "task" --interval hourly
|
||||
praisonai schedule list
|
||||
praisonai schedule logs <name> [--follow]
|
||||
praisonai schedule stop <name>
|
||||
praisonai schedule restart <name>
|
||||
praisonai schedule delete <name>
|
||||
praisonai schedule describe <name>
|
||||
praisonai schedule save <name> [file.yaml]
|
||||
praisonai schedule "task" --interval hourly # foreground mode
|
||||
praisonai schedule agents.yaml # foreground mode
|
||||
```
|
||||
|
||||
### Image Processing CLI:
|
||||
```bash
|
||||
# Process images with vision-based tasks
|
||||
@@ -2399,6 +2450,7 @@ PraisonAI provides zero-dependency persistent memory for agents. For detailed ex
|
||||
- 🤝 **Agent Handoffs** - Transfer context between specialised agents
|
||||
- 🛡️ **Guardrails** - Input/output validation and safety checks
|
||||
- ✅ **Human Approval** - Require human confirmation for critical actions
|
||||
- 🔐 **Tool Approval CLI** - `--trust` (auto-approve all) and `--approve-level` (risk-based approval)
|
||||
- 💬 **Sessions Management** - Isolated conversation contexts
|
||||
- 🔄 **Stateful Agents** - Maintain state across interactions
|
||||
|
||||
|
||||
@@ -16,7 +16,7 @@ RUN mkdir -p /root/.praison
|
||||
# Install Python packages (using latest versions)
|
||||
RUN pip install --no-cache-dir \
|
||||
praisonai_tools \
|
||||
"praisonai>=2.3.52" \
|
||||
"praisonai>=2.3.77" \
|
||||
"praisonai[chat]" \
|
||||
"embedchain[github,youtube]"
|
||||
|
||||
|
||||
@@ -20,7 +20,7 @@ RUN mkdir -p /root/.praison
|
||||
# Install Python packages (using latest versions)
|
||||
RUN pip install --no-cache-dir \
|
||||
praisonai_tools \
|
||||
"praisonai>=2.3.52" \
|
||||
"praisonai>=2.3.77" \
|
||||
"praisonai[ui]" \
|
||||
"praisonai[chat]" \
|
||||
"praisonai[realtime]" \
|
||||
|
||||
@@ -16,7 +16,7 @@ RUN mkdir -p /root/.praison
|
||||
# Install Python packages (using latest versions)
|
||||
RUN pip install --no-cache-dir \
|
||||
praisonai_tools \
|
||||
"praisonai>=2.3.52" \
|
||||
"praisonai>=2.3.77" \
|
||||
"praisonai[ui]" \
|
||||
"praisonai[crewai]"
|
||||
|
||||
|
||||
@@ -0,0 +1,85 @@
|
||||
"""
|
||||
CLI Compare Example - Advanced Usage
|
||||
|
||||
This example demonstrates advanced comparison features including
|
||||
custom mode configurations and result analysis.
|
||||
"""
|
||||
|
||||
from praisonai.cli.features.compare import (
|
||||
CompareHandler,
|
||||
CompareResult,
|
||||
ModeResult,
|
||||
get_mode_config,
|
||||
save_compare_result,
|
||||
format_comparison_table,
|
||||
)
|
||||
|
||||
# Create custom mode configurations
|
||||
custom_modes = {
|
||||
"fast": {"llm": "gpt-4o-mini"},
|
||||
"accurate": {"llm": "gpt-4o", "planning": True},
|
||||
"research": {"research": True, "web_search": True},
|
||||
}
|
||||
|
||||
# Create handler with verbose output
|
||||
handler = CompareHandler(verbose=True)
|
||||
|
||||
# Example: Manually create comparison results for analysis
|
||||
# (useful for testing without API calls)
|
||||
mock_result = CompareResult(
|
||||
query="What are the latest AI trends?",
|
||||
comparisons=[
|
||||
ModeResult(
|
||||
mode="basic",
|
||||
output="AI trends include LLMs, multimodal models, and agents.",
|
||||
execution_time_ms=1234.5,
|
||||
model_used="gpt-4o-mini",
|
||||
tools_used=[]
|
||||
),
|
||||
ModeResult(
|
||||
mode="tools",
|
||||
output="Based on recent search results, AI trends in 2024 include...",
|
||||
execution_time_ms=2567.8,
|
||||
model_used="gpt-4o-mini",
|
||||
tools_used=["internet_search"]
|
||||
),
|
||||
ModeResult(
|
||||
mode="research",
|
||||
output="Comprehensive research shows that AI trends...",
|
||||
execution_time_ms=5432.1,
|
||||
model_used="gpt-4o-mini",
|
||||
tools_used=["internet_search", "web_scraper"]
|
||||
),
|
||||
]
|
||||
)
|
||||
|
||||
# Get summary statistics
|
||||
summary = mock_result.get_summary()
|
||||
print("Comparison Summary:")
|
||||
print(f" Fastest mode: {summary['fastest']} ({summary['fastest_time_ms']:.1f}ms)")
|
||||
print(f" Slowest mode: {summary['slowest']} ({summary['slowest_time_ms']:.1f}ms)")
|
||||
|
||||
# Format as table
|
||||
print("\n" + format_comparison_table(mock_result))
|
||||
|
||||
# Convert to JSON for storage
|
||||
print("\nJSON Output:")
|
||||
print(mock_result.to_json())
|
||||
|
||||
# Save to file
|
||||
# save_compare_result(mock_result, "advanced_comparison.json")
|
||||
|
||||
print("\n--- CLI Usage Examples ---")
|
||||
print("""
|
||||
# Basic comparison
|
||||
praisonai "What is AI?" --compare "basic,tools"
|
||||
|
||||
# Compare with specific model
|
||||
praisonai "Explain quantum computing" --compare "basic,planning,research" --model gpt-4o
|
||||
|
||||
# Save results to file
|
||||
praisonai "Latest tech news" --compare "basic,tools,web_search" --compare-output results.json
|
||||
|
||||
# Verbose output
|
||||
praisonai "Write a poem" --compare "basic,planning" --verbose
|
||||
""")
|
||||
@@ -0,0 +1,45 @@
|
||||
"""
|
||||
CLI Compare Example - Basic Usage
|
||||
|
||||
This example demonstrates how to compare different CLI modes programmatically.
|
||||
"""
|
||||
|
||||
from praisonai.cli.features.compare import (
|
||||
CompareHandler,
|
||||
get_mode_config,
|
||||
list_available_modes,
|
||||
parse_modes,
|
||||
)
|
||||
|
||||
# List all available comparison modes
|
||||
print("Available modes:")
|
||||
for mode in list_available_modes():
|
||||
config = get_mode_config(mode)
|
||||
print(f" - {mode}: {config}")
|
||||
|
||||
# Parse modes from comma-separated string
|
||||
modes = parse_modes("basic,tools,planning")
|
||||
print(f"\nParsed modes: {modes}")
|
||||
|
||||
# Create compare handler
|
||||
handler = CompareHandler(verbose=True)
|
||||
|
||||
# Compare modes (this will run actual agent calls)
|
||||
# Uncomment to run:
|
||||
# result = handler.compare(
|
||||
# query="What is artificial intelligence?",
|
||||
# modes=["basic", "tools"],
|
||||
# model="gpt-4o-mini"
|
||||
# )
|
||||
# handler.print_result(result)
|
||||
|
||||
# Save results to file
|
||||
# result = handler.execute(
|
||||
# query="Explain machine learning",
|
||||
# modes_str="basic,planning",
|
||||
# output_path="comparison_results.json"
|
||||
# )
|
||||
|
||||
print("\nTo run comparison via CLI:")
|
||||
print(' praisonai "What is AI?" --compare "basic,tools,planning"')
|
||||
print(' praisonai "What is AI?" --compare "basic,research" --compare-output results.json')
|
||||
@@ -0,0 +1,23 @@
|
||||
# Example: Data Collection Agent with 30-minute Schedule
|
||||
# Collects and analyzes data every 30 minutes
|
||||
|
||||
framework: praisonai
|
||||
|
||||
agents:
|
||||
- name: "Data Collector"
|
||||
role: "Data Analysis Specialist"
|
||||
goal: "Collect and analyze market data at regular intervals"
|
||||
instructions: |
|
||||
You are a data analysis specialist focused on collecting and analyzing
|
||||
market trends and metrics. Provide actionable insights from the data.
|
||||
tools:
|
||||
- search_tool
|
||||
verbose: false
|
||||
|
||||
task: "Collect latest market data and identify key trends"
|
||||
|
||||
# Schedule: Every 30 minutes with immediate execution
|
||||
schedule:
|
||||
interval: "*/30m"
|
||||
max_retries: 5
|
||||
run_immediately: true
|
||||
@@ -0,0 +1,53 @@
|
||||
# Example: 24/7 AI News Monitor with Scheduler
|
||||
# This agent runs continuously at regular intervals to monitor AI news
|
||||
|
||||
framework: praisonai
|
||||
|
||||
agents:
|
||||
- name: "AI News Monitor"
|
||||
role: "Technology News Analyst"
|
||||
goal: "Monitor and summarize the latest AI developments"
|
||||
instructions: |
|
||||
You are an expert technology news analyst specializing in AI and machine learning.
|
||||
Your job is to search for the latest AI news, identify the most important developments,
|
||||
and provide clear, concise summaries with sources.
|
||||
|
||||
Focus on:
|
||||
- Major AI product launches
|
||||
- Research breakthroughs
|
||||
- Industry trends
|
||||
- Policy and regulation updates
|
||||
tools:
|
||||
- search_tool
|
||||
verbose: true
|
||||
|
||||
# Task to execute on each scheduled run
|
||||
task: |
|
||||
Search for the latest AI news from the past 24 hours.
|
||||
Provide the top 3 most important stories with:
|
||||
1. Headline
|
||||
2. Brief summary (2-3 sentences)
|
||||
3. Source URL
|
||||
4. Why it's significant
|
||||
|
||||
# Schedule configuration
|
||||
schedule:
|
||||
# Interval options:
|
||||
# - "hourly" - Every hour
|
||||
# - "daily" - Every 24 hours
|
||||
# - "*/30m" - Every 30 minutes
|
||||
# - "*/6h" - Every 6 hours
|
||||
# - "3600" - Custom seconds
|
||||
interval: "hourly"
|
||||
|
||||
# Maximum retry attempts if execution fails
|
||||
max_retries: 3
|
||||
|
||||
# Run immediately when scheduler starts (before first scheduled interval)
|
||||
run_immediately: true
|
||||
|
||||
# Optional: Variables for customization
|
||||
variables:
|
||||
news_category: "AI and Machine Learning"
|
||||
region: "Global"
|
||||
time_range: "24 hours"
|
||||
@@ -0,0 +1,69 @@
|
||||
"""
|
||||
Accuracy Evaluation Example
|
||||
|
||||
This example demonstrates how to evaluate agent output accuracy
|
||||
by comparing against expected outputs using LLM-as-judge.
|
||||
"""
|
||||
|
||||
import os
|
||||
from praisonaiagents import Agent
|
||||
from praisonaiagents.eval import AccuracyEvaluator
|
||||
|
||||
# Check if we have an API key
|
||||
has_api_key = os.getenv("OPENAI_API_KEY") is not None
|
||||
|
||||
if has_api_key:
|
||||
# Create a simple math tutor agent
|
||||
agent = Agent(
|
||||
instructions="You are a math tutor. Answer math questions concisely with just the number."
|
||||
)
|
||||
|
||||
# Create accuracy evaluator
|
||||
evaluator = AccuracyEvaluator(
|
||||
agent=agent,
|
||||
input_text="What is 2 + 2?",
|
||||
expected_output="4",
|
||||
num_iterations=3, # Run 3 times for statistical significance
|
||||
verbose=True
|
||||
)
|
||||
|
||||
# Run evaluation
|
||||
result = evaluator.run(print_summary=True)
|
||||
|
||||
# Check results
|
||||
print(f"\nAverage Score: {result.avg_score}/10")
|
||||
print(f"Passed: {result.passed}")
|
||||
else:
|
||||
print("⚠️ No OPENAI_API_KEY found. Using mock evaluation...")
|
||||
|
||||
# Create evaluator without agent for mock testing
|
||||
evaluator = AccuracyEvaluator(
|
||||
func=lambda x: "4", # Mock function
|
||||
input_text="What is 2 + 2?",
|
||||
expected_output="4",
|
||||
num_iterations=1,
|
||||
verbose=True
|
||||
)
|
||||
|
||||
# Test with mock output
|
||||
try:
|
||||
result = evaluator.run(print_summary=True)
|
||||
print(f"\nMock Average Score: {result.avg_score}/10")
|
||||
print(f"Mock Passed: {result.passed}")
|
||||
except Exception as e:
|
||||
print(f"Mock evaluation failed: {e}")
|
||||
print("This is expected without an API key")
|
||||
|
||||
# You can also evaluate pre-generated outputs
|
||||
print("\n--- Testing pre-generated output evaluation ---")
|
||||
output = "The answer is 4"
|
||||
|
||||
if has_api_key:
|
||||
try:
|
||||
result2 = evaluator.evaluate_output(output, print_summary=True)
|
||||
print(f"Pre-generated output score: {result2.avg_score}/10")
|
||||
except Exception as e:
|
||||
print(f"Pre-generated evaluation failed: {e}")
|
||||
else:
|
||||
print("⚠️ Skipping pre-generated evaluation (no API key)")
|
||||
print("To run full evaluation, set OPENAI_API_KEY environment variable")
|
||||
@@ -0,0 +1,105 @@
|
||||
"""
|
||||
Batch Evaluation Example
|
||||
|
||||
This example demonstrates how to run batch evaluations
|
||||
from a JSON test file.
|
||||
"""
|
||||
|
||||
import os
|
||||
import json
|
||||
import tempfile
|
||||
from praisonaiagents.eval import AccuracyEvaluator
|
||||
|
||||
# Check if we have an API key
|
||||
has_api_key = os.getenv("OPENAI_API_KEY") is not None
|
||||
|
||||
# Create a test file with multiple test cases
|
||||
test_cases = [
|
||||
{
|
||||
"input": "What is 2 + 2?",
|
||||
"expected": "4"
|
||||
},
|
||||
{
|
||||
"input": "What is the capital of France?",
|
||||
"expected": "Paris"
|
||||
},
|
||||
{
|
||||
"input": "What color is the sky?",
|
||||
"expected": "Blue"
|
||||
}
|
||||
]
|
||||
|
||||
# Save test cases to a temporary file
|
||||
with tempfile.NamedTemporaryFile(mode='w', suffix='.json', delete=False) as f:
|
||||
json.dump(test_cases, f)
|
||||
test_file = f.name
|
||||
|
||||
print(f"Test file created: {test_file}")
|
||||
print(f"Running {len(test_cases)} test cases...")
|
||||
|
||||
if has_api_key:
|
||||
# You can use the CLI for batch evaluation:
|
||||
# praisonai eval batch --agent agents.yaml --test-file tests.json --batch-type accuracy
|
||||
|
||||
# Or programmatically:
|
||||
from praisonaiagents import Agent
|
||||
|
||||
agent = Agent(
|
||||
instructions="You are a helpful assistant. Answer questions concisely."
|
||||
)
|
||||
|
||||
results = []
|
||||
for i, test_case in enumerate(test_cases):
|
||||
print(f"\nTest {i + 1}: {test_case['input']}")
|
||||
|
||||
evaluator = AccuracyEvaluator(
|
||||
agent=agent,
|
||||
input_text=test_case["input"],
|
||||
expected_output=test_case["expected"],
|
||||
num_iterations=1
|
||||
)
|
||||
|
||||
result = evaluator.run()
|
||||
results.append({
|
||||
"input": test_case["input"],
|
||||
"expected": test_case["expected"],
|
||||
"score": result.avg_score,
|
||||
"passed": result.passed
|
||||
})
|
||||
|
||||
print(f" Score: {result.avg_score}/10")
|
||||
print(f" Passed: {result.passed}")
|
||||
|
||||
# Summary
|
||||
passed = sum(1 for r in results if r["passed"])
|
||||
print(f"\n{'='*50}")
|
||||
print(f"Summary: {passed}/{len(results)} tests passed")
|
||||
print(f"Pass Rate: {passed/len(results):.1%}")
|
||||
else:
|
||||
print("⚠️ No OPENAI_API_KEY found. Running mock batch evaluation...")
|
||||
|
||||
# Mock evaluation without API key
|
||||
results = []
|
||||
for i, test_case in enumerate(test_cases):
|
||||
print(f"\nTest {i + 1}: {test_case['input']}")
|
||||
|
||||
# Mock result (all would fail without API key)
|
||||
mock_score = 0.0
|
||||
mock_passed = False
|
||||
|
||||
results.append({
|
||||
"input": test_case["input"],
|
||||
"expected": test_case["expected"],
|
||||
"score": mock_score,
|
||||
"passed": mock_passed
|
||||
})
|
||||
|
||||
print(f" Score: {mock_score}/10 (mock)")
|
||||
print(f" Passed: {mock_passed} (expected without API key)")
|
||||
|
||||
# Summary
|
||||
passed = sum(1 for r in results if r["passed"])
|
||||
print(f"\n{'='*50}")
|
||||
print(f"Summary: {passed}/{len(results)} tests passed (mock evaluation)")
|
||||
print(f"Pass Rate: {passed/len(results):.1%}")
|
||||
print("\nTo run real batch evaluation, set OPENAI_API_KEY environment variable")
|
||||
@@ -0,0 +1,82 @@
|
||||
"""
|
||||
Criteria Evaluation Example
|
||||
|
||||
This example demonstrates how to evaluate agent outputs
|
||||
against custom criteria using LLM-as-judge.
|
||||
"""
|
||||
|
||||
import os
|
||||
from praisonaiagents import Agent
|
||||
from praisonaiagents.eval import CriteriaEvaluator
|
||||
|
||||
# Check if we have an API key
|
||||
has_api_key = os.getenv("OPENAI_API_KEY") is not None
|
||||
|
||||
if has_api_key:
|
||||
print("--- Testing Criteria Evaluation with Agent ---")
|
||||
# Create a customer service agent
|
||||
agent = Agent(
|
||||
instructions="You are a friendly customer service agent. Be helpful and empathetic."
|
||||
)
|
||||
|
||||
# Create criteria evaluator with numeric scoring
|
||||
evaluator = CriteriaEvaluator(
|
||||
criteria="Response is helpful, empathetic, and provides a clear solution",
|
||||
agent=agent,
|
||||
input_text="I'm frustrated because my order hasn't arrived yet.",
|
||||
scoring_type="numeric", # Score 1-10
|
||||
threshold=7.0, # Pass if score >= 7
|
||||
num_iterations=2,
|
||||
verbose=True
|
||||
)
|
||||
|
||||
# Run evaluation
|
||||
result = evaluator.run(print_summary=True)
|
||||
|
||||
print("\nNumeric Scoring Results:")
|
||||
print(f" Average Score: {result.avg_score}/10")
|
||||
print(f" Pass Rate: {result.pass_rate:.1%}")
|
||||
print(f" Passed: {result.passed}")
|
||||
|
||||
# Binary scoring example (pass/fail)
|
||||
print("\n--- Testing Binary Scoring ---")
|
||||
binary_evaluator = CriteriaEvaluator(
|
||||
criteria="Response does not contain any offensive language",
|
||||
agent=agent,
|
||||
input_text="Tell me a joke",
|
||||
scoring_type="binary", # Pass or Fail
|
||||
verbose=True
|
||||
)
|
||||
|
||||
binary_result = binary_evaluator.run(print_summary=True)
|
||||
|
||||
# With on_fail callback
|
||||
print("\n--- Testing Failure Callback ---")
|
||||
def handle_failure(score):
|
||||
print(f"ALERT: Evaluation failed with score {score.score}")
|
||||
print(f"Reasoning: {score.reasoning}")
|
||||
|
||||
callback_evaluator = CriteriaEvaluator(
|
||||
criteria="Response is professional and helpful",
|
||||
agent=agent,
|
||||
input_text="Help me",
|
||||
on_fail=handle_failure,
|
||||
threshold=8.0
|
||||
)
|
||||
|
||||
callback_evaluator.run()
|
||||
else:
|
||||
print("⚠️ No OPENAI_API_KEY found. Skipping agent-based criteria evaluation...")
|
||||
|
||||
# Test pre-generated output evaluation (doesn't need agent)
|
||||
print("\n--- Testing Pre-generated Output Evaluation ---")
|
||||
if has_api_key:
|
||||
output = "I understand your frustration. Let me check on your order right away."
|
||||
try:
|
||||
result2 = evaluator.evaluate_output(output, print_summary=True)
|
||||
print(f"Pre-generated output score: {result2.avg_score}/10")
|
||||
except Exception as e:
|
||||
print(f"Pre-generated evaluation failed: {e}")
|
||||
else:
|
||||
print("⚠️ Skipping pre-generated evaluation (no API key)")
|
||||
print("To run full evaluation, set OPENAI_API_KEY environment variable")
|
||||
@@ -0,0 +1,59 @@
|
||||
"""
|
||||
Performance Evaluation Example
|
||||
|
||||
This example demonstrates how to benchmark agent performance
|
||||
by measuring runtime and memory usage across multiple iterations.
|
||||
"""
|
||||
|
||||
import os
|
||||
from praisonaiagents import Agent
|
||||
from praisonaiagents.eval import PerformanceEvaluator
|
||||
|
||||
# Check if we have an API key
|
||||
has_api_key = os.getenv("OPENAI_API_KEY") is not None
|
||||
|
||||
if has_api_key:
|
||||
print("--- Testing Agent Performance ---")
|
||||
# Create a simple agent
|
||||
agent = Agent(
|
||||
instructions="You are a helpful assistant. Answer questions briefly."
|
||||
)
|
||||
|
||||
# Create performance evaluator
|
||||
evaluator = PerformanceEvaluator(
|
||||
agent=agent,
|
||||
input_text="What is the capital of France?",
|
||||
num_iterations=10, # Run 10 benchmark iterations
|
||||
warmup_runs=2, # 2 warmup runs before measurement
|
||||
track_memory=True, # Track memory usage
|
||||
verbose=True
|
||||
)
|
||||
|
||||
# Run evaluation
|
||||
result = evaluator.run(print_summary=True)
|
||||
|
||||
# Access detailed metrics
|
||||
print("\nAgent Performance Results:")
|
||||
print(f" Average Time: {result.avg_run_time:.4f}s")
|
||||
print(f" Min Time: {result.min_run_time:.4f}s")
|
||||
print(f" Max Time: {result.max_run_time:.4f}s")
|
||||
print(f" Median Time: {result.median_run_time:.4f}s")
|
||||
print(f" P95 Time: {result.p95_run_time:.4f}s")
|
||||
print(f" Avg Memory: {result.avg_memory:.2f} MB")
|
||||
else:
|
||||
print("⚠️ No OPENAI_API_KEY found. Skipping agent performance test...")
|
||||
|
||||
# You can also benchmark any function
|
||||
def my_function():
|
||||
import time
|
||||
time.sleep(0.1)
|
||||
return "done"
|
||||
|
||||
func_evaluator = PerformanceEvaluator(
|
||||
func=my_function,
|
||||
num_iterations=5,
|
||||
warmup_runs=1,
|
||||
track_memory=False
|
||||
)
|
||||
|
||||
func_result = func_evaluator.run(print_summary=True)
|
||||
@@ -0,0 +1,72 @@
|
||||
"""
|
||||
Reliability Evaluation Example
|
||||
|
||||
This example demonstrates how to verify that agents call
|
||||
the expected tools during execution.
|
||||
"""
|
||||
|
||||
import os
|
||||
from praisonaiagents import Agent
|
||||
from praisonaiagents.eval import ReliabilityEvaluator
|
||||
|
||||
|
||||
def search_web(query: str) -> str:
|
||||
"""Search the web for information."""
|
||||
return f"Search results for: {query}"
|
||||
|
||||
|
||||
def calculate(expression: str) -> str:
|
||||
"""Calculate a math expression."""
|
||||
return str(eval(expression))
|
||||
|
||||
|
||||
# Check if we have an API key
|
||||
has_api_key = os.getenv("OPENAI_API_KEY") is not None
|
||||
|
||||
if has_api_key:
|
||||
print("--- Testing Agent Tool Call Reliability ---")
|
||||
# Create an agent with tools
|
||||
agent = Agent(
|
||||
instructions="You are a helpful assistant with access to search and calculator tools.",
|
||||
tools=[search_web, calculate]
|
||||
)
|
||||
|
||||
# Create reliability evaluator
|
||||
evaluator = ReliabilityEvaluator(
|
||||
agent=agent,
|
||||
input_text="Search for the weather in Paris and calculate 25 * 4",
|
||||
expected_tools=["search_web", "calculate"], # Tools that should be called
|
||||
forbidden_tools=["delete_file"], # Tools that should NOT be called
|
||||
verbose=True
|
||||
)
|
||||
|
||||
# Run evaluation
|
||||
result = evaluator.run(print_summary=True)
|
||||
|
||||
# Check results
|
||||
print("\nAgent Reliability Results:")
|
||||
print(f" Passed: {result.passed}")
|
||||
print(f" Pass Rate: {result.pass_rate:.1%}")
|
||||
else:
|
||||
print("⚠️ No OPENAI_API_KEY found. Skipping agent reliability test...")
|
||||
print("Agent would fail to call tools without API key (expected behavior)")
|
||||
|
||||
# You can also evaluate pre-recorded tool calls (doesn't need API key)
|
||||
print("\n--- Testing Pre-recorded Tool Call Evaluation ---")
|
||||
# Create a mock agent for tool call evaluation
|
||||
mock_agent = Agent(instructions="Mock agent")
|
||||
evaluator = ReliabilityEvaluator(
|
||||
agent=mock_agent,
|
||||
expected_tools=["search_web", "calculate"], # Tools that should be called
|
||||
forbidden_tools=["delete_file"], # Tools that should NOT be called
|
||||
verbose=True
|
||||
)
|
||||
|
||||
result2 = evaluator.evaluate_tool_calls(
|
||||
actual_tools=["search_web", "calculate"],
|
||||
print_summary=True
|
||||
)
|
||||
|
||||
print("\nPre-recorded Tool Call Results:")
|
||||
print(f" Passed: {result2.passed}")
|
||||
print(f" Pass Rate: {result2.pass_rate:.1%}")
|
||||
@@ -0,0 +1,115 @@
|
||||
"""
|
||||
24/7 News Checker Agent Example
|
||||
|
||||
This example demonstrates how to create a continuously running agent
|
||||
that checks for AI news every hour and prints the results.
|
||||
|
||||
Usage:
|
||||
python news_checker_agent.py
|
||||
|
||||
The agent will:
|
||||
1. Run immediately on start
|
||||
2. Check for latest AI news every 1 hour
|
||||
3. Print results to console
|
||||
4. Continue running until stopped (Ctrl+C)
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
from datetime import datetime
|
||||
|
||||
# Add parent directory to path for imports
|
||||
sys.path.insert(0, os.path.join(os.path.dirname(__file__), '../../..'))
|
||||
|
||||
from praisonaiagents import Agent
|
||||
from praisonai.agent_scheduler import AgentScheduler
|
||||
from tools import search_tool
|
||||
|
||||
|
||||
def on_success(result):
|
||||
"""Callback function called when agent execution succeeds."""
|
||||
print("\n" + "="*80)
|
||||
print(f"✅ SUCCESS - {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}")
|
||||
print("="*80)
|
||||
print(result)
|
||||
print("="*80 + "\n")
|
||||
|
||||
|
||||
def on_failure(error):
|
||||
"""Callback function called when agent execution fails."""
|
||||
print("\n" + "="*80)
|
||||
print(f"❌ FAILURE - {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}")
|
||||
print("="*80)
|
||||
print(f"Error: {error}")
|
||||
print("="*80 + "\n")
|
||||
|
||||
|
||||
def main():
|
||||
"""Main function to set up and run the scheduled news checker agent."""
|
||||
|
||||
print("\n🤖 Starting 24/7 News Checker Agent")
|
||||
print("="*80)
|
||||
|
||||
# Create the agent with web search capability
|
||||
agent = Agent(
|
||||
name="NewsChecker",
|
||||
role="AI News Monitor",
|
||||
goal="Monitor and summarize the latest AI and technology news",
|
||||
backstory="""You are an expert AI news analyst who stays up-to-date
|
||||
with the latest developments in artificial intelligence, machine learning,
|
||||
and technology. You provide concise, accurate summaries of important news.""",
|
||||
tools=[search_tool],
|
||||
verbose=True
|
||||
)
|
||||
|
||||
# Define the task
|
||||
task = """Search for the latest AI news from today.
|
||||
Focus on major developments, breakthroughs, or announcements.
|
||||
Provide a summary of the top 3-5 most important news items.
|
||||
Include the source and key details for each item."""
|
||||
|
||||
# Create scheduler with callbacks
|
||||
scheduler = AgentScheduler(
|
||||
agent=agent,
|
||||
task=task,
|
||||
on_success=on_success,
|
||||
on_failure=on_failure
|
||||
)
|
||||
|
||||
print(f"Agent: {agent.name}")
|
||||
print(f"Task: {task}")
|
||||
print(f"Schedule: Every 1 hour")
|
||||
print("="*80)
|
||||
print("\n⏰ Starting scheduler... (Press Ctrl+C to stop)\n")
|
||||
|
||||
# Start the scheduler
|
||||
# - "hourly" or "*/1h" = every 1 hour
|
||||
# - run_immediately=True = run once before starting the schedule
|
||||
scheduler.start(
|
||||
schedule_expr="hourly", # Run every hour
|
||||
max_retries=3, # Retry up to 3 times on failure
|
||||
run_immediately=True # Run immediately on start
|
||||
)
|
||||
|
||||
try:
|
||||
# Keep the main thread alive
|
||||
while scheduler.is_running:
|
||||
import time
|
||||
time.sleep(1)
|
||||
|
||||
except KeyboardInterrupt:
|
||||
print("\n\n🛑 Stopping scheduler...")
|
||||
scheduler.stop()
|
||||
|
||||
# Print final statistics
|
||||
stats = scheduler.get_stats()
|
||||
print("\n📊 Final Statistics:")
|
||||
print(f" Total Executions: {stats['total_executions']}")
|
||||
print(f" Successful: {stats['successful_executions']}")
|
||||
print(f" Failed: {stats['failed_executions']}")
|
||||
print(f" Success Rate: {stats['success_rate']:.1f}%")
|
||||
print("\n✅ Agent stopped successfully\n")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,143 @@
|
||||
"""
|
||||
Live News Checker Agent - 24/7 Scheduled Agent
|
||||
|
||||
This example demonstrates a real working agent that checks AI news
|
||||
every hour using the AgentScheduler with actual API calls.
|
||||
|
||||
Requirements:
|
||||
- OPENAI_API_KEY environment variable set
|
||||
- praisonaiagents package installed
|
||||
|
||||
Usage:
|
||||
python news_checker_live.py
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
from datetime import datetime
|
||||
|
||||
# Add parent directory to path
|
||||
sys.path.insert(0, os.path.join(os.path.dirname(__file__), '../../..'))
|
||||
sys.path.insert(0, os.path.join(os.path.dirname(__file__), '../../../..'))
|
||||
|
||||
from praisonaiagents import Agent
|
||||
from praisonai.scheduler import AgentScheduler
|
||||
|
||||
# Import search_tool from root tools.py
|
||||
try:
|
||||
from tools import search_tool
|
||||
except ImportError:
|
||||
# Fallback: create inline search tool
|
||||
from duckduckgo_search import DDGS
|
||||
|
||||
def search_tool(query: str) -> list:
|
||||
"""Search the web using DuckDuckGo."""
|
||||
try:
|
||||
results = []
|
||||
ddgs = DDGS()
|
||||
for result in ddgs.text(keywords=query, max_results=5):
|
||||
results.append({
|
||||
"title": result.get("title", ""),
|
||||
"url": result.get("href", ""),
|
||||
"snippet": result.get("body", ""),
|
||||
})
|
||||
return results
|
||||
except Exception as e:
|
||||
print(f"Search error: {e}")
|
||||
return []
|
||||
|
||||
|
||||
def on_success(result):
|
||||
"""Callback when agent execution succeeds."""
|
||||
print("\n" + "="*80)
|
||||
print(f"✅ SUCCESS - {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}")
|
||||
print("="*80)
|
||||
print(result)
|
||||
print("="*80 + "\n")
|
||||
|
||||
|
||||
def on_failure(error):
|
||||
"""Callback when agent execution fails."""
|
||||
print("\n" + "="*80)
|
||||
print(f"❌ FAILURE - {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}")
|
||||
print("="*80)
|
||||
print(f"Error: {error}")
|
||||
print("="*80 + "\n")
|
||||
|
||||
|
||||
def main():
|
||||
"""Main function to run the scheduled news checker."""
|
||||
|
||||
# Check for API key
|
||||
if not os.getenv("OPENAI_API_KEY"):
|
||||
print("❌ Error: OPENAI_API_KEY environment variable not set")
|
||||
print("Please set it with: export OPENAI_API_KEY=your_key_here")
|
||||
sys.exit(1)
|
||||
|
||||
print("\n🤖 Starting 24/7 AI News Checker Agent")
|
||||
print("="*80)
|
||||
|
||||
# Create the agent with web search capability
|
||||
agent = Agent(
|
||||
name="AI News Monitor",
|
||||
role="Technology News Analyst",
|
||||
goal="Monitor and summarize the latest AI and technology news",
|
||||
backstory="""You are an expert AI news analyst who stays up-to-date
|
||||
with the latest developments in artificial intelligence, machine learning,
|
||||
and technology. You provide concise, accurate summaries of important news.""",
|
||||
tools=[search_tool],
|
||||
verbose=True,
|
||||
self_reflect=False # Disable reflection for faster execution
|
||||
)
|
||||
|
||||
# Define the task
|
||||
task = """Search for the latest AI news from today.
|
||||
Focus on major developments, breakthroughs, or announcements.
|
||||
Provide a summary of the top 3 most important news items.
|
||||
Include the source and key details for each item."""
|
||||
|
||||
# Create scheduler with callbacks
|
||||
scheduler = AgentScheduler(
|
||||
agent=agent,
|
||||
task=task,
|
||||
on_success=on_success,
|
||||
on_failure=on_failure
|
||||
)
|
||||
|
||||
print(f"Agent: {agent.name}")
|
||||
print(f"Task: {task}")
|
||||
print(f"Schedule: Every 1 hour")
|
||||
print("="*80)
|
||||
print("\n⏰ Starting scheduler... (Press Ctrl+C to stop)\n")
|
||||
|
||||
# Start the scheduler
|
||||
# For testing: use "*/2m" (every 2 minutes) instead of "hourly"
|
||||
# For production: use "hourly"
|
||||
scheduler.start(
|
||||
schedule_expr="*/2m", # Every 2 minutes for testing
|
||||
max_retries=3,
|
||||
run_immediately=True # Run once immediately
|
||||
)
|
||||
|
||||
try:
|
||||
# Keep the main thread alive
|
||||
while scheduler.is_running:
|
||||
import time
|
||||
time.sleep(1)
|
||||
|
||||
except KeyboardInterrupt:
|
||||
print("\n\n🛑 Stopping scheduler...")
|
||||
scheduler.stop()
|
||||
|
||||
# Print final statistics
|
||||
stats = scheduler.get_stats()
|
||||
print("\n📊 Final Statistics:")
|
||||
print(f" Total Executions: {stats['total_executions']}")
|
||||
print(f" Successful: {stats['successful_executions']}")
|
||||
print(f" Failed: {stats['failed_executions']}")
|
||||
print(f" Success Rate: {stats['success_rate']:.1f}%")
|
||||
print("\n✅ Agent stopped successfully\n")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,61 @@
|
||||
"""
|
||||
Simple test script for 24/7 Agent Scheduler
|
||||
|
||||
This script demonstrates the basic usage of AgentScheduler
|
||||
for running agents periodically.
|
||||
|
||||
Usage:
|
||||
python simple_test.py
|
||||
"""
|
||||
|
||||
import sys
|
||||
import os
|
||||
sys.path.insert(0, os.path.join(os.path.dirname(__file__), '../../..'))
|
||||
|
||||
from unittest.mock import Mock
|
||||
from praisonai.scheduler import AgentScheduler
|
||||
|
||||
# Create a mock agent for testing
|
||||
mock_agent = Mock()
|
||||
mock_agent.name = "TestAgent"
|
||||
mock_agent.start = Mock(return_value="Test execution successful!")
|
||||
|
||||
# Create scheduler
|
||||
scheduler = AgentScheduler(
|
||||
agent=mock_agent,
|
||||
task="Test task execution"
|
||||
)
|
||||
|
||||
print("🤖 Testing AgentScheduler")
|
||||
print("=" * 60)
|
||||
|
||||
# Test 1: Execute once
|
||||
print("\n✅ Test 1: Execute once")
|
||||
result = scheduler.execute_once()
|
||||
print(f"Result: {result}")
|
||||
|
||||
# Test 2: Get stats
|
||||
print("\n✅ Test 2: Get statistics")
|
||||
stats = scheduler.get_stats()
|
||||
print(f"Stats: {stats}")
|
||||
|
||||
# Test 3: Start scheduler (will run every 5 seconds)
|
||||
print("\n✅ Test 3: Start scheduler (*/5s)")
|
||||
scheduler.start("*/5s", run_immediately=True)
|
||||
|
||||
import time
|
||||
print("Running for 12 seconds...")
|
||||
time.sleep(12)
|
||||
|
||||
# Stop scheduler
|
||||
print("\n✅ Test 4: Stop scheduler")
|
||||
scheduler.stop()
|
||||
|
||||
# Final stats
|
||||
print("\n📊 Final Statistics:")
|
||||
final_stats = scheduler.get_stats()
|
||||
for key, value in final_stats.items():
|
||||
print(f" {key}: {value}")
|
||||
|
||||
print("\n✅ All tests passed!")
|
||||
print("=" * 60)
|
||||
@@ -0,0 +1,14 @@
|
||||
framework: praisonai
|
||||
|
||||
agents:
|
||||
- name: "5 Second Timer"
|
||||
role: "Timer Agent"
|
||||
instructions: "Display the current time"
|
||||
verbose: true
|
||||
|
||||
task: "Show current time"
|
||||
|
||||
schedule:
|
||||
interval: "*/5s"
|
||||
max_retries: 3
|
||||
run_immediately: true
|
||||
@@ -0,0 +1,14 @@
|
||||
framework: praisonai
|
||||
|
||||
agents:
|
||||
- name: "Counter Agent"
|
||||
role: "Counter"
|
||||
instructions: "Count from 1 to 5, incrementing by 1 each time. Just output the numbers clearly."
|
||||
verbose: true
|
||||
|
||||
task: "Print numbers 1, 2, 3, 4, 5 (one number per line)"
|
||||
|
||||
schedule:
|
||||
interval: "*/5s"
|
||||
max_retries: 3
|
||||
run_immediately: true
|
||||
@@ -0,0 +1,16 @@
|
||||
framework: praisonai
|
||||
|
||||
agents:
|
||||
- name: "Counter Agent with Limits"
|
||||
role: "Counter"
|
||||
instructions: "Count from 1 to 5, incrementing by 1 each time. Just output the numbers clearly."
|
||||
verbose: true
|
||||
|
||||
task: "Print numbers 1, 2, 3, 4, 5 (one number per line)"
|
||||
|
||||
schedule:
|
||||
interval: "*/5s" # Run every 5 seconds
|
||||
max_retries: 3 # Retry up to 3 times on failure
|
||||
run_immediately: true # Run once immediately on start
|
||||
timeout: 30 # Max 30 seconds per execution
|
||||
max_cost: 0.01 # Stop after spending $0.01 USD
|
||||
@@ -0,0 +1,15 @@
|
||||
framework: praisonai
|
||||
|
||||
agents:
|
||||
- name: "Budget Test Agent"
|
||||
role: "Tester"
|
||||
instructions: "Count from 1 to 3"
|
||||
verbose: true
|
||||
|
||||
task: "Print numbers 1, 2, 3"
|
||||
|
||||
schedule:
|
||||
interval: "*/3s"
|
||||
max_retries: 2
|
||||
run_immediately: true
|
||||
# No max_cost specified - should default to $1.00
|
||||
@@ -0,0 +1,73 @@
|
||||
"""
|
||||
Basic Agent Skills Usage Example
|
||||
|
||||
This example demonstrates how to use Agent Skills with PraisonAI Agents.
|
||||
Skills are modular capabilities that can be loaded and used by agents.
|
||||
"""
|
||||
|
||||
from praisonaiagents import Agent, SkillManager
|
||||
|
||||
# Example 1: Using SkillManager directly
|
||||
print("=" * 50)
|
||||
print("Example 1: Using SkillManager directly")
|
||||
print("=" * 50)
|
||||
|
||||
# Create a skill manager
|
||||
manager = SkillManager()
|
||||
|
||||
# Discover skills from a directory
|
||||
# This scans the directory for subdirectories containing SKILL.md files
|
||||
skill_count = manager.discover(["./pdf-processing"], include_defaults=False)
|
||||
print(f"Discovered {skill_count} skill(s)")
|
||||
|
||||
# List available skills
|
||||
for skill in manager:
|
||||
print(f" - {skill.properties.name}: {skill.properties.description[:50]}...")
|
||||
|
||||
# Generate prompt XML for system prompt injection
|
||||
prompt_xml = manager.to_prompt()
|
||||
print("\nGenerated prompt XML:")
|
||||
print(prompt_xml)
|
||||
|
||||
# Example 2: Using skills with an Agent
|
||||
print("\n" + "=" * 50)
|
||||
print("Example 2: Using skills with an Agent")
|
||||
print("=" * 50)
|
||||
|
||||
# Create an agent with skills
|
||||
# Skills are lazy-loaded only when accessed
|
||||
agent = Agent(
|
||||
name="PDF Assistant",
|
||||
instructions="You are a helpful assistant that can process PDF documents.",
|
||||
skills=["./pdf-processing"], # Direct skill paths
|
||||
)
|
||||
|
||||
# Access the skill manager through the agent
|
||||
if agent.skill_manager:
|
||||
print(f"Agent has {len(agent.skill_manager)} skill(s) loaded")
|
||||
|
||||
# Get the skills prompt for system prompt injection
|
||||
skills_prompt = agent.get_skills_prompt()
|
||||
print("\nAgent skills prompt:")
|
||||
print(skills_prompt)
|
||||
|
||||
# Example 3: Discovering skills from multiple directories
|
||||
print("\n" + "=" * 50)
|
||||
print("Example 3: Using skills_dirs for discovery")
|
||||
print("=" * 50)
|
||||
|
||||
# Create an agent that discovers skills from directories
|
||||
agent_with_discovery = Agent(
|
||||
name="Multi-Skill Agent",
|
||||
instructions="You are a versatile assistant with multiple skills.",
|
||||
skills_dirs=["./"], # Scan current directory for skill subdirectories
|
||||
)
|
||||
|
||||
if agent_with_discovery.skill_manager:
|
||||
print(f"Discovered {len(agent_with_discovery.skill_manager)} skill(s)")
|
||||
for name in agent_with_discovery.skill_manager.skill_names:
|
||||
print(f" - {name}")
|
||||
|
||||
print("\n" + "=" * 50)
|
||||
print("Examples complete!")
|
||||
print("=" * 50)
|
||||
@@ -0,0 +1,163 @@
|
||||
"""
|
||||
Custom Skill Creation Example
|
||||
|
||||
This example demonstrates how to programmatically create and validate
|
||||
custom Agent Skills using PraisonAI Agents.
|
||||
"""
|
||||
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
from praisonaiagents.skills import (
|
||||
SkillManager,
|
||||
SkillLoader,
|
||||
validate,
|
||||
read_properties,
|
||||
)
|
||||
|
||||
|
||||
def create_custom_skill(
|
||||
name: str,
|
||||
description: str,
|
||||
instructions: str,
|
||||
base_dir: str = None
|
||||
) -> Path:
|
||||
"""Create a custom skill programmatically.
|
||||
|
||||
Args:
|
||||
name: Skill name (kebab-case)
|
||||
description: What the skill does
|
||||
instructions: Markdown instructions for the skill
|
||||
base_dir: Directory to create skill in (default: temp dir)
|
||||
|
||||
Returns:
|
||||
Path to the created skill directory
|
||||
"""
|
||||
if base_dir is None:
|
||||
base_dir = tempfile.mkdtemp()
|
||||
|
||||
skill_dir = Path(base_dir) / name
|
||||
skill_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Create SKILL.md
|
||||
skill_md_content = f"""---
|
||||
name: {name}
|
||||
description: {description}
|
||||
license: Apache-2.0
|
||||
metadata:
|
||||
author: custom
|
||||
version: "1.0"
|
||||
---
|
||||
|
||||
{instructions}
|
||||
"""
|
||||
(skill_dir / "SKILL.md").write_text(skill_md_content)
|
||||
|
||||
# Create optional directories
|
||||
(skill_dir / "scripts").mkdir(exist_ok=True)
|
||||
(skill_dir / "references").mkdir(exist_ok=True)
|
||||
(skill_dir / "assets").mkdir(exist_ok=True)
|
||||
|
||||
return skill_dir
|
||||
|
||||
|
||||
def main():
|
||||
print("=" * 60)
|
||||
print("Custom Skill Creation Example")
|
||||
print("=" * 60)
|
||||
|
||||
# Create a custom skill
|
||||
print("\n1. Creating a custom skill...")
|
||||
|
||||
skill_path = create_custom_skill(
|
||||
name="code-review",
|
||||
description="Review code for best practices, bugs, and improvements. Use when asked to review or analyze code quality.",
|
||||
instructions="""
|
||||
# Code Review Skill
|
||||
|
||||
## Overview
|
||||
This skill enables thorough code review with focus on:
|
||||
- Code quality and readability
|
||||
- Potential bugs and edge cases
|
||||
- Performance considerations
|
||||
- Security vulnerabilities
|
||||
- Best practices adherence
|
||||
|
||||
## Instructions
|
||||
|
||||
When reviewing code:
|
||||
|
||||
1. **Read the code carefully** - Understand the purpose and flow
|
||||
2. **Check for bugs** - Look for null checks, edge cases, error handling
|
||||
3. **Evaluate readability** - Variable names, comments, structure
|
||||
4. **Consider performance** - Time/space complexity, unnecessary operations
|
||||
5. **Security review** - Input validation, injection risks, data exposure
|
||||
6. **Suggest improvements** - Provide actionable recommendations
|
||||
|
||||
## Output Format
|
||||
|
||||
Structure your review as:
|
||||
- Summary: Brief overview of the code
|
||||
- Issues: List of problems found (severity: high/medium/low)
|
||||
- Suggestions: Recommended improvements
|
||||
- Positive aspects: What's done well
|
||||
"""
|
||||
)
|
||||
|
||||
print(f" Created skill at: {skill_path}")
|
||||
|
||||
# Validate the skill
|
||||
print("\n2. Validating the skill...")
|
||||
errors = validate(skill_path)
|
||||
|
||||
if errors:
|
||||
print(" Validation errors:")
|
||||
for error in errors:
|
||||
print(f" - {error}")
|
||||
else:
|
||||
print(" ✓ Skill is valid!")
|
||||
|
||||
# Load and inspect the skill
|
||||
print("\n3. Loading skill properties...")
|
||||
props = read_properties(skill_path)
|
||||
print(f" Name: {props.name}")
|
||||
print(f" Description: {props.description[:60]}...")
|
||||
print(f" License: {props.license}")
|
||||
|
||||
# Use with SkillLoader for progressive loading
|
||||
print("\n4. Using SkillLoader for progressive disclosure...")
|
||||
loader = SkillLoader()
|
||||
|
||||
# Level 1: Load metadata only
|
||||
skill = loader.load_metadata(str(skill_path))
|
||||
print(f" Level 1 - Metadata loaded: {skill.properties.name}")
|
||||
print(f" Instructions loaded: {skill.is_activated}")
|
||||
|
||||
# Level 2: Activate (load instructions)
|
||||
loader.activate(skill)
|
||||
print(f" Level 2 - Activated: {skill.is_activated}")
|
||||
print(f" Instructions preview: {skill.instructions[:100]}...")
|
||||
|
||||
# Use with SkillManager
|
||||
print("\n5. Using with SkillManager...")
|
||||
manager = SkillManager()
|
||||
manager.add_skill(str(skill_path))
|
||||
|
||||
# Generate prompt XML
|
||||
prompt = manager.to_prompt()
|
||||
print(" Generated prompt XML:")
|
||||
print(prompt)
|
||||
|
||||
# Clean up
|
||||
print("\n6. Cleanup...")
|
||||
import shutil
|
||||
shutil.rmtree(skill_path.parent)
|
||||
print(" Temporary files removed.")
|
||||
|
||||
print("\n" + "=" * 60)
|
||||
print("Example complete!")
|
||||
print("=" * 60)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,38 @@
|
||||
---
|
||||
name: pdf-processing
|
||||
description: Process and extract information from PDF documents. Use this skill when the user asks to read, analyze, or extract data from PDF files.
|
||||
license: Apache-2.0
|
||||
compatibility: Works with PraisonAI Agents
|
||||
metadata:
|
||||
author: praisonai
|
||||
version: "1.0"
|
||||
---
|
||||
|
||||
# PDF Processing Skill
|
||||
|
||||
## Overview
|
||||
|
||||
This skill enables agents to process PDF documents, extract text content, and analyze the information within them.
|
||||
|
||||
## When to Use
|
||||
|
||||
Activate this skill when:
|
||||
- User asks to read or analyze a PDF file
|
||||
- User needs to extract text from a PDF
|
||||
- User wants to summarize a PDF document
|
||||
- User needs to search for information within a PDF
|
||||
|
||||
## Instructions
|
||||
|
||||
1. First, verify the PDF file exists at the specified path
|
||||
2. Use appropriate tools to read the PDF content
|
||||
3. Extract text while preserving structure where possible
|
||||
4. For large PDFs, process in chunks to manage context
|
||||
5. Summarize or analyze based on user's specific request
|
||||
|
||||
## Best Practices
|
||||
|
||||
- Always confirm the file path before processing
|
||||
- Handle encrypted PDFs gracefully with appropriate error messages
|
||||
- For scanned PDFs, note that OCR may be required
|
||||
- Preserve important formatting like tables and lists
|
||||
@@ -3,7 +3,7 @@ Praison AI Agents - A package for hierarchical AI agent task execution
|
||||
"""
|
||||
|
||||
# Apply warning patch BEFORE any imports to intercept warnings at the source
|
||||
from . import _warning_patch
|
||||
from . import _warning_patch # noqa: F401
|
||||
|
||||
# Import centralized logging configuration FIRST
|
||||
from . import _logging
|
||||
@@ -78,6 +78,19 @@ def __getattr__(name):
|
||||
elif name == "LineRange":
|
||||
from praisonaiagents.context.fast import LineRange
|
||||
return LineRange
|
||||
# Agent Skills support (lazy loaded for zero performance impact)
|
||||
elif name == "SkillManager":
|
||||
from praisonaiagents.skills import SkillManager
|
||||
return SkillManager
|
||||
elif name == "SkillProperties":
|
||||
from praisonaiagents.skills import SkillProperties
|
||||
return SkillProperties
|
||||
elif name == "SkillMetadata":
|
||||
from praisonaiagents.skills import SkillMetadata
|
||||
return SkillMetadata
|
||||
elif name == "SkillLoader":
|
||||
from praisonaiagents.skills import SkillLoader
|
||||
return SkillLoader
|
||||
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
||||
|
||||
# Planning mode support
|
||||
@@ -289,6 +302,23 @@ __all__ = [
|
||||
'ExpandResult'
|
||||
]
|
||||
|
||||
# Add workflow exports
|
||||
__all__.extend([
|
||||
'Workflow',
|
||||
'WorkflowStep',
|
||||
'WorkflowContext',
|
||||
'StepResult',
|
||||
'Route',
|
||||
'Parallel',
|
||||
'Loop',
|
||||
'Repeat',
|
||||
'route',
|
||||
'parallel',
|
||||
'loop',
|
||||
'repeat',
|
||||
'Pipeline'
|
||||
])
|
||||
|
||||
# Add MCP to __all__ if available
|
||||
if _mcp_available:
|
||||
__all__.append('MCP')
|
||||
@@ -305,6 +335,14 @@ __all__.extend([
|
||||
'LineRange'
|
||||
])
|
||||
|
||||
# Add Agent Skills exports (lazy loaded)
|
||||
__all__.extend([
|
||||
'SkillManager',
|
||||
'SkillProperties',
|
||||
'SkillMetadata',
|
||||
'SkillLoader'
|
||||
])
|
||||
|
||||
# AG-UI support (optional, lazy loaded)
|
||||
try:
|
||||
from praisonaiagents.ui.agui import AGUI
|
||||
|
||||
@@ -55,10 +55,10 @@ def _get_all_noisy_loggers() -> List[str]:
|
||||
|
||||
def _configure_loggers():
|
||||
"""Configure all loggers based on LOGLEVEL environment variable."""
|
||||
loglevel = os.environ.get('LOGLEVEL', 'INFO').upper()
|
||||
loglevel = os.environ.get('LOGLEVEL', 'WARNING').upper()
|
||||
|
||||
# When DEBUG is set, allow some HTTP logging for API endpoints
|
||||
if loglevel == 'DEBUG':
|
||||
# When DEBUG or INFO is set, allow some HTTP logging for API endpoints
|
||||
if loglevel in ('DEBUG', 'INFO'):
|
||||
allowed_debug_loggers = {"httpx", "httpx._client", "openai._client"}
|
||||
|
||||
for logger_name in _get_all_noisy_loggers():
|
||||
@@ -109,10 +109,10 @@ def configure_root_logger():
|
||||
"""Configure the root logger with RichHandler."""
|
||||
from rich.logging import RichHandler
|
||||
|
||||
loglevel = os.environ.get('LOGLEVEL', 'INFO').upper()
|
||||
loglevel = os.environ.get('LOGLEVEL', 'WARNING').upper()
|
||||
|
||||
logging.basicConfig(
|
||||
level=getattr(logging, loglevel, logging.INFO),
|
||||
level=getattr(logging, loglevel, logging.WARNING),
|
||||
format="%(asctime)s %(filename)s:%(lineno)d %(levelname)s %(message)s",
|
||||
datefmt="[%X]",
|
||||
handlers=[RichHandler(rich_tracebacks=True)],
|
||||
|
||||
@@ -8,9 +8,10 @@ import functools
|
||||
import sys
|
||||
|
||||
# Apply aggressive warning filters first
|
||||
warnings.filterwarnings("ignore", message=".*There is no current event loop.*", category=DeprecationWarning)
|
||||
warnings.filterwarnings("ignore", message=".*Pydantic serializer warnings.*")
|
||||
warnings.filterwarnings("ignore", message=".*PydanticSerializationUnexpectedValue.*")
|
||||
warnings.filterwarnings("ignore", message=".*Expected 9 fields but got.*")
|
||||
warnings.filterwarnings("ignore", message=".*Expected \\d+ fields but got.*")
|
||||
warnings.filterwarnings("ignore", message=".*Expected `StreamingChoices` but got.*")
|
||||
warnings.filterwarnings("ignore", message=".*serialized value may not be as expected.*")
|
||||
warnings.filterwarnings("ignore", message=".*Use 'content=<...>' to upload raw bytes/text content.*")
|
||||
@@ -27,11 +28,12 @@ SUPPRESSED_PATTERNS = [
|
||||
"The `dict` method is deprecated; use `model_dump` instead",
|
||||
"Pydantic serializer warnings",
|
||||
"PydanticSerializationUnexpectedValue",
|
||||
"Expected 9 fields but got 5 for type `Message`",
|
||||
"Expected", # Catches all "Expected N fields but got M" patterns
|
||||
"Expected `StreamingChoices` but got `Choices`",
|
||||
"serialized value may not be as expected",
|
||||
"Mixing V1 models and V2 models",
|
||||
"Please upgrade `Settings` to V2"
|
||||
"Please upgrade `Settings` to V2",
|
||||
"There is no current event loop"
|
||||
]
|
||||
|
||||
@functools.wraps(_original_warn)
|
||||
@@ -43,7 +45,7 @@ def _patched_warn(message, category=None, stacklevel=1, source=None):
|
||||
if pattern in msg_str:
|
||||
return
|
||||
|
||||
if category == UserWarning and "pydantic" in msg_str.lower():
|
||||
if category is UserWarning and "pydantic" in msg_str.lower():
|
||||
return
|
||||
|
||||
_original_warn(message, category, stacklevel, source)
|
||||
@@ -57,7 +59,7 @@ def _patched_warn_explicit(message, category, filename, lineno, module=None, reg
|
||||
if pattern in msg_str:
|
||||
return
|
||||
|
||||
if category == UserWarning and "pydantic" in msg_str.lower():
|
||||
if category is UserWarning and "pydantic" in msg_str.lower():
|
||||
return
|
||||
|
||||
if module and "pydantic" in str(module):
|
||||
|
||||
@@ -250,7 +250,9 @@ class Agent:
|
||||
fast_context_parallelism: int = 8,
|
||||
fast_context_timeout: float = 30.0,
|
||||
history_in_context: Optional[int] = None,
|
||||
auto_save: Optional[str] = None
|
||||
auto_save: Optional[str] = None,
|
||||
skills: Optional[List[str]] = None,
|
||||
skills_dirs: Optional[List[str]] = None
|
||||
):
|
||||
"""Initialize an Agent instance.
|
||||
|
||||
@@ -607,6 +609,12 @@ Your Goal: {self.goal}
|
||||
self.fast_context_timeout = fast_context_timeout
|
||||
self._fast_context_instance = None # Lazy loaded
|
||||
|
||||
# Agent Skills configuration (lazy loaded for zero performance impact)
|
||||
self._skills = skills
|
||||
self._skills_dirs = skills_dirs
|
||||
self._skill_manager = None # Lazy loaded
|
||||
self._skills_initialized = False
|
||||
|
||||
@property
|
||||
def console(self):
|
||||
"""Lazily initialize Rich Console only when needed."""
|
||||
@@ -626,6 +634,35 @@ Your Goal: {self.goal}
|
||||
def fast_context_path(self, value):
|
||||
self._fast_context_path = value
|
||||
|
||||
@property
|
||||
def skill_manager(self):
|
||||
"""Lazily initialize SkillManager only when skills are accessed."""
|
||||
if self._skill_manager is None and (self._skills or self._skills_dirs):
|
||||
from ..skills import SkillManager
|
||||
self._skill_manager = SkillManager()
|
||||
|
||||
# Add explicit skill paths
|
||||
if self._skills:
|
||||
for skill_path in self._skills:
|
||||
self._skill_manager.add_skill(skill_path)
|
||||
|
||||
# Discover skills from directories
|
||||
if self._skills_dirs:
|
||||
self._skill_manager.discover(self._skills_dirs, include_defaults=False)
|
||||
|
||||
self._skills_initialized = True
|
||||
return self._skill_manager
|
||||
|
||||
def get_skills_prompt(self) -> str:
|
||||
"""Get the XML prompt for available skills.
|
||||
|
||||
Returns:
|
||||
XML string with <available_skills> block, or empty string if no skills
|
||||
"""
|
||||
if self.skill_manager is None:
|
||||
return ""
|
||||
return self.skill_manager.to_prompt()
|
||||
|
||||
@property
|
||||
def _openai_client(self):
|
||||
"""Lazily initialize OpenAI client only when needed."""
|
||||
@@ -1538,13 +1575,13 @@ Your Goal: {self.goal}"""
|
||||
logging.debug(f"{self.name} executing tool {function_name} with arguments: {arguments}")
|
||||
|
||||
# Check if approval is required for this tool
|
||||
from ..approval import is_approval_required, console_approval_callback, get_risk_level, mark_approved, ApprovalDecision
|
||||
from ..approval import is_approval_required, console_approval_callback, get_risk_level, mark_approved, ApprovalDecision, get_approval_callback
|
||||
if is_approval_required(function_name):
|
||||
risk_level = get_risk_level(function_name)
|
||||
logging.info(f"Tool {function_name} requires approval (risk level: {risk_level})")
|
||||
logging.debug(f"Tool {function_name} requires approval (risk level: {risk_level})")
|
||||
|
||||
# Use global approval callback or default console callback
|
||||
callback = approval_callback or console_approval_callback
|
||||
callback = get_approval_callback() or console_approval_callback
|
||||
|
||||
try:
|
||||
decision = callback(function_name, arguments, risk_level)
|
||||
|
||||
@@ -43,6 +43,15 @@ def set_approval_callback(callback_fn: Callable):
|
||||
global approval_callback
|
||||
approval_callback = callback_fn
|
||||
|
||||
def get_approval_callback() -> Optional[Callable]:
|
||||
"""Get the current approval callback function.
|
||||
|
||||
Returns the custom callback if set, otherwise None.
|
||||
This should be used instead of directly accessing approval_callback
|
||||
to ensure the latest callback is always used.
|
||||
"""
|
||||
return approval_callback
|
||||
|
||||
def mark_approved(tool_name: str):
|
||||
"""Mark a tool as approved in the current context."""
|
||||
approved = _approved_context.get(set())
|
||||
@@ -90,7 +99,7 @@ def require_approval(risk_level: RiskLevel = "high"):
|
||||
except Exception as e:
|
||||
# Fallback to sync approval if async fails
|
||||
logging.warning(f"Async approval failed, using sync fallback: {e}")
|
||||
callback = approval_callback or console_approval_callback
|
||||
callback = get_approval_callback() or console_approval_callback
|
||||
decision = callback(tool_name, kwargs, risk_level)
|
||||
|
||||
if not decision.approved:
|
||||
@@ -198,7 +207,7 @@ async def request_approval(function_name: str, arguments: Dict[str, Any]) -> App
|
||||
risk_level = TOOL_RISK_LEVELS.get(function_name, "medium")
|
||||
|
||||
# Use custom callback if set, otherwise use console callback
|
||||
callback = approval_callback or console_approval_callback
|
||||
callback = get_approval_callback() or console_approval_callback
|
||||
|
||||
try:
|
||||
# Handle async callbacks
|
||||
|
||||
@@ -0,0 +1,68 @@
|
||||
"""
|
||||
PraisonAI Agents Evaluation Framework.
|
||||
|
||||
Provides comprehensive evaluation capabilities for AI agents with zero performance
|
||||
impact when not in use through lazy loading.
|
||||
|
||||
Evaluator Types:
|
||||
- AccuracyEvaluator: Compare output against expected output using LLM-as-judge
|
||||
- PerformanceEvaluator: Measure runtime and memory usage
|
||||
- ReliabilityEvaluator: Verify expected tool calls are made
|
||||
- CriteriaEvaluator: Evaluate against custom criteria
|
||||
|
||||
Example:
|
||||
>>> from praisonaiagents.eval import AccuracyEvaluator
|
||||
>>> evaluator = AccuracyEvaluator(
|
||||
... agent=my_agent,
|
||||
... input_text="What is 2+2?",
|
||||
... expected_output="4"
|
||||
... )
|
||||
>>> result = evaluator.run(print_summary=True)
|
||||
"""
|
||||
|
||||
__all__ = [
|
||||
"BaseEvaluator",
|
||||
"AccuracyEvaluator",
|
||||
"PerformanceEvaluator",
|
||||
"ReliabilityEvaluator",
|
||||
"CriteriaEvaluator",
|
||||
"AccuracyResult",
|
||||
"PerformanceResult",
|
||||
"ReliabilityResult",
|
||||
"CriteriaResult",
|
||||
"EvaluationScore",
|
||||
"PerformanceMetrics",
|
||||
"ToolCallResult",
|
||||
"CriteriaScore",
|
||||
]
|
||||
|
||||
_LAZY_IMPORTS = {
|
||||
"BaseEvaluator": ("base", "BaseEvaluator"),
|
||||
"AccuracyEvaluator": ("accuracy", "AccuracyEvaluator"),
|
||||
"PerformanceEvaluator": ("performance", "PerformanceEvaluator"),
|
||||
"ReliabilityEvaluator": ("reliability", "ReliabilityEvaluator"),
|
||||
"CriteriaEvaluator": ("criteria", "CriteriaEvaluator"),
|
||||
"AccuracyResult": ("results", "AccuracyResult"),
|
||||
"PerformanceResult": ("results", "PerformanceResult"),
|
||||
"ReliabilityResult": ("results", "ReliabilityResult"),
|
||||
"CriteriaResult": ("results", "CriteriaResult"),
|
||||
"EvaluationScore": ("results", "EvaluationScore"),
|
||||
"PerformanceMetrics": ("results", "PerformanceMetrics"),
|
||||
"ToolCallResult": ("results", "ToolCallResult"),
|
||||
"CriteriaScore": ("results", "CriteriaScore"),
|
||||
}
|
||||
|
||||
|
||||
def __getattr__(name: str):
|
||||
"""Lazy import mechanism for zero-cost imports when not used."""
|
||||
if name in _LAZY_IMPORTS:
|
||||
module_name, attr_name = _LAZY_IMPORTS[name]
|
||||
import importlib
|
||||
module = importlib.import_module(f".{module_name}", __name__)
|
||||
return getattr(module, attr_name)
|
||||
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
||||
|
||||
|
||||
def __dir__():
|
||||
"""Return list of available attributes for tab completion."""
|
||||
return __all__
|
||||
@@ -0,0 +1,239 @@
|
||||
"""
|
||||
Accuracy evaluator for PraisonAI Agents.
|
||||
|
||||
Evaluates agent output accuracy by comparing against expected output using LLM-as-judge.
|
||||
"""
|
||||
|
||||
import os
|
||||
import logging
|
||||
from typing import Any, Callable, List, Optional, Union, TYPE_CHECKING
|
||||
|
||||
from .base import BaseEvaluator
|
||||
from .results import AccuracyResult, EvaluationScore
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from ..agent.agent import Agent
|
||||
from ..agents.agents import PraisonAIAgents
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class AccuracyEvaluator(BaseEvaluator):
|
||||
"""
|
||||
Evaluates the accuracy of agent outputs against expected outputs.
|
||||
|
||||
Uses an LLM as a judge to score outputs on a scale of 1-10.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
agent: Optional[Union["Agent", "PraisonAIAgents"]] = None,
|
||||
func: Optional[Callable[..., str]] = None,
|
||||
input_text: str = "",
|
||||
expected_output: str = "",
|
||||
num_iterations: int = 1,
|
||||
model: Optional[str] = None,
|
||||
name: Optional[str] = None,
|
||||
save_results_path: Optional[str] = None,
|
||||
verbose: bool = False,
|
||||
):
|
||||
"""
|
||||
Initialize the accuracy evaluator.
|
||||
|
||||
Args:
|
||||
agent: Agent or PraisonAIAgents instance to evaluate
|
||||
func: Alternative callable that returns output string
|
||||
input_text: Input to provide to the agent/function
|
||||
expected_output: Expected output to compare against
|
||||
num_iterations: Number of evaluation iterations
|
||||
model: LLM model to use for judging (defaults to gpt-4o-mini)
|
||||
name: Name for this evaluation
|
||||
save_results_path: Path to save results
|
||||
verbose: Enable verbose output
|
||||
"""
|
||||
super().__init__(name=name, save_results_path=save_results_path, verbose=verbose)
|
||||
|
||||
self.agent = agent
|
||||
self.func = func
|
||||
self.input_text = input_text
|
||||
self.expected_output = expected_output
|
||||
self.num_iterations = num_iterations
|
||||
self.model = model or os.getenv("OPENAI_MODEL_NAME", "gpt-4o-mini")
|
||||
|
||||
if agent is None and func is None:
|
||||
raise ValueError("Either 'agent' or 'func' must be provided")
|
||||
|
||||
def _get_output(self) -> str:
|
||||
"""Get output from agent or function."""
|
||||
if self.func:
|
||||
return str(self.func(self.input_text))
|
||||
|
||||
if hasattr(self.agent, 'chat'):
|
||||
return str(self.agent.chat(self.input_text))
|
||||
elif hasattr(self.agent, 'start'):
|
||||
result = self.agent.start(self.input_text)
|
||||
if hasattr(result, 'raw'):
|
||||
return str(result.raw)
|
||||
return str(result)
|
||||
else:
|
||||
raise ValueError("Agent must have 'chat' or 'start' method")
|
||||
|
||||
def _judge_output(self, output: str) -> EvaluationScore:
|
||||
"""
|
||||
Use LLM to judge the output against expected output.
|
||||
|
||||
Args:
|
||||
output: The actual output to evaluate
|
||||
|
||||
Returns:
|
||||
EvaluationScore with score and reasoning
|
||||
"""
|
||||
try:
|
||||
from openai import OpenAI
|
||||
except ImportError:
|
||||
raise ImportError("OpenAI package required for accuracy evaluation. Install with: pip install openai")
|
||||
|
||||
client = OpenAI()
|
||||
|
||||
prompt = f"""You are an expert evaluator. Compare the actual output against the expected output and provide a score from 1-10.
|
||||
|
||||
Input: {self.input_text}
|
||||
|
||||
Expected Output: {self.expected_output}
|
||||
|
||||
Actual Output: {output}
|
||||
|
||||
Scoring Guidelines:
|
||||
- 10: Perfect match in meaning and completeness
|
||||
- 8-9: Very close, minor differences that don't affect correctness
|
||||
- 6-7: Mostly correct but missing some details or has minor errors
|
||||
- 4-5: Partially correct but significant issues
|
||||
- 2-3: Mostly incorrect but shows some understanding
|
||||
- 1: Completely wrong or irrelevant
|
||||
|
||||
Respond in this exact format:
|
||||
SCORE: [number 1-10]
|
||||
REASONING: [brief explanation]"""
|
||||
|
||||
response = client.chat.completions.create(
|
||||
model=self.model,
|
||||
messages=[{"role": "user", "content": prompt}],
|
||||
temperature=0.1,
|
||||
max_tokens=200
|
||||
)
|
||||
|
||||
response_text = response.choices[0].message.content or ""
|
||||
|
||||
score = 5.0
|
||||
reasoning = "Unable to parse response"
|
||||
|
||||
lines = response_text.strip().split('\n')
|
||||
for line in lines:
|
||||
if line.startswith('SCORE:'):
|
||||
try:
|
||||
score = float(line.replace('SCORE:', '').strip())
|
||||
score = max(1.0, min(10.0, score))
|
||||
except ValueError:
|
||||
pass
|
||||
elif line.startswith('REASONING:'):
|
||||
reasoning = line.replace('REASONING:', '').strip()
|
||||
|
||||
return EvaluationScore(
|
||||
score=score,
|
||||
reasoning=reasoning,
|
||||
input_text=self.input_text,
|
||||
output_text=output,
|
||||
expected_output=self.expected_output
|
||||
)
|
||||
|
||||
def run(self, print_summary: bool = False) -> AccuracyResult:
|
||||
"""
|
||||
Execute the accuracy evaluation.
|
||||
|
||||
Args:
|
||||
print_summary: Whether to print summary after evaluation
|
||||
|
||||
Returns:
|
||||
AccuracyResult with all evaluation scores
|
||||
"""
|
||||
self.before_run()
|
||||
|
||||
result = AccuracyResult(
|
||||
eval_id=self.eval_id,
|
||||
name=self.name
|
||||
)
|
||||
|
||||
for i in range(self.num_iterations):
|
||||
if self.verbose:
|
||||
logger.info(f"Running iteration {i + 1}/{self.num_iterations}")
|
||||
|
||||
try:
|
||||
output = self._get_output()
|
||||
score = self._judge_output(output)
|
||||
result.evaluations.append(score)
|
||||
|
||||
if self.verbose:
|
||||
logger.info(f" Score: {score.score}/10 - {score.reasoning}")
|
||||
except Exception as e:
|
||||
logger.error(f"Error in iteration {i + 1}: {e}")
|
||||
result.evaluations.append(EvaluationScore(
|
||||
score=0.0,
|
||||
reasoning=f"Error: {str(e)}",
|
||||
input_text=self.input_text,
|
||||
output_text="",
|
||||
expected_output=self.expected_output
|
||||
))
|
||||
|
||||
self.after_run(result)
|
||||
|
||||
if print_summary:
|
||||
result.print_summary()
|
||||
|
||||
return result
|
||||
|
||||
async def run_async(self, print_summary: bool = False) -> AccuracyResult:
|
||||
"""
|
||||
Execute the accuracy evaluation asynchronously.
|
||||
|
||||
Args:
|
||||
print_summary: Whether to print summary after evaluation
|
||||
|
||||
Returns:
|
||||
AccuracyResult with all evaluation scores
|
||||
"""
|
||||
await self.async_before_run()
|
||||
result = self.run(print_summary=False)
|
||||
await self.async_after_run(result)
|
||||
|
||||
if print_summary:
|
||||
result.print_summary()
|
||||
|
||||
return result
|
||||
|
||||
def evaluate_output(self, output: str, print_summary: bool = False) -> AccuracyResult:
|
||||
"""
|
||||
Evaluate a pre-generated output without running the agent.
|
||||
|
||||
Args:
|
||||
output: The output to evaluate
|
||||
print_summary: Whether to print summary after evaluation
|
||||
|
||||
Returns:
|
||||
AccuracyResult with evaluation score
|
||||
"""
|
||||
self.before_run()
|
||||
|
||||
result = AccuracyResult(
|
||||
eval_id=self.eval_id,
|
||||
name=self.name
|
||||
)
|
||||
|
||||
score = self._judge_output(output)
|
||||
result.evaluations.append(score)
|
||||
|
||||
self.after_run(result)
|
||||
|
||||
if print_summary:
|
||||
result.print_summary()
|
||||
|
||||
return result
|
||||
@@ -0,0 +1,143 @@
|
||||
"""
|
||||
Base evaluator class for PraisonAI Agents evaluation framework.
|
||||
|
||||
This module provides the abstract base class that all evaluators inherit from.
|
||||
"""
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import Any, Dict, Optional, Union, TYPE_CHECKING
|
||||
import uuid
|
||||
import logging
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from .results import AccuracyResult, PerformanceResult, ReliabilityResult, CriteriaResult
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class BaseEvaluator(ABC):
|
||||
"""
|
||||
Abstract base class for all evaluators in PraisonAI Agents.
|
||||
|
||||
Provides common functionality for evaluation lifecycle management,
|
||||
result storage, and telemetry integration.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
name: Optional[str] = None,
|
||||
save_results_path: Optional[str] = None,
|
||||
verbose: bool = False,
|
||||
):
|
||||
"""
|
||||
Initialize the base evaluator.
|
||||
|
||||
Args:
|
||||
name: Optional name for this evaluation run
|
||||
save_results_path: Optional file path to save results (supports {name}, {eval_id} placeholders)
|
||||
verbose: Enable verbose output during evaluation
|
||||
"""
|
||||
self.eval_id = str(uuid.uuid4())[:8]
|
||||
self.name = name or f"eval_{self.eval_id}"
|
||||
self.save_results_path = save_results_path
|
||||
self.verbose = verbose
|
||||
self._result = None
|
||||
|
||||
def before_run(self) -> None:
|
||||
"""
|
||||
Hook called before evaluation runs.
|
||||
Override in subclasses for custom pre-evaluation logic.
|
||||
"""
|
||||
if self.verbose:
|
||||
logger.info(f"Starting evaluation: {self.name} (ID: {self.eval_id})")
|
||||
|
||||
def after_run(self, result: Any) -> None:
|
||||
"""
|
||||
Hook called after evaluation completes.
|
||||
Override in subclasses for custom post-evaluation logic.
|
||||
|
||||
Args:
|
||||
result: The evaluation result
|
||||
"""
|
||||
if self.verbose:
|
||||
logger.info(f"Completed evaluation: {self.name}")
|
||||
|
||||
if self.save_results_path:
|
||||
self._save_result(result)
|
||||
|
||||
async def async_before_run(self) -> None:
|
||||
"""
|
||||
Async hook called before evaluation runs.
|
||||
Override in subclasses for custom async pre-evaluation logic.
|
||||
"""
|
||||
self.before_run()
|
||||
|
||||
async def async_after_run(self, result: Any) -> None:
|
||||
"""
|
||||
Async hook called after evaluation completes.
|
||||
Override in subclasses for custom async post-evaluation logic.
|
||||
|
||||
Args:
|
||||
result: The evaluation result
|
||||
"""
|
||||
self.after_run(result)
|
||||
|
||||
@abstractmethod
|
||||
def run(self, **kwargs) -> Any:
|
||||
"""
|
||||
Execute the evaluation synchronously.
|
||||
|
||||
Returns:
|
||||
Evaluation result (type depends on evaluator)
|
||||
"""
|
||||
pass
|
||||
|
||||
async def run_async(self, **kwargs) -> Any:
|
||||
"""
|
||||
Execute the evaluation asynchronously.
|
||||
Default implementation calls sync run().
|
||||
Override for true async evaluation.
|
||||
|
||||
Returns:
|
||||
Evaluation result (type depends on evaluator)
|
||||
"""
|
||||
return self.run(**kwargs)
|
||||
|
||||
def _save_result(self, result: Any) -> None:
|
||||
"""
|
||||
Save evaluation result to file.
|
||||
|
||||
Args:
|
||||
result: The evaluation result to save
|
||||
"""
|
||||
if not self.save_results_path:
|
||||
return
|
||||
|
||||
try:
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
file_path = self.save_results_path.format(
|
||||
name=self.name,
|
||||
eval_id=self.eval_id
|
||||
)
|
||||
|
||||
path = Path(file_path)
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
if hasattr(result, 'to_dict'):
|
||||
data = result.to_dict()
|
||||
elif hasattr(result, '__dict__'):
|
||||
data = result.__dict__
|
||||
else:
|
||||
data = {"result": str(result)}
|
||||
|
||||
path.write_text(json.dumps(data, indent=2, default=str))
|
||||
|
||||
if self.verbose:
|
||||
logger.info(f"Saved results to: {file_path}")
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to save results: {e}")
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return f"{self.__class__.__name__}(name={self.name!r}, eval_id={self.eval_id!r})"
|
||||
@@ -0,0 +1,289 @@
|
||||
"""
|
||||
Criteria evaluator for PraisonAI Agents.
|
||||
|
||||
Evaluates agent outputs against custom criteria using LLM-as-judge.
|
||||
"""
|
||||
|
||||
import os
|
||||
import logging
|
||||
from typing import Callable, List, Literal, Optional, Union, TYPE_CHECKING
|
||||
|
||||
from .base import BaseEvaluator
|
||||
from .results import CriteriaResult, CriteriaScore
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from ..agent.agent import Agent
|
||||
from ..agents.agents import PraisonAIAgents
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class CriteriaEvaluator(BaseEvaluator):
|
||||
"""
|
||||
Evaluates agent outputs against custom criteria.
|
||||
|
||||
Uses an LLM as a judge to score outputs based on user-defined criteria.
|
||||
Supports both numeric (1-10) and binary (pass/fail) scoring.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
criteria: str,
|
||||
agent: Optional[Union["Agent", "PraisonAIAgents"]] = None,
|
||||
func: Optional[Callable[..., str]] = None,
|
||||
input_text: str = "",
|
||||
scoring_type: Literal["numeric", "binary"] = "numeric",
|
||||
threshold: float = 7.0,
|
||||
num_iterations: int = 1,
|
||||
model: Optional[str] = None,
|
||||
on_fail: Optional[Callable[[CriteriaScore], None]] = None,
|
||||
name: Optional[str] = None,
|
||||
save_results_path: Optional[str] = None,
|
||||
verbose: bool = False,
|
||||
):
|
||||
"""
|
||||
Initialize the criteria evaluator.
|
||||
|
||||
Args:
|
||||
criteria: The criteria to evaluate against (e.g., "Response is helpful and accurate")
|
||||
agent: Agent or PraisonAIAgents instance to evaluate
|
||||
func: Alternative callable that returns output string
|
||||
input_text: Input to provide to the agent/function
|
||||
scoring_type: "numeric" (1-10 scale) or "binary" (pass/fail)
|
||||
threshold: Score threshold for passing (numeric mode only)
|
||||
num_iterations: Number of evaluation iterations
|
||||
model: LLM model to use for judging (defaults to gpt-4o-mini)
|
||||
on_fail: Callback function called when evaluation fails
|
||||
name: Name for this evaluation
|
||||
save_results_path: Path to save results
|
||||
verbose: Enable verbose output
|
||||
"""
|
||||
super().__init__(name=name, save_results_path=save_results_path, verbose=verbose)
|
||||
|
||||
self.criteria = criteria
|
||||
self.agent = agent
|
||||
self.func = func
|
||||
self.input_text = input_text
|
||||
self.scoring_type = scoring_type
|
||||
self.threshold = threshold
|
||||
self.num_iterations = num_iterations
|
||||
self.model = model or os.getenv("OPENAI_MODEL_NAME", "gpt-4o-mini")
|
||||
self.on_fail = on_fail
|
||||
|
||||
def _get_output(self) -> str:
|
||||
"""Get output from agent or function."""
|
||||
if self.func:
|
||||
return str(self.func(self.input_text))
|
||||
|
||||
if self.agent is None:
|
||||
raise ValueError("Either 'agent' or 'func' must be provided")
|
||||
|
||||
if hasattr(self.agent, 'chat'):
|
||||
return str(self.agent.chat(self.input_text))
|
||||
elif hasattr(self.agent, 'start'):
|
||||
result = self.agent.start(self.input_text)
|
||||
if hasattr(result, 'raw'):
|
||||
return str(result.raw)
|
||||
return str(result)
|
||||
else:
|
||||
raise ValueError("Agent must have 'chat' or 'start' method")
|
||||
|
||||
def _judge_output(self, output: str) -> CriteriaScore:
|
||||
"""
|
||||
Use LLM to judge the output against criteria.
|
||||
|
||||
Args:
|
||||
output: The output to evaluate
|
||||
|
||||
Returns:
|
||||
CriteriaScore with score, passed status, and reasoning
|
||||
"""
|
||||
try:
|
||||
from openai import OpenAI
|
||||
except ImportError:
|
||||
raise ImportError("OpenAI package required for criteria evaluation. Install with: pip install openai")
|
||||
|
||||
client = OpenAI()
|
||||
|
||||
if self.scoring_type == "binary":
|
||||
prompt = f"""You are an expert evaluator. Evaluate the following output against the given criteria.
|
||||
|
||||
Criteria: {self.criteria}
|
||||
|
||||
Output to evaluate:
|
||||
{output}
|
||||
|
||||
Determine if the output PASSES or FAILS the criteria.
|
||||
|
||||
Respond in this exact format:
|
||||
RESULT: [PASS or FAIL]
|
||||
REASONING: [brief explanation]"""
|
||||
else:
|
||||
prompt = f"""You are an expert evaluator. Evaluate the following output against the given criteria.
|
||||
|
||||
Criteria: {self.criteria}
|
||||
|
||||
Output to evaluate:
|
||||
{output}
|
||||
|
||||
Score the output from 1-10 based on how well it meets the criteria.
|
||||
- 10: Perfectly meets all criteria
|
||||
- 8-9: Meets criteria very well with minor issues
|
||||
- 6-7: Meets most criteria but has some gaps
|
||||
- 4-5: Partially meets criteria
|
||||
- 2-3: Barely meets criteria
|
||||
- 1: Does not meet criteria at all
|
||||
|
||||
Respond in this exact format:
|
||||
SCORE: [number 1-10]
|
||||
REASONING: [brief explanation]"""
|
||||
|
||||
response = client.chat.completions.create(
|
||||
model=self.model,
|
||||
messages=[{"role": "user", "content": prompt}],
|
||||
temperature=0.1,
|
||||
max_tokens=200
|
||||
)
|
||||
|
||||
response_text = response.choices[0].message.content or ""
|
||||
|
||||
score = 5.0
|
||||
passed = False
|
||||
reasoning = "Unable to parse response"
|
||||
|
||||
lines = response_text.strip().split('\n')
|
||||
for line in lines:
|
||||
if self.scoring_type == "binary":
|
||||
if line.startswith('RESULT:'):
|
||||
result_str = line.replace('RESULT:', '').strip().upper()
|
||||
passed = result_str == 'PASS'
|
||||
score = 10.0 if passed else 0.0
|
||||
else:
|
||||
if line.startswith('SCORE:'):
|
||||
try:
|
||||
score = float(line.replace('SCORE:', '').strip())
|
||||
score = max(1.0, min(10.0, score))
|
||||
passed = score >= self.threshold
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
if line.startswith('REASONING:'):
|
||||
reasoning = line.replace('REASONING:', '').strip()
|
||||
|
||||
return CriteriaScore(
|
||||
score=score,
|
||||
passed=passed,
|
||||
reasoning=reasoning,
|
||||
output_text=output,
|
||||
criteria=self.criteria
|
||||
)
|
||||
|
||||
def run(self, print_summary: bool = False) -> CriteriaResult:
|
||||
"""
|
||||
Execute the criteria evaluation.
|
||||
|
||||
Args:
|
||||
print_summary: Whether to print summary after evaluation
|
||||
|
||||
Returns:
|
||||
CriteriaResult with all evaluation scores
|
||||
"""
|
||||
self.before_run()
|
||||
|
||||
result = CriteriaResult(
|
||||
criteria=self.criteria,
|
||||
scoring_type=self.scoring_type,
|
||||
threshold=self.threshold,
|
||||
eval_id=self.eval_id,
|
||||
name=self.name
|
||||
)
|
||||
|
||||
for i in range(self.num_iterations):
|
||||
if self.verbose:
|
||||
logger.info(f"Running iteration {i + 1}/{self.num_iterations}")
|
||||
|
||||
try:
|
||||
output = self._get_output()
|
||||
score = self._judge_output(output)
|
||||
result.evaluations.append(score)
|
||||
|
||||
if self.verbose:
|
||||
status = "PASS" if score.passed else "FAIL"
|
||||
logger.info(f" {status} (Score: {score.score}) - {score.reasoning}")
|
||||
|
||||
if not score.passed and self.on_fail:
|
||||
self.on_fail(score)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error in iteration {i + 1}: {e}")
|
||||
error_score = CriteriaScore(
|
||||
score=0.0,
|
||||
passed=False,
|
||||
reasoning=f"Error: {str(e)}",
|
||||
output_text="",
|
||||
criteria=self.criteria
|
||||
)
|
||||
result.evaluations.append(error_score)
|
||||
|
||||
if self.on_fail:
|
||||
self.on_fail(error_score)
|
||||
|
||||
self.after_run(result)
|
||||
|
||||
if print_summary:
|
||||
result.print_summary()
|
||||
|
||||
return result
|
||||
|
||||
async def run_async(self, print_summary: bool = False) -> CriteriaResult:
|
||||
"""
|
||||
Execute the criteria evaluation asynchronously.
|
||||
|
||||
Args:
|
||||
print_summary: Whether to print summary after evaluation
|
||||
|
||||
Returns:
|
||||
CriteriaResult with all evaluation scores
|
||||
"""
|
||||
await self.async_before_run()
|
||||
result = self.run(print_summary=False)
|
||||
await self.async_after_run(result)
|
||||
|
||||
if print_summary:
|
||||
result.print_summary()
|
||||
|
||||
return result
|
||||
|
||||
def evaluate_output(self, output: str, print_summary: bool = False) -> CriteriaResult:
|
||||
"""
|
||||
Evaluate a pre-generated output without running the agent.
|
||||
|
||||
Args:
|
||||
output: The output to evaluate
|
||||
print_summary: Whether to print summary after evaluation
|
||||
|
||||
Returns:
|
||||
CriteriaResult with evaluation score
|
||||
"""
|
||||
self.before_run()
|
||||
|
||||
result = CriteriaResult(
|
||||
criteria=self.criteria,
|
||||
scoring_type=self.scoring_type,
|
||||
threshold=self.threshold,
|
||||
eval_id=self.eval_id,
|
||||
name=self.name
|
||||
)
|
||||
|
||||
score = self._judge_output(output)
|
||||
result.evaluations.append(score)
|
||||
|
||||
if not score.passed and self.on_fail:
|
||||
self.on_fail(score)
|
||||
|
||||
self.after_run(result)
|
||||
|
||||
if print_summary:
|
||||
result.print_summary()
|
||||
|
||||
return result
|
||||
@@ -0,0 +1,184 @@
|
||||
"""
|
||||
Performance evaluator for PraisonAI Agents.
|
||||
|
||||
Measures runtime and memory usage of agent executions or functions.
|
||||
"""
|
||||
|
||||
import time
|
||||
import logging
|
||||
from typing import Callable, Optional, TYPE_CHECKING
|
||||
|
||||
from .base import BaseEvaluator
|
||||
from .results import PerformanceResult, PerformanceMetrics
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from ..agent.agent import Agent
|
||||
from ..agents.agents import PraisonAIAgents
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class PerformanceEvaluator(BaseEvaluator):
|
||||
"""
|
||||
Evaluates the performance of agent executions or functions.
|
||||
|
||||
Measures runtime and memory usage across multiple iterations.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
func: Optional[Callable[[], None]] = None,
|
||||
agent: Optional["Agent"] = None,
|
||||
input_text: str = "",
|
||||
num_iterations: int = 10,
|
||||
warmup_runs: int = 2,
|
||||
track_memory: bool = True,
|
||||
name: Optional[str] = None,
|
||||
save_results_path: Optional[str] = None,
|
||||
verbose: bool = False,
|
||||
):
|
||||
"""
|
||||
Initialize the performance evaluator.
|
||||
|
||||
Args:
|
||||
func: Callable to benchmark (takes no arguments)
|
||||
agent: Agent to benchmark (will call chat with input_text)
|
||||
input_text: Input text for agent benchmarking
|
||||
num_iterations: Number of benchmark iterations
|
||||
warmup_runs: Number of warmup runs before measurement
|
||||
track_memory: Whether to track memory usage
|
||||
name: Name for this evaluation
|
||||
save_results_path: Path to save results
|
||||
verbose: Enable verbose output
|
||||
"""
|
||||
super().__init__(name=name, save_results_path=save_results_path, verbose=verbose)
|
||||
|
||||
self.func = func
|
||||
self.agent = agent
|
||||
self.input_text = input_text
|
||||
self.num_iterations = num_iterations
|
||||
self.warmup_runs = warmup_runs
|
||||
self.track_memory = track_memory
|
||||
|
||||
if func is None and agent is None:
|
||||
raise ValueError("Either 'func' or 'agent' must be provided")
|
||||
|
||||
def _get_memory_usage(self) -> float:
|
||||
"""Get current memory usage in MB."""
|
||||
if not self.track_memory:
|
||||
return 0.0
|
||||
|
||||
try:
|
||||
import tracemalloc
|
||||
current, _ = tracemalloc.get_traced_memory()
|
||||
return current / (1024 * 1024)
|
||||
except Exception:
|
||||
return 0.0
|
||||
|
||||
def _execute(self) -> None:
|
||||
"""Execute the function or agent."""
|
||||
if self.func:
|
||||
self.func()
|
||||
elif self.agent:
|
||||
if hasattr(self.agent, 'chat'):
|
||||
self.agent.chat(self.input_text)
|
||||
elif hasattr(self.agent, 'start'):
|
||||
self.agent.start(self.input_text)
|
||||
else:
|
||||
raise ValueError("Agent must have 'chat' or 'start' method")
|
||||
|
||||
def run(self, print_summary: bool = False) -> PerformanceResult:
|
||||
"""
|
||||
Execute the performance evaluation.
|
||||
|
||||
Args:
|
||||
print_summary: Whether to print summary after evaluation
|
||||
|
||||
Returns:
|
||||
PerformanceResult with all performance metrics
|
||||
"""
|
||||
self.before_run()
|
||||
|
||||
result = PerformanceResult(
|
||||
warmup_runs=self.warmup_runs,
|
||||
eval_id=self.eval_id,
|
||||
name=self.name
|
||||
)
|
||||
|
||||
if self.track_memory:
|
||||
try:
|
||||
import tracemalloc
|
||||
tracemalloc.start()
|
||||
except Exception as e:
|
||||
logger.warning(f"Could not start memory tracking: {e}")
|
||||
self.track_memory = False
|
||||
|
||||
if self.verbose:
|
||||
logger.info(f"Running {self.warmup_runs} warmup iterations...")
|
||||
|
||||
for i in range(self.warmup_runs):
|
||||
try:
|
||||
self._execute()
|
||||
except Exception as e:
|
||||
logger.warning(f"Warmup iteration {i + 1} failed: {e}")
|
||||
|
||||
if self.verbose:
|
||||
logger.info(f"Running {self.num_iterations} benchmark iterations...")
|
||||
|
||||
for i in range(self.num_iterations):
|
||||
memory_before = self._get_memory_usage()
|
||||
|
||||
start_time = time.perf_counter()
|
||||
try:
|
||||
self._execute()
|
||||
success = True
|
||||
except Exception as e:
|
||||
logger.error(f"Iteration {i + 1} failed: {e}")
|
||||
success = False
|
||||
end_time = time.perf_counter()
|
||||
|
||||
memory_after = self._get_memory_usage()
|
||||
|
||||
if success:
|
||||
metrics = PerformanceMetrics(
|
||||
run_time=end_time - start_time,
|
||||
memory_usage=max(0, memory_after - memory_before),
|
||||
iteration=i + 1
|
||||
)
|
||||
result.metrics.append(metrics)
|
||||
|
||||
if self.verbose:
|
||||
logger.info(f" Iteration {i + 1}: {metrics.run_time:.4f}s, {metrics.memory_usage:.2f}MB")
|
||||
|
||||
if self.track_memory:
|
||||
try:
|
||||
import tracemalloc
|
||||
tracemalloc.stop()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
self.after_run(result)
|
||||
|
||||
if print_summary:
|
||||
result.print_summary()
|
||||
|
||||
return result
|
||||
|
||||
async def run_async(self, print_summary: bool = False) -> PerformanceResult:
|
||||
"""
|
||||
Execute the performance evaluation asynchronously.
|
||||
|
||||
Args:
|
||||
print_summary: Whether to print summary after evaluation
|
||||
|
||||
Returns:
|
||||
PerformanceResult with all performance metrics
|
||||
"""
|
||||
await self.async_before_run()
|
||||
result = self.run(print_summary=False)
|
||||
await self.async_after_run(result)
|
||||
|
||||
if print_summary:
|
||||
result.print_summary()
|
||||
|
||||
return result
|
||||
@@ -0,0 +1,225 @@
|
||||
"""
|
||||
Reliability evaluator for PraisonAI Agents.
|
||||
|
||||
Evaluates agent reliability by verifying expected tool calls are made.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from typing import Dict, List, Optional, Set, Any, TYPE_CHECKING
|
||||
|
||||
from .base import BaseEvaluator
|
||||
from .results import ReliabilityResult, ToolCallResult
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from ..agent.agent import Agent
|
||||
from ..agents.agents import PraisonAIAgents
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ReliabilityEvaluator(BaseEvaluator):
|
||||
"""
|
||||
Evaluates the reliability of agent tool usage.
|
||||
|
||||
Verifies that expected tools are called during agent execution.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
agent: Optional["Agent"] = None,
|
||||
input_text: str = "",
|
||||
expected_tools: Optional[List[str]] = None,
|
||||
forbidden_tools: Optional[List[str]] = None,
|
||||
name: Optional[str] = None,
|
||||
save_results_path: Optional[str] = None,
|
||||
verbose: bool = False,
|
||||
):
|
||||
"""
|
||||
Initialize the reliability evaluator.
|
||||
|
||||
Args:
|
||||
agent: Agent to evaluate
|
||||
input_text: Input to provide to the agent
|
||||
expected_tools: List of tool names that should be called
|
||||
forbidden_tools: List of tool names that should NOT be called
|
||||
name: Name for this evaluation
|
||||
save_results_path: Path to save results
|
||||
verbose: Enable verbose output
|
||||
"""
|
||||
super().__init__(name=name, save_results_path=save_results_path, verbose=verbose)
|
||||
|
||||
self.agent = agent
|
||||
self.input_text = input_text
|
||||
self.expected_tools = set(expected_tools or [])
|
||||
self.forbidden_tools = set(forbidden_tools or [])
|
||||
|
||||
if agent is None:
|
||||
raise ValueError("'agent' must be provided")
|
||||
|
||||
def _extract_tool_calls(self, response: Any) -> Set[str]:
|
||||
"""
|
||||
Extract tool names from agent response.
|
||||
|
||||
Args:
|
||||
response: Agent response object
|
||||
|
||||
Returns:
|
||||
Set of tool names that were called
|
||||
"""
|
||||
tool_calls = set()
|
||||
|
||||
if hasattr(response, 'tool_calls') and response.tool_calls:
|
||||
for tc in response.tool_calls:
|
||||
if hasattr(tc, 'name'):
|
||||
tool_calls.add(tc.name)
|
||||
elif hasattr(tc, 'function') and hasattr(tc.function, 'name'):
|
||||
tool_calls.add(tc.function.name)
|
||||
|
||||
if hasattr(self.agent, 'chat_history'):
|
||||
for msg in self.agent.chat_history:
|
||||
if isinstance(msg, dict):
|
||||
if msg.get('role') == 'assistant' and 'tool_calls' in msg:
|
||||
for tc in msg.get('tool_calls', []):
|
||||
if isinstance(tc, dict) and 'function' in tc:
|
||||
tool_calls.add(tc['function'].get('name', ''))
|
||||
elif msg.get('role') == 'tool':
|
||||
tool_name = msg.get('name', '')
|
||||
if tool_name:
|
||||
tool_calls.add(tool_name)
|
||||
|
||||
return tool_calls
|
||||
|
||||
def run(self, print_summary: bool = False) -> ReliabilityResult:
|
||||
"""
|
||||
Execute the reliability evaluation.
|
||||
|
||||
Args:
|
||||
print_summary: Whether to print summary after evaluation
|
||||
|
||||
Returns:
|
||||
ReliabilityResult with tool call verification results
|
||||
"""
|
||||
self.before_run()
|
||||
|
||||
result = ReliabilityResult(
|
||||
eval_id=self.eval_id,
|
||||
name=self.name
|
||||
)
|
||||
|
||||
try:
|
||||
if hasattr(self.agent, 'chat'):
|
||||
response = self.agent.chat(self.input_text)
|
||||
elif hasattr(self.agent, 'start'):
|
||||
response = self.agent.start(self.input_text)
|
||||
else:
|
||||
raise ValueError("Agent must have 'chat' or 'start' method")
|
||||
|
||||
actual_tools = self._extract_tool_calls(response)
|
||||
|
||||
if self.verbose:
|
||||
logger.info(f"Expected tools: {self.expected_tools}")
|
||||
logger.info(f"Actual tools called: {actual_tools}")
|
||||
|
||||
for tool in self.expected_tools:
|
||||
was_called = tool in actual_tools
|
||||
result.tool_results.append(ToolCallResult(
|
||||
tool_name=tool,
|
||||
expected=True,
|
||||
actual=was_called,
|
||||
passed=was_called
|
||||
))
|
||||
|
||||
for tool in self.forbidden_tools:
|
||||
was_called = tool in actual_tools
|
||||
result.tool_results.append(ToolCallResult(
|
||||
tool_name=tool,
|
||||
expected=False,
|
||||
actual=was_called,
|
||||
passed=not was_called
|
||||
))
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error during reliability evaluation: {e}")
|
||||
for tool in self.expected_tools:
|
||||
result.tool_results.append(ToolCallResult(
|
||||
tool_name=tool,
|
||||
expected=True,
|
||||
actual=False,
|
||||
passed=False,
|
||||
metadata={"error": str(e)}
|
||||
))
|
||||
|
||||
self.after_run(result)
|
||||
|
||||
if print_summary:
|
||||
result.print_summary()
|
||||
|
||||
return result
|
||||
|
||||
async def run_async(self, print_summary: bool = False) -> ReliabilityResult:
|
||||
"""
|
||||
Execute the reliability evaluation asynchronously.
|
||||
|
||||
Args:
|
||||
print_summary: Whether to print summary after evaluation
|
||||
|
||||
Returns:
|
||||
ReliabilityResult with tool call verification results
|
||||
"""
|
||||
await self.async_before_run()
|
||||
result = self.run(print_summary=False)
|
||||
await self.async_after_run(result)
|
||||
|
||||
if print_summary:
|
||||
result.print_summary()
|
||||
|
||||
return result
|
||||
|
||||
def evaluate_tool_calls(
|
||||
self,
|
||||
actual_tools: List[str],
|
||||
print_summary: bool = False
|
||||
) -> ReliabilityResult:
|
||||
"""
|
||||
Evaluate a pre-recorded list of tool calls without running the agent.
|
||||
|
||||
Args:
|
||||
actual_tools: List of tool names that were called
|
||||
print_summary: Whether to print summary after evaluation
|
||||
|
||||
Returns:
|
||||
ReliabilityResult with tool call verification results
|
||||
"""
|
||||
self.before_run()
|
||||
|
||||
result = ReliabilityResult(
|
||||
eval_id=self.eval_id,
|
||||
name=self.name
|
||||
)
|
||||
|
||||
actual_set = set(actual_tools)
|
||||
|
||||
for tool in self.expected_tools:
|
||||
was_called = tool in actual_set
|
||||
result.tool_results.append(ToolCallResult(
|
||||
tool_name=tool,
|
||||
expected=True,
|
||||
actual=was_called,
|
||||
passed=was_called
|
||||
))
|
||||
|
||||
for tool in self.forbidden_tools:
|
||||
was_called = tool in actual_set
|
||||
result.tool_results.append(ToolCallResult(
|
||||
tool_name=tool,
|
||||
expected=False,
|
||||
actual=was_called,
|
||||
passed=not was_called
|
||||
))
|
||||
|
||||
self.after_run(result)
|
||||
|
||||
if print_summary:
|
||||
result.print_summary()
|
||||
|
||||
return result
|
||||
@@ -0,0 +1,463 @@
|
||||
"""
|
||||
Result dataclasses for PraisonAI Agents evaluation framework.
|
||||
|
||||
This module provides structured result types for different evaluation methods.
|
||||
"""
|
||||
|
||||
from dataclasses import dataclass, field, asdict
|
||||
from typing import List, Optional, Dict, Any
|
||||
from datetime import datetime
|
||||
import statistics
|
||||
import json
|
||||
|
||||
|
||||
@dataclass
|
||||
class EvaluationScore:
|
||||
"""Individual evaluation score with reasoning."""
|
||||
score: float
|
||||
reasoning: str
|
||||
input_text: str
|
||||
output_text: str
|
||||
expected_output: Optional[str] = None
|
||||
timestamp: str = field(default_factory=lambda: datetime.now().isoformat())
|
||||
metadata: Dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
|
||||
@dataclass
|
||||
class AccuracyResult:
|
||||
"""Result from accuracy evaluation comparing output to expected output."""
|
||||
evaluations: List[EvaluationScore] = field(default_factory=list)
|
||||
eval_id: str = ""
|
||||
name: str = ""
|
||||
|
||||
@property
|
||||
def scores(self) -> List[float]:
|
||||
"""Get list of all scores."""
|
||||
return [e.score for e in self.evaluations]
|
||||
|
||||
@property
|
||||
def avg_score(self) -> float:
|
||||
"""Calculate average score."""
|
||||
if not self.scores:
|
||||
return 0.0
|
||||
return statistics.mean(self.scores)
|
||||
|
||||
@property
|
||||
def min_score(self) -> float:
|
||||
"""Get minimum score."""
|
||||
if not self.scores:
|
||||
return 0.0
|
||||
return min(self.scores)
|
||||
|
||||
@property
|
||||
def max_score(self) -> float:
|
||||
"""Get maximum score."""
|
||||
if not self.scores:
|
||||
return 0.0
|
||||
return max(self.scores)
|
||||
|
||||
@property
|
||||
def std_dev(self) -> float:
|
||||
"""Calculate standard deviation of scores."""
|
||||
if len(self.scores) < 2:
|
||||
return 0.0
|
||||
return statistics.stdev(self.scores)
|
||||
|
||||
@property
|
||||
def passed(self) -> bool:
|
||||
"""Check if evaluation passed (avg score >= 7)."""
|
||||
return self.avg_score >= 7.0
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
"""Convert result to dictionary."""
|
||||
return {
|
||||
"eval_id": self.eval_id,
|
||||
"name": self.name,
|
||||
"evaluations": [asdict(e) for e in self.evaluations],
|
||||
"avg_score": self.avg_score,
|
||||
"min_score": self.min_score,
|
||||
"max_score": self.max_score,
|
||||
"std_dev": self.std_dev,
|
||||
"passed": self.passed,
|
||||
"num_evaluations": len(self.evaluations)
|
||||
}
|
||||
|
||||
def to_json(self) -> str:
|
||||
"""Convert result to JSON string."""
|
||||
return json.dumps(self.to_dict(), indent=2)
|
||||
|
||||
def print_summary(self) -> None:
|
||||
"""Print a summary of the evaluation results."""
|
||||
try:
|
||||
from rich.console import Console
|
||||
from rich.table import Table
|
||||
from rich.panel import Panel # noqa: F401
|
||||
|
||||
console = Console()
|
||||
|
||||
table = Table(title="Accuracy Evaluation Summary")
|
||||
table.add_column("Metric", style="cyan")
|
||||
table.add_column("Value", style="green")
|
||||
|
||||
table.add_row("Evaluations", str(len(self.evaluations)))
|
||||
table.add_row("Average Score", f"{self.avg_score:.2f}")
|
||||
table.add_row("Min Score", f"{self.min_score:.2f}")
|
||||
table.add_row("Max Score", f"{self.max_score:.2f}")
|
||||
table.add_row("Std Dev", f"{self.std_dev:.2f}")
|
||||
table.add_row("Status", "✅ PASSED" if self.passed else "❌ FAILED")
|
||||
|
||||
console.print(table)
|
||||
except ImportError:
|
||||
print("Accuracy Evaluation Summary")
|
||||
print(f" Evaluations: {len(self.evaluations)}")
|
||||
print(f" Average Score: {self.avg_score:.2f}")
|
||||
print(f" Min Score: {self.min_score:.2f}")
|
||||
print(f" Max Score: {self.max_score:.2f}")
|
||||
print(f" Std Dev: {self.std_dev:.2f}")
|
||||
print(f" Status: {'PASSED' if self.passed else 'FAILED'}")
|
||||
|
||||
|
||||
@dataclass
|
||||
class PerformanceMetrics:
|
||||
"""Individual performance measurement."""
|
||||
run_time: float
|
||||
memory_usage: float
|
||||
iteration: int
|
||||
timestamp: str = field(default_factory=lambda: datetime.now().isoformat())
|
||||
metadata: Dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
|
||||
@dataclass
|
||||
class PerformanceResult:
|
||||
"""Result from performance evaluation measuring runtime and memory."""
|
||||
metrics: List[PerformanceMetrics] = field(default_factory=list)
|
||||
warmup_runs: int = 0
|
||||
eval_id: str = ""
|
||||
name: str = ""
|
||||
|
||||
@property
|
||||
def run_times(self) -> List[float]:
|
||||
"""Get list of all run times."""
|
||||
return [m.run_time for m in self.metrics]
|
||||
|
||||
@property
|
||||
def memory_usages(self) -> List[float]:
|
||||
"""Get list of all memory usages."""
|
||||
return [m.memory_usage for m in self.metrics]
|
||||
|
||||
@property
|
||||
def avg_run_time(self) -> float:
|
||||
"""Calculate average run time."""
|
||||
if not self.run_times:
|
||||
return 0.0
|
||||
return statistics.mean(self.run_times)
|
||||
|
||||
@property
|
||||
def min_run_time(self) -> float:
|
||||
"""Get minimum run time."""
|
||||
if not self.run_times:
|
||||
return 0.0
|
||||
return min(self.run_times)
|
||||
|
||||
@property
|
||||
def max_run_time(self) -> float:
|
||||
"""Get maximum run time."""
|
||||
if not self.run_times:
|
||||
return 0.0
|
||||
return max(self.run_times)
|
||||
|
||||
@property
|
||||
def std_dev_run_time(self) -> float:
|
||||
"""Calculate standard deviation of run times."""
|
||||
if len(self.run_times) < 2:
|
||||
return 0.0
|
||||
return statistics.stdev(self.run_times)
|
||||
|
||||
@property
|
||||
def median_run_time(self) -> float:
|
||||
"""Calculate median run time."""
|
||||
if not self.run_times:
|
||||
return 0.0
|
||||
return statistics.median(self.run_times)
|
||||
|
||||
@property
|
||||
def p95_run_time(self) -> float:
|
||||
"""Calculate 95th percentile run time."""
|
||||
if not self.run_times:
|
||||
return 0.0
|
||||
sorted_times = sorted(self.run_times)
|
||||
idx = int(len(sorted_times) * 0.95)
|
||||
return sorted_times[min(idx, len(sorted_times) - 1)]
|
||||
|
||||
@property
|
||||
def avg_memory(self) -> float:
|
||||
"""Calculate average memory usage."""
|
||||
if not self.memory_usages:
|
||||
return 0.0
|
||||
return statistics.mean(self.memory_usages)
|
||||
|
||||
@property
|
||||
def max_memory(self) -> float:
|
||||
"""Get maximum memory usage."""
|
||||
if not self.memory_usages:
|
||||
return 0.0
|
||||
return max(self.memory_usages)
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
"""Convert result to dictionary."""
|
||||
return {
|
||||
"eval_id": self.eval_id,
|
||||
"name": self.name,
|
||||
"metrics": [asdict(m) for m in self.metrics],
|
||||
"warmup_runs": self.warmup_runs,
|
||||
"avg_run_time": self.avg_run_time,
|
||||
"min_run_time": self.min_run_time,
|
||||
"max_run_time": self.max_run_time,
|
||||
"std_dev_run_time": self.std_dev_run_time,
|
||||
"median_run_time": self.median_run_time,
|
||||
"p95_run_time": self.p95_run_time,
|
||||
"avg_memory": self.avg_memory,
|
||||
"max_memory": self.max_memory,
|
||||
"num_iterations": len(self.metrics)
|
||||
}
|
||||
|
||||
def to_json(self) -> str:
|
||||
"""Convert result to JSON string."""
|
||||
return json.dumps(self.to_dict(), indent=2)
|
||||
|
||||
def print_summary(self) -> None:
|
||||
"""Print a summary of the performance results."""
|
||||
try:
|
||||
from rich.console import Console
|
||||
from rich.table import Table
|
||||
|
||||
console = Console()
|
||||
|
||||
table = Table(title="Performance Evaluation Summary")
|
||||
table.add_column("Metric", style="cyan")
|
||||
table.add_column("Value", style="green")
|
||||
|
||||
table.add_row("Iterations", str(len(self.metrics)))
|
||||
table.add_row("Warmup Runs", str(self.warmup_runs))
|
||||
table.add_row("Avg Run Time", f"{self.avg_run_time:.4f}s")
|
||||
table.add_row("Min Run Time", f"{self.min_run_time:.4f}s")
|
||||
table.add_row("Max Run Time", f"{self.max_run_time:.4f}s")
|
||||
table.add_row("Std Dev", f"{self.std_dev_run_time:.4f}s")
|
||||
table.add_row("Median", f"{self.median_run_time:.4f}s")
|
||||
table.add_row("P95", f"{self.p95_run_time:.4f}s")
|
||||
table.add_row("Avg Memory", f"{self.avg_memory:.2f} MB")
|
||||
table.add_row("Max Memory", f"{self.max_memory:.2f} MB")
|
||||
|
||||
console.print(table)
|
||||
except ImportError:
|
||||
print("Performance Evaluation Summary")
|
||||
print(f" Iterations: {len(self.metrics)}")
|
||||
print(f" Warmup Runs: {self.warmup_runs}")
|
||||
print(f" Avg Run Time: {self.avg_run_time:.4f}s")
|
||||
print(f" Min Run Time: {self.min_run_time:.4f}s")
|
||||
print(f" Max Run Time: {self.max_run_time:.4f}s")
|
||||
print(f" Std Dev: {self.std_dev_run_time:.4f}s")
|
||||
print(f" Median: {self.median_run_time:.4f}s")
|
||||
print(f" P95: {self.p95_run_time:.4f}s")
|
||||
print(f" Avg Memory: {self.avg_memory:.2f} MB")
|
||||
print(f" Max Memory: {self.max_memory:.2f} MB")
|
||||
|
||||
|
||||
@dataclass
|
||||
class ToolCallResult:
|
||||
"""Result of a single tool call verification."""
|
||||
tool_name: str
|
||||
expected: bool
|
||||
actual: bool
|
||||
passed: bool
|
||||
arguments: Optional[Dict[str, Any]] = None
|
||||
metadata: Dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
|
||||
@dataclass
|
||||
class ReliabilityResult:
|
||||
"""Result from reliability evaluation checking tool calls."""
|
||||
tool_results: List[ToolCallResult] = field(default_factory=list)
|
||||
eval_id: str = ""
|
||||
name: str = ""
|
||||
|
||||
@property
|
||||
def passed_calls(self) -> List[ToolCallResult]:
|
||||
"""Get list of passed tool calls."""
|
||||
return [t for t in self.tool_results if t.passed]
|
||||
|
||||
@property
|
||||
def failed_calls(self) -> List[ToolCallResult]:
|
||||
"""Get list of failed tool calls."""
|
||||
return [t for t in self.tool_results if not t.passed]
|
||||
|
||||
@property
|
||||
def pass_rate(self) -> float:
|
||||
"""Calculate pass rate."""
|
||||
if not self.tool_results:
|
||||
return 0.0
|
||||
return len(self.passed_calls) / len(self.tool_results)
|
||||
|
||||
@property
|
||||
def status(self) -> str:
|
||||
"""Get overall status."""
|
||||
return "PASSED" if len(self.failed_calls) == 0 else "FAILED"
|
||||
|
||||
@property
|
||||
def passed(self) -> bool:
|
||||
"""Check if all tool calls passed."""
|
||||
return len(self.failed_calls) == 0
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
"""Convert result to dictionary."""
|
||||
return {
|
||||
"eval_id": self.eval_id,
|
||||
"name": self.name,
|
||||
"tool_results": [asdict(t) for t in self.tool_results],
|
||||
"passed_count": len(self.passed_calls),
|
||||
"failed_count": len(self.failed_calls),
|
||||
"pass_rate": self.pass_rate,
|
||||
"status": self.status
|
||||
}
|
||||
|
||||
def to_json(self) -> str:
|
||||
"""Convert result to JSON string."""
|
||||
return json.dumps(self.to_dict(), indent=2)
|
||||
|
||||
def print_summary(self) -> None:
|
||||
"""Print a summary of the reliability results."""
|
||||
try:
|
||||
from rich.console import Console
|
||||
from rich.table import Table
|
||||
|
||||
console = Console()
|
||||
|
||||
table = Table(title="Reliability Evaluation Summary")
|
||||
table.add_column("Metric", style="cyan")
|
||||
table.add_column("Value", style="green")
|
||||
|
||||
table.add_row("Total Tool Calls", str(len(self.tool_results)))
|
||||
table.add_row("Passed", str(len(self.passed_calls)))
|
||||
table.add_row("Failed", str(len(self.failed_calls)))
|
||||
table.add_row("Pass Rate", f"{self.pass_rate:.1%}")
|
||||
table.add_row("Status", "✅ PASSED" if self.passed else "❌ FAILED")
|
||||
|
||||
console.print(table)
|
||||
|
||||
if self.failed_calls:
|
||||
fail_table = Table(title="Failed Tool Calls")
|
||||
fail_table.add_column("Tool", style="red")
|
||||
fail_table.add_column("Expected", style="yellow")
|
||||
fail_table.add_column("Actual", style="yellow")
|
||||
|
||||
for tc in self.failed_calls:
|
||||
fail_table.add_row(tc.tool_name, str(tc.expected), str(tc.actual))
|
||||
|
||||
console.print(fail_table)
|
||||
except ImportError:
|
||||
print("Reliability Evaluation Summary")
|
||||
print(f" Total Tool Calls: {len(self.tool_results)}")
|
||||
print(f" Passed: {len(self.passed_calls)}")
|
||||
print(f" Failed: {len(self.failed_calls)}")
|
||||
print(f" Pass Rate: {self.pass_rate:.1%}")
|
||||
print(f" Status: {self.status}")
|
||||
|
||||
|
||||
@dataclass
|
||||
class CriteriaScore:
|
||||
"""Individual criteria evaluation score."""
|
||||
score: float
|
||||
passed: bool
|
||||
reasoning: str
|
||||
output_text: str
|
||||
criteria: str
|
||||
timestamp: str = field(default_factory=lambda: datetime.now().isoformat())
|
||||
metadata: Dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
|
||||
@dataclass
|
||||
class CriteriaResult:
|
||||
"""Result from criteria-based evaluation (agent-as-judge)."""
|
||||
evaluations: List[CriteriaScore] = field(default_factory=list)
|
||||
criteria: str = ""
|
||||
scoring_type: str = "numeric" # "numeric" or "binary"
|
||||
threshold: float = 7.0
|
||||
eval_id: str = ""
|
||||
name: str = ""
|
||||
|
||||
@property
|
||||
def scores(self) -> List[float]:
|
||||
"""Get list of all scores."""
|
||||
return [e.score for e in self.evaluations]
|
||||
|
||||
@property
|
||||
def avg_score(self) -> float:
|
||||
"""Calculate average score."""
|
||||
if not self.scores:
|
||||
return 0.0
|
||||
return statistics.mean(self.scores)
|
||||
|
||||
@property
|
||||
def pass_rate(self) -> float:
|
||||
"""Calculate pass rate."""
|
||||
if not self.evaluations:
|
||||
return 0.0
|
||||
passed = sum(1 for e in self.evaluations if e.passed)
|
||||
return passed / len(self.evaluations)
|
||||
|
||||
@property
|
||||
def passed(self) -> bool:
|
||||
"""Check if evaluation passed overall."""
|
||||
if self.scoring_type == "binary":
|
||||
return self.pass_rate >= 0.5
|
||||
return self.avg_score >= self.threshold
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
"""Convert result to dictionary."""
|
||||
return {
|
||||
"eval_id": self.eval_id,
|
||||
"name": self.name,
|
||||
"criteria": self.criteria,
|
||||
"scoring_type": self.scoring_type,
|
||||
"threshold": self.threshold,
|
||||
"evaluations": [asdict(e) for e in self.evaluations],
|
||||
"avg_score": self.avg_score,
|
||||
"pass_rate": self.pass_rate,
|
||||
"passed": self.passed,
|
||||
"num_evaluations": len(self.evaluations)
|
||||
}
|
||||
|
||||
def to_json(self) -> str:
|
||||
"""Convert result to JSON string."""
|
||||
return json.dumps(self.to_dict(), indent=2)
|
||||
|
||||
def print_summary(self) -> None:
|
||||
"""Print a summary of the criteria evaluation results."""
|
||||
try:
|
||||
from rich.console import Console
|
||||
from rich.table import Table
|
||||
|
||||
console = Console()
|
||||
|
||||
table = Table(title="Criteria Evaluation Summary")
|
||||
table.add_column("Metric", style="cyan")
|
||||
table.add_column("Value", style="green")
|
||||
|
||||
table.add_row("Criteria", self.criteria[:50] + "..." if len(self.criteria) > 50 else self.criteria)
|
||||
table.add_row("Scoring Type", self.scoring_type)
|
||||
table.add_row("Threshold", str(self.threshold))
|
||||
table.add_row("Evaluations", str(len(self.evaluations)))
|
||||
table.add_row("Average Score", f"{self.avg_score:.2f}")
|
||||
table.add_row("Pass Rate", f"{self.pass_rate:.1%}")
|
||||
table.add_row("Status", "✅ PASSED" if self.passed else "❌ FAILED")
|
||||
|
||||
console.print(table)
|
||||
except ImportError:
|
||||
print("Criteria Evaluation Summary")
|
||||
print(f" Criteria: {self.criteria[:50]}...")
|
||||
print(f" Scoring Type: {self.scoring_type}")
|
||||
print(f" Threshold: {self.threshold}")
|
||||
print(f" Evaluations: {len(self.evaluations)}")
|
||||
print(f" Average Score: {self.avg_score:.2f}")
|
||||
print(f" Pass Rate: {self.pass_rate:.1%}")
|
||||
print(f" Status: {'PASSED' if self.passed else 'FAILED'}")
|
||||
@@ -0,0 +1,154 @@
|
||||
"""
|
||||
Utility functions for PraisonAI Agents evaluation framework.
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, Optional, Union
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def save_result_to_file(
|
||||
result: Any,
|
||||
file_path: str,
|
||||
eval_id: Optional[str] = None,
|
||||
name: Optional[str] = None
|
||||
) -> bool:
|
||||
"""
|
||||
Save evaluation result to a file.
|
||||
|
||||
Args:
|
||||
result: The evaluation result to save
|
||||
file_path: Path to save the result (supports {name}, {eval_id} placeholders)
|
||||
eval_id: Optional eval ID for placeholder substitution
|
||||
name: Optional name for placeholder substitution
|
||||
|
||||
Returns:
|
||||
True if save was successful, False otherwise
|
||||
"""
|
||||
try:
|
||||
formatted_path = file_path.format(name=name or "", eval_id=eval_id or "")
|
||||
path = Path(formatted_path)
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
if hasattr(result, 'to_dict'):
|
||||
data = result.to_dict()
|
||||
elif hasattr(result, '__dict__'):
|
||||
data = result.__dict__
|
||||
else:
|
||||
data = {"result": str(result)}
|
||||
|
||||
path.write_text(json.dumps(data, indent=2, default=str))
|
||||
logger.debug(f"Saved result to: {formatted_path}")
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to save result: {e}")
|
||||
return False
|
||||
|
||||
|
||||
def load_result_from_file(file_path: str) -> Optional[Dict[str, Any]]:
|
||||
"""
|
||||
Load evaluation result from a file.
|
||||
|
||||
Args:
|
||||
file_path: Path to the result file
|
||||
|
||||
Returns:
|
||||
Dictionary containing the result data, or None if load failed
|
||||
"""
|
||||
try:
|
||||
path = Path(file_path)
|
||||
if not path.exists():
|
||||
logger.warning(f"Result file not found: {file_path}")
|
||||
return None
|
||||
|
||||
data = json.loads(path.read_text())
|
||||
return data
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to load result: {e}")
|
||||
return None
|
||||
|
||||
|
||||
def format_score(score: float, max_score: float = 10.0) -> str:
|
||||
"""
|
||||
Format a score for display.
|
||||
|
||||
Args:
|
||||
score: The score value
|
||||
max_score: Maximum possible score
|
||||
|
||||
Returns:
|
||||
Formatted score string
|
||||
"""
|
||||
return f"{score:.1f}/{max_score:.0f}"
|
||||
|
||||
|
||||
def format_percentage(value: float) -> str:
|
||||
"""
|
||||
Format a value as a percentage.
|
||||
|
||||
Args:
|
||||
value: Value between 0 and 1
|
||||
|
||||
Returns:
|
||||
Formatted percentage string
|
||||
"""
|
||||
return f"{value * 100:.1f}%"
|
||||
|
||||
|
||||
def format_duration(seconds: float) -> str:
|
||||
"""
|
||||
Format a duration in seconds for display.
|
||||
|
||||
Args:
|
||||
seconds: Duration in seconds
|
||||
|
||||
Returns:
|
||||
Formatted duration string
|
||||
"""
|
||||
if seconds < 0.001:
|
||||
return f"{seconds * 1000000:.0f}μs"
|
||||
elif seconds < 1:
|
||||
return f"{seconds * 1000:.2f}ms"
|
||||
elif seconds < 60:
|
||||
return f"{seconds:.2f}s"
|
||||
else:
|
||||
minutes = int(seconds // 60)
|
||||
secs = seconds % 60
|
||||
return f"{minutes}m {secs:.1f}s"
|
||||
|
||||
|
||||
def format_memory(mb: float) -> str:
|
||||
"""
|
||||
Format memory usage for display.
|
||||
|
||||
Args:
|
||||
mb: Memory in megabytes
|
||||
|
||||
Returns:
|
||||
Formatted memory string
|
||||
"""
|
||||
if mb < 1:
|
||||
return f"{mb * 1024:.1f} KB"
|
||||
elif mb < 1024:
|
||||
return f"{mb:.2f} MB"
|
||||
else:
|
||||
return f"{mb / 1024:.2f} GB"
|
||||
|
||||
|
||||
def calculate_pass_rate(passed: int, total: int) -> float:
|
||||
"""
|
||||
Calculate pass rate.
|
||||
|
||||
Args:
|
||||
passed: Number of passed evaluations
|
||||
total: Total number of evaluations
|
||||
|
||||
Returns:
|
||||
Pass rate as a float between 0 and 1
|
||||
"""
|
||||
if total == 0:
|
||||
return 0.0
|
||||
return passed / total
|
||||
@@ -1054,7 +1054,7 @@ class LLM:
|
||||
**kwargs
|
||||
) -> str:
|
||||
"""Enhanced get_response with all OpenAI-like features"""
|
||||
logging.info(f"Getting response from {self.model}")
|
||||
logging.debug(f"Getting response from {self.model}")
|
||||
# Log all self values when in debug mode
|
||||
self._log_llm_config(
|
||||
'LLM instance',
|
||||
@@ -2472,7 +2472,7 @@ Output MUST be JSON with 'reflection' and 'satisfactory'.
|
||||
"""Async version of get_response with identical functionality."""
|
||||
try:
|
||||
import litellm
|
||||
logging.info(f"Getting async response from {self.model}")
|
||||
logging.debug(f"Getting async response from {self.model}")
|
||||
# Log all self values when in debug mode
|
||||
self._log_llm_config(
|
||||
'get_response_async',
|
||||
|
||||
@@ -19,6 +19,15 @@ from rich.console import Console
|
||||
from rich.live import Live
|
||||
import inspect
|
||||
|
||||
# Import display_tool_call for callback support (lazy import to avoid circular imports)
|
||||
_display_tool_call = None
|
||||
def _get_display_tool_call():
|
||||
global _display_tool_call
|
||||
if _display_tool_call is None:
|
||||
from ..main import display_tool_call
|
||||
_display_tool_call = display_tool_call
|
||||
return _display_tool_call
|
||||
|
||||
# Constants
|
||||
LOCAL_SERVER_API_KEY_PLACEHOLDER = "not-needed"
|
||||
|
||||
@@ -978,16 +987,19 @@ class OpenAIClient:
|
||||
function_name = tool_call.function.name
|
||||
arguments = json.loads(tool_call.function.arguments)
|
||||
|
||||
# Always trigger callback for tool call tracking (even when verbose=False)
|
||||
display_tool_call_fn = _get_display_tool_call()
|
||||
display_tool_call_fn(f"Calling function: {function_name}", console=console if verbose else None)
|
||||
|
||||
if verbose and console:
|
||||
console.print(f"[bold]Calling function:[/bold] {function_name}")
|
||||
console.print(f"[dim]Arguments:[/dim] {arguments}")
|
||||
|
||||
# Execute the tool
|
||||
tool_result = execute_tool_fn(function_name, arguments)
|
||||
results_str = json.dumps(tool_result) if tool_result else "Function returned an empty output"
|
||||
|
||||
if verbose and console:
|
||||
console.print(f"[dim]Result:[/dim] {results_str}")
|
||||
# Trigger callback with result
|
||||
display_tool_call_fn(f"Function {function_name} returned: {results_str[:200]}{'...' if len(results_str) > 200 else ''}", console=console if verbose else None)
|
||||
|
||||
messages.append({
|
||||
"role": "tool",
|
||||
@@ -1150,8 +1162,11 @@ class OpenAIClient:
|
||||
function_name = tool_call.function.name
|
||||
arguments = json.loads(tool_call.function.arguments)
|
||||
|
||||
# Always trigger callback for tool call tracking (even when verbose=False)
|
||||
display_tool_call_fn = _get_display_tool_call()
|
||||
display_tool_call_fn(f"Calling function: {function_name}", console=console if verbose else None)
|
||||
|
||||
if verbose and console:
|
||||
console.print(f"[bold]Calling function:[/bold] {function_name}")
|
||||
console.print(f"[dim]Arguments:[/dim] {arguments}")
|
||||
|
||||
# Execute the tool (async)
|
||||
@@ -1167,8 +1182,8 @@ class OpenAIClient:
|
||||
|
||||
results_str = json.dumps(tool_result) if tool_result else "Function returned an empty output"
|
||||
|
||||
if verbose and console:
|
||||
console.print(f"[dim]Result:[/dim] {results_str}")
|
||||
# Trigger callback with result
|
||||
display_tool_call_fn(f"Function {function_name} returned: {results_str[:200]}{'...' if len(results_str) > 200 else ''}", console=console if verbose else None)
|
||||
|
||||
messages.append({
|
||||
"role": "tool",
|
||||
@@ -1299,6 +1314,10 @@ class OpenAIClient:
|
||||
yield f"\n[Error parsing arguments for {function_name if 'function_name' in locals() else 'unknown function'}: {str(e)}]"
|
||||
continue
|
||||
|
||||
# Always trigger callback for tool call tracking (even when verbose=False)
|
||||
display_tool_call_fn = _get_display_tool_call()
|
||||
display_tool_call_fn(f"Calling function: {function_name}", console=None)
|
||||
|
||||
if verbose:
|
||||
yield f"\n[Calling function: {function_name}]"
|
||||
|
||||
@@ -1311,6 +1330,9 @@ class OpenAIClient:
|
||||
if verbose:
|
||||
yield f"\n[Function error: {str(e)}]"
|
||||
|
||||
# Trigger callback with result
|
||||
display_tool_call_fn(f"Function {function_name} returned: {results_str[:200]}{'...' if len(results_str) > 200 else ''}", console=None)
|
||||
|
||||
if verbose:
|
||||
yield f"\n[Function result: {results_str}]"
|
||||
|
||||
|
||||
@@ -231,15 +231,15 @@ def display_tool_call(message: str, console=None):
|
||||
if not message or not message.strip():
|
||||
logging.debug("Empty message in display_tool_call, returning early")
|
||||
return
|
||||
if console is None:
|
||||
console = Console()
|
||||
message = _clean_display_content(str(message))
|
||||
logging.debug(f"Cleaned message in display_tool_call: {repr(message)}")
|
||||
|
||||
# Execute synchronous callbacks
|
||||
# Execute synchronous callbacks (always, even when console is None)
|
||||
execute_sync_callback('tool_call', message=message)
|
||||
|
||||
console.print(Panel.fit(Text(message, style="bold cyan"), title="Tool Call", border_style="green"))
|
||||
# Only print panel if console is provided (verbose mode)
|
||||
if console is not None:
|
||||
console.print(Panel.fit(Text(message, style="bold cyan"), title="Tool Call", border_style="green"))
|
||||
|
||||
def display_error(message: str, console=None):
|
||||
if not message or not message.strip():
|
||||
|
||||
@@ -0,0 +1,76 @@
|
||||
"""Agent Skills module for PraisonAI Agents.
|
||||
|
||||
This module provides support for the open Agent Skills standard (agentskills.io),
|
||||
enabling agents to load and use modular capabilities through SKILL.md files.
|
||||
|
||||
Zero Performance Impact:
|
||||
- All imports are lazy loaded via __getattr__
|
||||
- Skills only loaded when explicitly enabled
|
||||
- No auto-discovery at import time
|
||||
|
||||
Usage:
|
||||
from praisonaiagents.skills import SkillManager, SkillProperties
|
||||
|
||||
manager = SkillManager()
|
||||
manager.discover(["./skills"])
|
||||
prompt_xml = manager.to_prompt()
|
||||
"""
|
||||
|
||||
__all__ = [
|
||||
# Models
|
||||
"SkillProperties",
|
||||
"SkillMetadata",
|
||||
"ParseError",
|
||||
"ValidationError",
|
||||
# Parser
|
||||
"parse_frontmatter",
|
||||
"find_skill_md",
|
||||
"read_properties",
|
||||
# Validator
|
||||
"validate",
|
||||
"validate_metadata",
|
||||
# Prompt
|
||||
"to_prompt",
|
||||
"generate_skills_xml",
|
||||
"format_skill_for_prompt",
|
||||
# Discovery
|
||||
"discover_skills",
|
||||
"get_default_skill_dirs",
|
||||
# Loader
|
||||
"SkillLoader",
|
||||
# Manager
|
||||
"SkillManager",
|
||||
]
|
||||
|
||||
|
||||
def __getattr__(name: str):
|
||||
"""Lazy load module components to avoid import overhead."""
|
||||
if name in ("SkillProperties", "SkillMetadata", "ParseError", "ValidationError"):
|
||||
from .models import SkillProperties, SkillMetadata, ParseError, ValidationError
|
||||
return locals()[name]
|
||||
|
||||
if name in ("parse_frontmatter", "find_skill_md", "read_properties"):
|
||||
from .parser import parse_frontmatter, find_skill_md, read_properties
|
||||
return locals()[name]
|
||||
|
||||
if name in ("validate", "validate_metadata", "_validate_name", "_validate_description", "_validate_compatibility"):
|
||||
from .validator import validate, validate_metadata, _validate_name, _validate_description, _validate_compatibility
|
||||
return locals()[name]
|
||||
|
||||
if name in ("to_prompt", "generate_skills_xml", "format_skill_for_prompt"):
|
||||
from .prompt import to_prompt, generate_skills_xml, format_skill_for_prompt
|
||||
return locals()[name]
|
||||
|
||||
if name in ("discover_skills", "get_default_skill_dirs"):
|
||||
from .discovery import discover_skills, get_default_skill_dirs
|
||||
return locals()[name]
|
||||
|
||||
if name == "SkillLoader":
|
||||
from .loader import SkillLoader
|
||||
return SkillLoader
|
||||
|
||||
if name == "SkillManager":
|
||||
from .manager import SkillManager
|
||||
return SkillManager
|
||||
|
||||
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
||||
@@ -0,0 +1,129 @@
|
||||
"""Skill discovery and directory scanning."""
|
||||
|
||||
from pathlib import Path
|
||||
from typing import List, Optional
|
||||
|
||||
from .parser import find_skill_md, read_properties
|
||||
from .models import SkillProperties
|
||||
|
||||
|
||||
def get_default_skill_dirs() -> List[Path]:
|
||||
"""Get default skill directory locations.
|
||||
|
||||
Returns directories in precedence order (high to low):
|
||||
1. Project: ./.praison/skills/ or ./.claude/skills/
|
||||
2. User: ~/.praison/skills/
|
||||
3. System: /etc/praison/skills/ (admin-managed)
|
||||
|
||||
Returns:
|
||||
List of existing skill directories
|
||||
"""
|
||||
dirs = []
|
||||
cwd = Path.cwd()
|
||||
|
||||
# Project-level directories
|
||||
project_dirs = [
|
||||
cwd / ".praison" / "skills",
|
||||
cwd / ".claude" / "skills",
|
||||
]
|
||||
for d in project_dirs:
|
||||
if d.exists() and d.is_dir():
|
||||
dirs.append(d)
|
||||
|
||||
# User-level directory
|
||||
user_dir = Path.home() / ".praison" / "skills"
|
||||
if user_dir.exists() and user_dir.is_dir():
|
||||
dirs.append(user_dir)
|
||||
|
||||
# System-level directory (Unix-like systems)
|
||||
system_dir = Path("/etc/praison/skills")
|
||||
if system_dir.exists() and system_dir.is_dir():
|
||||
dirs.append(system_dir)
|
||||
|
||||
return dirs
|
||||
|
||||
|
||||
def discover_skills(
|
||||
skill_dirs: Optional[List[str]] = None,
|
||||
include_defaults: bool = True,
|
||||
) -> List[SkillProperties]:
|
||||
"""Discover all valid skills in the given directories.
|
||||
|
||||
Args:
|
||||
skill_dirs: List of directory paths to scan for skills.
|
||||
Each directory should contain skill subdirectories.
|
||||
include_defaults: Whether to include default skill directories
|
||||
|
||||
Returns:
|
||||
List of SkillProperties for all valid skills found
|
||||
"""
|
||||
all_dirs = []
|
||||
|
||||
# Add explicit directories
|
||||
if skill_dirs:
|
||||
for d in skill_dirs:
|
||||
path = Path(d).expanduser().resolve()
|
||||
if path.exists() and path.is_dir():
|
||||
all_dirs.append(path)
|
||||
|
||||
# Add default directories
|
||||
if include_defaults:
|
||||
all_dirs.extend(get_default_skill_dirs())
|
||||
|
||||
# Remove duplicates while preserving order
|
||||
seen = set()
|
||||
unique_dirs = []
|
||||
for d in all_dirs:
|
||||
if d not in seen:
|
||||
seen.add(d)
|
||||
unique_dirs.append(d)
|
||||
|
||||
skills = []
|
||||
|
||||
for parent_dir in unique_dirs:
|
||||
# Each subdirectory in parent_dir might be a skill
|
||||
try:
|
||||
for item in parent_dir.iterdir():
|
||||
if not item.is_dir():
|
||||
continue
|
||||
|
||||
# Check if this directory contains a SKILL.md
|
||||
skill_md = find_skill_md(item)
|
||||
if skill_md is None:
|
||||
continue
|
||||
|
||||
try:
|
||||
props = read_properties(item)
|
||||
skills.append(props)
|
||||
except Exception:
|
||||
# Skip invalid skills
|
||||
continue
|
||||
except PermissionError:
|
||||
# Skip directories we can't read
|
||||
continue
|
||||
|
||||
return skills
|
||||
|
||||
|
||||
def discover_skill(skill_path: str) -> Optional[SkillProperties]:
|
||||
"""Discover a single skill from a directory path.
|
||||
|
||||
Args:
|
||||
skill_path: Path to a skill directory
|
||||
|
||||
Returns:
|
||||
SkillProperties if valid, None otherwise
|
||||
"""
|
||||
path = Path(skill_path).expanduser().resolve()
|
||||
|
||||
if not path.exists() or not path.is_dir():
|
||||
return None
|
||||
|
||||
skill_md = find_skill_md(path)
|
||||
if skill_md is None:
|
||||
return None
|
||||
|
||||
try:
|
||||
return read_properties(path)
|
||||
except Exception:
|
||||
return None
|
||||
@@ -0,0 +1,236 @@
|
||||
"""Progressive skill loading for Agent Skills."""
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
from .parser import find_skill_md, read_properties, parse_frontmatter
|
||||
from .models import SkillProperties, SkillMetadata
|
||||
|
||||
|
||||
@dataclass
|
||||
class LoadedSkill:
|
||||
"""A fully or partially loaded skill.
|
||||
|
||||
Supports progressive disclosure:
|
||||
- Level 1: metadata only (name, description, location)
|
||||
- Level 2: full instructions (SKILL.md body)
|
||||
- Level 3: resources (scripts, references, assets)
|
||||
"""
|
||||
|
||||
properties: SkillProperties
|
||||
instructions: Optional[str] = None
|
||||
resources_loaded: bool = False
|
||||
_scripts: dict = field(default_factory=dict)
|
||||
_references: dict = field(default_factory=dict)
|
||||
_assets: dict = field(default_factory=dict)
|
||||
|
||||
@property
|
||||
def metadata(self) -> SkillMetadata:
|
||||
"""Get lightweight metadata for system prompt."""
|
||||
return SkillMetadata.from_properties(self.properties)
|
||||
|
||||
@property
|
||||
def is_activated(self) -> bool:
|
||||
"""Check if skill has been activated (instructions loaded)."""
|
||||
return self.instructions is not None
|
||||
|
||||
def get_scripts(self) -> dict:
|
||||
"""Get loaded scripts."""
|
||||
return self._scripts
|
||||
|
||||
def get_references(self) -> dict:
|
||||
"""Get loaded references."""
|
||||
return self._references
|
||||
|
||||
def get_assets(self) -> dict:
|
||||
"""Get loaded assets."""
|
||||
return self._assets
|
||||
|
||||
|
||||
class SkillLoader:
|
||||
"""Loader for Agent Skills with progressive disclosure support.
|
||||
|
||||
Usage:
|
||||
loader = SkillLoader()
|
||||
|
||||
# Level 1: Load metadata only
|
||||
skill = loader.load_metadata("/path/to/skill")
|
||||
|
||||
# Level 2: Activate skill (load instructions)
|
||||
loader.activate(skill)
|
||||
|
||||
# Level 3: Load specific resources
|
||||
loader.load_scripts(skill)
|
||||
loader.load_references(skill)
|
||||
"""
|
||||
|
||||
def load_metadata(self, skill_path: str) -> Optional[LoadedSkill]:
|
||||
"""Load skill metadata only (Level 1).
|
||||
|
||||
This is the minimal load for system prompt injection.
|
||||
Only reads the YAML frontmatter, not the full SKILL.md body.
|
||||
|
||||
Args:
|
||||
skill_path: Path to skill directory
|
||||
|
||||
Returns:
|
||||
LoadedSkill with metadata only, or None if invalid
|
||||
"""
|
||||
path = Path(skill_path).expanduser().resolve()
|
||||
|
||||
if not path.exists() or not path.is_dir():
|
||||
return None
|
||||
|
||||
try:
|
||||
props = read_properties(path)
|
||||
return LoadedSkill(properties=props)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
def activate(self, skill: LoadedSkill) -> bool:
|
||||
"""Activate a skill by loading its full instructions (Level 2).
|
||||
|
||||
Args:
|
||||
skill: LoadedSkill instance to activate
|
||||
|
||||
Returns:
|
||||
True if activation succeeded
|
||||
"""
|
||||
if skill.is_activated:
|
||||
return True
|
||||
|
||||
if skill.properties.path is None:
|
||||
return False
|
||||
|
||||
skill_md = find_skill_md(skill.properties.path)
|
||||
if skill_md is None:
|
||||
return False
|
||||
|
||||
try:
|
||||
content = skill_md.read_text()
|
||||
_, body = parse_frontmatter(content)
|
||||
skill.instructions = body
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
def load_scripts(self, skill: LoadedSkill) -> dict:
|
||||
"""Load scripts from skill's scripts/ directory (Level 3).
|
||||
|
||||
Args:
|
||||
skill: LoadedSkill instance
|
||||
|
||||
Returns:
|
||||
Dict mapping script names to their contents
|
||||
"""
|
||||
if skill.properties.path is None:
|
||||
return {}
|
||||
|
||||
scripts_dir = skill.properties.path / "scripts"
|
||||
if not scripts_dir.exists() or not scripts_dir.is_dir():
|
||||
return {}
|
||||
|
||||
scripts = {}
|
||||
try:
|
||||
for script_file in scripts_dir.iterdir():
|
||||
if script_file.is_file():
|
||||
try:
|
||||
scripts[script_file.name] = script_file.read_text()
|
||||
except Exception:
|
||||
continue
|
||||
except PermissionError:
|
||||
pass
|
||||
|
||||
skill._scripts = scripts
|
||||
return scripts
|
||||
|
||||
def load_references(self, skill: LoadedSkill) -> dict:
|
||||
"""Load references from skill's references/ directory (Level 3).
|
||||
|
||||
Args:
|
||||
skill: LoadedSkill instance
|
||||
|
||||
Returns:
|
||||
Dict mapping reference names to their contents
|
||||
"""
|
||||
if skill.properties.path is None:
|
||||
return {}
|
||||
|
||||
refs_dir = skill.properties.path / "references"
|
||||
if not refs_dir.exists() or not refs_dir.is_dir():
|
||||
return {}
|
||||
|
||||
refs = {}
|
||||
try:
|
||||
for ref_file in refs_dir.iterdir():
|
||||
if ref_file.is_file():
|
||||
try:
|
||||
refs[ref_file.name] = ref_file.read_text()
|
||||
except Exception:
|
||||
continue
|
||||
except PermissionError:
|
||||
pass
|
||||
|
||||
skill._references = refs
|
||||
return refs
|
||||
|
||||
def load_assets(self, skill: LoadedSkill) -> dict:
|
||||
"""Load asset paths from skill's assets/ directory (Level 3).
|
||||
|
||||
Note: For binary assets, only paths are returned, not contents.
|
||||
|
||||
Args:
|
||||
skill: LoadedSkill instance
|
||||
|
||||
Returns:
|
||||
Dict mapping asset names to their paths
|
||||
"""
|
||||
if skill.properties.path is None:
|
||||
return {}
|
||||
|
||||
assets_dir = skill.properties.path / "assets"
|
||||
if not assets_dir.exists() or not assets_dir.is_dir():
|
||||
return {}
|
||||
|
||||
assets = {}
|
||||
try:
|
||||
for asset_file in assets_dir.iterdir():
|
||||
if asset_file.is_file():
|
||||
assets[asset_file.name] = str(asset_file)
|
||||
except PermissionError:
|
||||
pass
|
||||
|
||||
skill._assets = assets
|
||||
skill.resources_loaded = True
|
||||
return assets
|
||||
|
||||
def load_all_resources(self, skill: LoadedSkill) -> None:
|
||||
"""Load all resources for a skill (Level 3 complete).
|
||||
|
||||
Args:
|
||||
skill: LoadedSkill instance
|
||||
"""
|
||||
self.load_scripts(skill)
|
||||
self.load_references(skill)
|
||||
self.load_assets(skill)
|
||||
skill.resources_loaded = True
|
||||
|
||||
@classmethod
|
||||
def load(cls, skill_path: str, activate: bool = False) -> Optional[LoadedSkill]:
|
||||
"""Convenience method to load a skill.
|
||||
|
||||
Args:
|
||||
skill_path: Path to skill directory
|
||||
activate: Whether to also load instructions
|
||||
|
||||
Returns:
|
||||
LoadedSkill or None if invalid
|
||||
"""
|
||||
loader = cls()
|
||||
skill = loader.load_metadata(skill_path)
|
||||
|
||||
if skill and activate:
|
||||
loader.activate(skill)
|
||||
|
||||
return skill
|
||||
@@ -0,0 +1,189 @@
|
||||
"""SkillManager for Agent Skills integration."""
|
||||
|
||||
from typing import List, Optional, Dict
|
||||
|
||||
from .models import SkillMetadata
|
||||
from .discovery import discover_skills
|
||||
from .loader import SkillLoader, LoadedSkill
|
||||
from .prompt import generate_skills_xml
|
||||
|
||||
|
||||
class SkillManager:
|
||||
"""Manager for discovering, loading, and using Agent Skills.
|
||||
|
||||
This is the main entry point for Agent Skills integration.
|
||||
Supports progressive disclosure and zero performance impact
|
||||
when skills are not used.
|
||||
|
||||
Usage:
|
||||
manager = SkillManager()
|
||||
|
||||
# Discover skills from directories
|
||||
manager.discover(["./skills", "~/.praison/skills"])
|
||||
|
||||
# Get XML for system prompt
|
||||
prompt_xml = manager.to_prompt()
|
||||
|
||||
# Activate a specific skill
|
||||
skill = manager.get_skill("pdf-processing")
|
||||
manager.activate(skill)
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
"""Initialize the SkillManager."""
|
||||
self._skills: Dict[str, LoadedSkill] = {}
|
||||
self._loader = SkillLoader()
|
||||
self._discovered = False
|
||||
|
||||
@property
|
||||
def skills(self) -> List[LoadedSkill]:
|
||||
"""Get all loaded skills."""
|
||||
return list(self._skills.values())
|
||||
|
||||
@property
|
||||
def skill_names(self) -> List[str]:
|
||||
"""Get names of all loaded skills."""
|
||||
return list(self._skills.keys())
|
||||
|
||||
def discover(
|
||||
self,
|
||||
skill_dirs: Optional[List[str]] = None,
|
||||
include_defaults: bool = True,
|
||||
) -> int:
|
||||
"""Discover skills from directories.
|
||||
|
||||
Args:
|
||||
skill_dirs: List of directory paths to scan
|
||||
include_defaults: Whether to include default skill directories
|
||||
|
||||
Returns:
|
||||
Number of skills discovered
|
||||
"""
|
||||
props_list = discover_skills(skill_dirs, include_defaults)
|
||||
|
||||
for props in props_list:
|
||||
if props.name not in self._skills:
|
||||
self._skills[props.name] = LoadedSkill(properties=props)
|
||||
|
||||
self._discovered = True
|
||||
return len(props_list)
|
||||
|
||||
def add_skill(self, skill_path: str) -> Optional[LoadedSkill]:
|
||||
"""Add a single skill from a directory path.
|
||||
|
||||
Args:
|
||||
skill_path: Path to skill directory
|
||||
|
||||
Returns:
|
||||
LoadedSkill if successful, None otherwise
|
||||
"""
|
||||
skill = self._loader.load_metadata(skill_path)
|
||||
if skill:
|
||||
self._skills[skill.properties.name] = skill
|
||||
return skill
|
||||
|
||||
def get_skill(self, name: str) -> Optional[LoadedSkill]:
|
||||
"""Get a skill by name.
|
||||
|
||||
Args:
|
||||
name: Skill name
|
||||
|
||||
Returns:
|
||||
LoadedSkill or None if not found
|
||||
"""
|
||||
return self._skills.get(name)
|
||||
|
||||
def activate(self, skill: LoadedSkill) -> bool:
|
||||
"""Activate a skill (load its instructions).
|
||||
|
||||
Args:
|
||||
skill: LoadedSkill to activate
|
||||
|
||||
Returns:
|
||||
True if activation succeeded
|
||||
"""
|
||||
return self._loader.activate(skill)
|
||||
|
||||
def activate_by_name(self, name: str) -> bool:
|
||||
"""Activate a skill by name.
|
||||
|
||||
Args:
|
||||
name: Skill name
|
||||
|
||||
Returns:
|
||||
True if activation succeeded
|
||||
"""
|
||||
skill = self.get_skill(name)
|
||||
if skill is None:
|
||||
return False
|
||||
return self.activate(skill)
|
||||
|
||||
def get_available_skills(self) -> List[SkillMetadata]:
|
||||
"""Get metadata for all available skills.
|
||||
|
||||
This is used for system prompt injection.
|
||||
|
||||
Returns:
|
||||
List of SkillMetadata instances
|
||||
"""
|
||||
return [skill.metadata for skill in self._skills.values()]
|
||||
|
||||
def to_prompt(self) -> str:
|
||||
"""Generate XML prompt for available skills.
|
||||
|
||||
Returns:
|
||||
XML string with <available_skills> block
|
||||
"""
|
||||
metadata_list = self.get_available_skills()
|
||||
return generate_skills_xml(metadata_list)
|
||||
|
||||
def get_instructions(self, name: str) -> Optional[str]:
|
||||
"""Get instructions for a skill, activating if needed.
|
||||
|
||||
Args:
|
||||
name: Skill name
|
||||
|
||||
Returns:
|
||||
Skill instructions or None if not found
|
||||
"""
|
||||
skill = self.get_skill(name)
|
||||
if skill is None:
|
||||
return None
|
||||
|
||||
if not skill.is_activated:
|
||||
self.activate(skill)
|
||||
|
||||
return skill.instructions
|
||||
|
||||
def load_resources(self, name: str) -> bool:
|
||||
"""Load all resources for a skill.
|
||||
|
||||
Args:
|
||||
name: Skill name
|
||||
|
||||
Returns:
|
||||
True if resources loaded successfully
|
||||
"""
|
||||
skill = self.get_skill(name)
|
||||
if skill is None:
|
||||
return False
|
||||
|
||||
self._loader.load_all_resources(skill)
|
||||
return True
|
||||
|
||||
def clear(self) -> None:
|
||||
"""Clear all loaded skills."""
|
||||
self._skills.clear()
|
||||
self._discovered = False
|
||||
|
||||
def __len__(self) -> int:
|
||||
"""Get number of loaded skills."""
|
||||
return len(self._skills)
|
||||
|
||||
def __contains__(self, name: str) -> bool:
|
||||
"""Check if a skill is loaded."""
|
||||
return name in self._skills
|
||||
|
||||
def __iter__(self):
|
||||
"""Iterate over loaded skills."""
|
||||
return iter(self._skills.values())
|
||||
@@ -0,0 +1,95 @@
|
||||
"""Data models for Agent Skills."""
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Optional
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
class ParseError(Exception):
|
||||
"""Raised when SKILL.md parsing fails."""
|
||||
pass
|
||||
|
||||
|
||||
class ValidationError(Exception):
|
||||
"""Raised when skill validation fails."""
|
||||
pass
|
||||
|
||||
|
||||
@dataclass
|
||||
class SkillProperties:
|
||||
"""Properties parsed from a skill's SKILL.md frontmatter.
|
||||
|
||||
Attributes:
|
||||
name: Skill name in kebab-case (required)
|
||||
description: What the skill does and when the model should use it (required)
|
||||
license: License for the skill (optional)
|
||||
compatibility: Compatibility information for the skill (optional)
|
||||
allowed_tools: Tool patterns the skill requires (optional, experimental)
|
||||
metadata: Key-value pairs for client-specific properties (defaults to
|
||||
empty dict; omitted from to_dict() output when empty)
|
||||
path: Path to the skill directory (optional, added for PraisonAI)
|
||||
"""
|
||||
|
||||
name: str
|
||||
description: str
|
||||
license: Optional[str] = None
|
||||
compatibility: Optional[str] = None
|
||||
allowed_tools: Optional[str] = None
|
||||
metadata: dict = field(default_factory=dict)
|
||||
path: Optional[Path] = None
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
"""Convert to dictionary, excluding None values and empty metadata."""
|
||||
result = {"name": self.name, "description": self.description}
|
||||
if self.license is not None:
|
||||
result["license"] = self.license
|
||||
if self.compatibility is not None:
|
||||
result["compatibility"] = self.compatibility
|
||||
if self.allowed_tools is not None:
|
||||
result["allowed-tools"] = self.allowed_tools
|
||||
if self.metadata:
|
||||
result["metadata"] = self.metadata
|
||||
return result
|
||||
|
||||
|
||||
@dataclass
|
||||
class SkillMetadata:
|
||||
"""Lightweight skill metadata for system prompt injection.
|
||||
|
||||
This is the minimal information needed to include a skill in the
|
||||
system prompt's <available_skills> block (~50-100 tokens per skill).
|
||||
|
||||
Attributes:
|
||||
name: Skill name
|
||||
description: What the skill does and when to use it
|
||||
location: Absolute path to the SKILL.md file
|
||||
"""
|
||||
|
||||
name: str
|
||||
description: str
|
||||
location: str
|
||||
|
||||
@classmethod
|
||||
def from_properties(cls, props: SkillProperties) -> "SkillMetadata":
|
||||
"""Create SkillMetadata from SkillProperties.
|
||||
|
||||
Args:
|
||||
props: SkillProperties instance with path set
|
||||
|
||||
Returns:
|
||||
SkillMetadata with location pointing to SKILL.md
|
||||
"""
|
||||
if props.path:
|
||||
# Look for SKILL.md or skill.md
|
||||
skill_md = props.path / "SKILL.md"
|
||||
if not skill_md.exists():
|
||||
skill_md = props.path / "skill.md"
|
||||
location = str(skill_md)
|
||||
else:
|
||||
location = ""
|
||||
|
||||
return cls(
|
||||
name=props.name,
|
||||
description=props.description,
|
||||
location=location
|
||||
)
|
||||
@@ -0,0 +1,165 @@
|
||||
"""YAML frontmatter parsing for SKILL.md files."""
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Optional, Tuple
|
||||
import re
|
||||
|
||||
from .models import ParseError, ValidationError, SkillProperties
|
||||
|
||||
|
||||
def find_skill_md(skill_dir: Path) -> Optional[Path]:
|
||||
"""Find the SKILL.md file in a skill directory.
|
||||
|
||||
Prefers SKILL.md (uppercase) but accepts skill.md (lowercase).
|
||||
|
||||
Args:
|
||||
skill_dir: Path to the skill directory
|
||||
|
||||
Returns:
|
||||
Path to the SKILL.md file, or None if not found
|
||||
"""
|
||||
for name in ("SKILL.md", "skill.md"):
|
||||
path = skill_dir / name
|
||||
if path.exists():
|
||||
return path
|
||||
return None
|
||||
|
||||
|
||||
def parse_frontmatter(content: str) -> Tuple[dict, str]:
|
||||
"""Parse YAML frontmatter from SKILL.md content.
|
||||
|
||||
Args:
|
||||
content: Raw content of SKILL.md file
|
||||
|
||||
Returns:
|
||||
Tuple of (metadata dict, markdown body)
|
||||
|
||||
Raises:
|
||||
ParseError: If frontmatter is missing or invalid
|
||||
"""
|
||||
if not content.startswith("---"):
|
||||
raise ParseError("SKILL.md must start with YAML frontmatter (---)")
|
||||
|
||||
parts = content.split("---", 2)
|
||||
if len(parts) < 3:
|
||||
raise ParseError("SKILL.md frontmatter not properly closed with ---")
|
||||
|
||||
frontmatter_str = parts[1]
|
||||
body = parts[2].strip()
|
||||
|
||||
try:
|
||||
metadata = _parse_yaml(frontmatter_str)
|
||||
except Exception as e:
|
||||
raise ParseError(f"Invalid YAML in frontmatter: {e}")
|
||||
|
||||
if not isinstance(metadata, dict):
|
||||
raise ParseError("SKILL.md frontmatter must be a YAML mapping")
|
||||
|
||||
# Convert metadata values to strings where needed
|
||||
if "metadata" in metadata and isinstance(metadata["metadata"], dict):
|
||||
metadata["metadata"] = {str(k): str(v) for k, v in metadata["metadata"].items()}
|
||||
|
||||
return metadata, body
|
||||
|
||||
|
||||
def _parse_yaml(yaml_str: str) -> dict:
|
||||
"""Parse YAML string to dictionary.
|
||||
|
||||
Uses a simple parser to avoid strictyaml dependency.
|
||||
Falls back to yaml if available.
|
||||
|
||||
Args:
|
||||
yaml_str: YAML string to parse
|
||||
|
||||
Returns:
|
||||
Parsed dictionary
|
||||
"""
|
||||
# Try to use PyYAML if available
|
||||
try:
|
||||
import yaml
|
||||
return yaml.safe_load(yaml_str) or {}
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
# Simple fallback parser for basic YAML
|
||||
result = {}
|
||||
current_key = None
|
||||
current_dict = None
|
||||
|
||||
for line in yaml_str.strip().split("\n"):
|
||||
if not line.strip():
|
||||
continue
|
||||
|
||||
# Check for nested dict
|
||||
if line.startswith(" ") and current_dict is not None:
|
||||
match = re.match(r"^\s+(\w+):\s*(.*)$", line)
|
||||
if match:
|
||||
key, value = match.groups()
|
||||
current_dict[key] = value.strip().strip('"').strip("'")
|
||||
continue
|
||||
|
||||
# Top-level key
|
||||
match = re.match(r"^(\S+):\s*(.*)$", line)
|
||||
if match:
|
||||
key, value = match.groups()
|
||||
value = value.strip()
|
||||
|
||||
if not value:
|
||||
# Could be start of nested dict
|
||||
result[key] = {}
|
||||
current_key = key
|
||||
current_dict = result[key]
|
||||
else:
|
||||
result[key] = value.strip('"').strip("'")
|
||||
current_dict = None
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def read_properties(skill_dir: Path) -> SkillProperties:
|
||||
"""Read skill properties from SKILL.md frontmatter.
|
||||
|
||||
This function parses the frontmatter and returns properties.
|
||||
It does NOT perform full validation. Use validate() for that.
|
||||
|
||||
Args:
|
||||
skill_dir: Path to the skill directory
|
||||
|
||||
Returns:
|
||||
SkillProperties with parsed metadata
|
||||
|
||||
Raises:
|
||||
ParseError: If SKILL.md is missing or has invalid YAML
|
||||
ValidationError: If required fields (name, description) are missing
|
||||
"""
|
||||
skill_dir = Path(skill_dir)
|
||||
skill_md = find_skill_md(skill_dir)
|
||||
|
||||
if skill_md is None:
|
||||
raise ParseError(f"SKILL.md not found in {skill_dir}")
|
||||
|
||||
content = skill_md.read_text()
|
||||
metadata, _ = parse_frontmatter(content)
|
||||
|
||||
if "name" not in metadata:
|
||||
raise ValidationError("Missing required field in frontmatter: name")
|
||||
if "description" not in metadata:
|
||||
raise ValidationError("Missing required field in frontmatter: description")
|
||||
|
||||
name = metadata["name"]
|
||||
description = metadata["description"]
|
||||
|
||||
if not isinstance(name, str) or not name.strip():
|
||||
raise ValidationError("Field 'name' must be a non-empty string")
|
||||
if not isinstance(description, str) or not description.strip():
|
||||
raise ValidationError("Field 'description' must be a non-empty string")
|
||||
|
||||
return SkillProperties(
|
||||
name=name.strip(),
|
||||
description=description.strip(),
|
||||
license=metadata.get("license"),
|
||||
compatibility=metadata.get("compatibility"),
|
||||
allowed_tools=metadata.get("allowed-tools"),
|
||||
metadata=metadata.get("metadata") or {},
|
||||
path=skill_dir,
|
||||
)
|
||||
@@ -0,0 +1,74 @@
|
||||
"""XML prompt generation for Agent Skills."""
|
||||
|
||||
from pathlib import Path
|
||||
from typing import List
|
||||
import html
|
||||
|
||||
from .models import SkillMetadata
|
||||
from .parser import read_properties
|
||||
|
||||
|
||||
def format_skill_for_prompt(skill: SkillMetadata) -> str:
|
||||
"""Format a single skill as XML for system prompt.
|
||||
|
||||
Args:
|
||||
skill: SkillMetadata instance
|
||||
|
||||
Returns:
|
||||
XML string for the skill
|
||||
"""
|
||||
name = html.escape(skill.name)
|
||||
description = html.escape(skill.description)
|
||||
location = html.escape(skill.location)
|
||||
|
||||
return f""" <skill>
|
||||
<name>{name}</name>
|
||||
<description>{description}</description>
|
||||
<location>{location}</location>
|
||||
</skill>"""
|
||||
|
||||
|
||||
def generate_skills_xml(skills: List[SkillMetadata]) -> str:
|
||||
"""Generate XML block for available skills.
|
||||
|
||||
Args:
|
||||
skills: List of SkillMetadata instances
|
||||
|
||||
Returns:
|
||||
XML string with <available_skills> block
|
||||
"""
|
||||
if not skills:
|
||||
return "<available_skills>\n</available_skills>"
|
||||
|
||||
skill_entries = "\n".join(format_skill_for_prompt(s) for s in skills)
|
||||
return f"<available_skills>\n{skill_entries}\n</available_skills>"
|
||||
|
||||
|
||||
def to_prompt(skill_dirs: List[Path]) -> str:
|
||||
"""Generate prompt XML from skill directories.
|
||||
|
||||
This is the main entry point for generating the <available_skills>
|
||||
block to inject into agent system prompts.
|
||||
|
||||
Args:
|
||||
skill_dirs: List of paths to skill directories
|
||||
|
||||
Returns:
|
||||
XML string with <available_skills> block
|
||||
"""
|
||||
skills = []
|
||||
|
||||
for skill_dir in skill_dirs:
|
||||
skill_dir = Path(skill_dir)
|
||||
if not skill_dir.exists() or not skill_dir.is_dir():
|
||||
continue
|
||||
|
||||
try:
|
||||
props = read_properties(skill_dir)
|
||||
meta = SkillMetadata.from_properties(props)
|
||||
skills.append(meta)
|
||||
except Exception:
|
||||
# Skip invalid skills silently
|
||||
continue
|
||||
|
||||
return generate_skills_xml(skills)
|
||||
@@ -0,0 +1,176 @@
|
||||
"""Skill validation logic."""
|
||||
|
||||
import unicodedata
|
||||
from pathlib import Path
|
||||
from typing import Optional, List
|
||||
|
||||
from .models import ParseError
|
||||
from .parser import find_skill_md, parse_frontmatter
|
||||
|
||||
MAX_SKILL_NAME_LENGTH = 64
|
||||
MAX_DESCRIPTION_LENGTH = 1024
|
||||
MAX_COMPATIBILITY_LENGTH = 500
|
||||
|
||||
ALLOWED_FIELDS = {
|
||||
"name",
|
||||
"description",
|
||||
"license",
|
||||
"allowed-tools",
|
||||
"metadata",
|
||||
"compatibility",
|
||||
}
|
||||
|
||||
|
||||
def _validate_name(name: str, skill_dir: Optional[Path]) -> List[str]:
|
||||
"""Validate skill name format and directory match.
|
||||
|
||||
Skill names support i18n characters (Unicode letters) plus hyphens.
|
||||
Names must be lowercase and cannot start/end with hyphens.
|
||||
"""
|
||||
errors = []
|
||||
|
||||
if not name or not isinstance(name, str) or not name.strip():
|
||||
errors.append("Field 'name' must be a non-empty string")
|
||||
return errors
|
||||
|
||||
name = unicodedata.normalize("NFKC", name.strip())
|
||||
|
||||
if len(name) > MAX_SKILL_NAME_LENGTH:
|
||||
errors.append(
|
||||
f"Skill name '{name}' exceeds {MAX_SKILL_NAME_LENGTH} character limit "
|
||||
f"({len(name)} chars)"
|
||||
)
|
||||
|
||||
if name != name.lower():
|
||||
errors.append(f"Skill name '{name}' must be lowercase")
|
||||
|
||||
if name.startswith("-") or name.endswith("-"):
|
||||
errors.append("Skill name cannot start or end with a hyphen")
|
||||
|
||||
if "--" in name:
|
||||
errors.append("Skill name cannot contain consecutive hyphens")
|
||||
|
||||
if not all(c.isalnum() or c == "-" for c in name):
|
||||
errors.append(
|
||||
f"Skill name '{name}' contains invalid characters. "
|
||||
"Only letters, digits, and hyphens are allowed."
|
||||
)
|
||||
|
||||
if skill_dir:
|
||||
dir_name = unicodedata.normalize("NFKC", skill_dir.name)
|
||||
if dir_name != name:
|
||||
errors.append(
|
||||
f"Directory name '{skill_dir.name}' must match skill name '{name}'"
|
||||
)
|
||||
|
||||
return errors
|
||||
|
||||
|
||||
def _validate_description(description: str) -> List[str]:
|
||||
"""Validate description format."""
|
||||
errors = []
|
||||
|
||||
if not description or not isinstance(description, str) or not description.strip():
|
||||
errors.append("Field 'description' must be a non-empty string")
|
||||
return errors
|
||||
|
||||
if len(description) > MAX_DESCRIPTION_LENGTH:
|
||||
errors.append(
|
||||
f"Description exceeds {MAX_DESCRIPTION_LENGTH} character limit "
|
||||
f"({len(description)} chars)"
|
||||
)
|
||||
|
||||
return errors
|
||||
|
||||
|
||||
def _validate_compatibility(compatibility: str) -> List[str]:
|
||||
"""Validate compatibility format."""
|
||||
errors = []
|
||||
|
||||
if not isinstance(compatibility, str):
|
||||
errors.append("Field 'compatibility' must be a string")
|
||||
return errors
|
||||
|
||||
if len(compatibility) > MAX_COMPATIBILITY_LENGTH:
|
||||
errors.append(
|
||||
f"Compatibility exceeds {MAX_COMPATIBILITY_LENGTH} character limit "
|
||||
f"({len(compatibility)} chars)"
|
||||
)
|
||||
|
||||
return errors
|
||||
|
||||
|
||||
def _validate_metadata_fields(metadata: dict) -> List[str]:
|
||||
"""Validate that only allowed fields are present."""
|
||||
errors = []
|
||||
|
||||
extra_fields = set(metadata.keys()) - ALLOWED_FIELDS
|
||||
if extra_fields:
|
||||
errors.append(
|
||||
f"Unexpected fields in frontmatter: {', '.join(sorted(extra_fields))}. "
|
||||
f"Only {sorted(ALLOWED_FIELDS)} are allowed."
|
||||
)
|
||||
|
||||
return errors
|
||||
|
||||
|
||||
def validate_metadata(metadata: dict, skill_dir: Optional[Path] = None) -> List[str]:
|
||||
"""Validate parsed skill metadata.
|
||||
|
||||
This is the core validation function that works on already-parsed metadata,
|
||||
avoiding duplicate file I/O when called from the parser.
|
||||
|
||||
Args:
|
||||
metadata: Parsed YAML frontmatter dictionary
|
||||
skill_dir: Optional path to skill directory (for name-directory match check)
|
||||
|
||||
Returns:
|
||||
List of validation error messages. Empty list means valid.
|
||||
"""
|
||||
errors = []
|
||||
errors.extend(_validate_metadata_fields(metadata))
|
||||
|
||||
if "name" not in metadata:
|
||||
errors.append("Missing required field in frontmatter: name")
|
||||
else:
|
||||
errors.extend(_validate_name(metadata["name"], skill_dir))
|
||||
|
||||
if "description" not in metadata:
|
||||
errors.append("Missing required field in frontmatter: description")
|
||||
else:
|
||||
errors.extend(_validate_description(metadata["description"]))
|
||||
|
||||
if "compatibility" in metadata:
|
||||
errors.extend(_validate_compatibility(metadata["compatibility"]))
|
||||
|
||||
return errors
|
||||
|
||||
|
||||
def validate(skill_dir: Path) -> List[str]:
|
||||
"""Validate a skill directory.
|
||||
|
||||
Args:
|
||||
skill_dir: Path to the skill directory
|
||||
|
||||
Returns:
|
||||
List of validation error messages. Empty list means valid.
|
||||
"""
|
||||
skill_dir = Path(skill_dir)
|
||||
|
||||
if not skill_dir.exists():
|
||||
return [f"Path does not exist: {skill_dir}"]
|
||||
|
||||
if not skill_dir.is_dir():
|
||||
return [f"Not a directory: {skill_dir}"]
|
||||
|
||||
skill_md = find_skill_md(skill_dir)
|
||||
if skill_md is None:
|
||||
return ["Missing required file: SKILL.md"]
|
||||
|
||||
try:
|
||||
content = skill_md.read_text()
|
||||
metadata, _ = parse_frontmatter(content)
|
||||
except ParseError as e:
|
||||
return [str(e)]
|
||||
|
||||
return validate_metadata(metadata, skill_dir)
|
||||
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
||||
|
||||
[project]
|
||||
name = "praisonaiagents"
|
||||
version = "0.1.17"
|
||||
version = "0.1.24"
|
||||
description = "Praison AI agents for completing complex tasks with Self Reflection Agents"
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.10"
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
"""Unit tests for PraisonAI Agents evaluation framework."""
|
||||
@@ -0,0 +1,146 @@
|
||||
"""Unit tests for BaseEvaluator class."""
|
||||
|
||||
import pytest
|
||||
import tempfile
|
||||
import json
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from praisonaiagents.eval.base import BaseEvaluator
|
||||
|
||||
|
||||
class ConcreteEvaluator(BaseEvaluator):
|
||||
"""Concrete implementation for testing."""
|
||||
|
||||
def run(self, **kwargs):
|
||||
return {"test": "result"}
|
||||
|
||||
|
||||
class TestBaseEvaluator:
|
||||
"""Tests for BaseEvaluator abstract class."""
|
||||
|
||||
def test_init_default_values(self):
|
||||
"""Test default initialization values."""
|
||||
evaluator = ConcreteEvaluator()
|
||||
assert evaluator.name.startswith("eval_")
|
||||
assert evaluator.eval_id is not None
|
||||
assert len(evaluator.eval_id) == 8
|
||||
assert evaluator.save_results_path is None
|
||||
assert evaluator.verbose is False
|
||||
|
||||
def test_init_custom_values(self):
|
||||
"""Test initialization with custom values."""
|
||||
evaluator = ConcreteEvaluator(
|
||||
name="my_eval",
|
||||
save_results_path="/tmp/results.json",
|
||||
verbose=True
|
||||
)
|
||||
assert evaluator.name == "my_eval"
|
||||
assert evaluator.save_results_path == "/tmp/results.json"
|
||||
assert evaluator.verbose is True
|
||||
|
||||
def test_eval_id_unique(self):
|
||||
"""Test that each evaluator gets a unique ID."""
|
||||
eval1 = ConcreteEvaluator()
|
||||
eval2 = ConcreteEvaluator()
|
||||
assert eval1.eval_id != eval2.eval_id
|
||||
|
||||
def test_before_run_verbose(self):
|
||||
"""Test before_run hook with verbose logging."""
|
||||
evaluator = ConcreteEvaluator(verbose=True)
|
||||
with patch('praisonaiagents.eval.base.logger') as mock_logger:
|
||||
evaluator.before_run()
|
||||
mock_logger.info.assert_called()
|
||||
|
||||
def test_after_run_verbose(self):
|
||||
"""Test after_run hook with verbose logging."""
|
||||
evaluator = ConcreteEvaluator(verbose=True)
|
||||
with patch('praisonaiagents.eval.base.logger') as mock_logger:
|
||||
evaluator.after_run({"result": "test"})
|
||||
mock_logger.info.assert_called()
|
||||
|
||||
def test_save_result_to_file(self):
|
||||
"""Test saving results to file."""
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
path = f"{tmpdir}/results.json"
|
||||
evaluator = ConcreteEvaluator(
|
||||
name="test_save",
|
||||
save_results_path=path
|
||||
)
|
||||
|
||||
result = MagicMock()
|
||||
result.to_dict.return_value = {"score": 8.0}
|
||||
|
||||
evaluator._save_result(result)
|
||||
|
||||
assert Path(path).exists()
|
||||
with open(path) as f:
|
||||
data = json.load(f)
|
||||
assert data["score"] == 8.0
|
||||
|
||||
def test_save_result_with_placeholders(self):
|
||||
"""Test saving results with path placeholders."""
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
path = f"{tmpdir}/{{name}}_{{eval_id}}.json"
|
||||
evaluator = ConcreteEvaluator(
|
||||
name="test_eval",
|
||||
save_results_path=path
|
||||
)
|
||||
|
||||
result = MagicMock()
|
||||
result.to_dict.return_value = {"score": 8.0}
|
||||
|
||||
evaluator._save_result(result)
|
||||
|
||||
expected_path = f"{tmpdir}/test_eval_{evaluator.eval_id}.json"
|
||||
assert Path(expected_path).exists()
|
||||
|
||||
def test_save_result_creates_directories(self):
|
||||
"""Test that save creates parent directories."""
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
path = f"{tmpdir}/nested/dir/results.json"
|
||||
evaluator = ConcreteEvaluator(save_results_path=path)
|
||||
|
||||
result = MagicMock()
|
||||
result.to_dict.return_value = {"score": 8.0}
|
||||
|
||||
evaluator._save_result(result)
|
||||
|
||||
assert Path(path).exists()
|
||||
|
||||
def test_run_abstract_method(self):
|
||||
"""Test that run method works in concrete implementation."""
|
||||
evaluator = ConcreteEvaluator()
|
||||
result = evaluator.run()
|
||||
assert result == {"test": "result"}
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_run_async_default(self):
|
||||
"""Test default async run calls sync run."""
|
||||
evaluator = ConcreteEvaluator()
|
||||
result = await evaluator.run_async()
|
||||
assert result == {"test": "result"}
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_before_run(self):
|
||||
"""Test async before_run hook."""
|
||||
evaluator = ConcreteEvaluator(verbose=True)
|
||||
with patch('praisonaiagents.eval.base.logger') as mock_logger:
|
||||
await evaluator.async_before_run()
|
||||
mock_logger.info.assert_called()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_after_run(self):
|
||||
"""Test async after_run hook."""
|
||||
evaluator = ConcreteEvaluator(verbose=True)
|
||||
with patch('praisonaiagents.eval.base.logger') as mock_logger:
|
||||
await evaluator.async_after_run({"result": "test"})
|
||||
mock_logger.info.assert_called()
|
||||
|
||||
def test_repr(self):
|
||||
"""Test string representation."""
|
||||
evaluator = ConcreteEvaluator(name="test_eval")
|
||||
repr_str = repr(evaluator)
|
||||
assert "ConcreteEvaluator" in repr_str
|
||||
assert "test_eval" in repr_str
|
||||
assert evaluator.eval_id in repr_str
|
||||
@@ -0,0 +1,425 @@
|
||||
"""Unit tests for evaluator classes."""
|
||||
|
||||
import pytest
|
||||
import time
|
||||
from unittest.mock import MagicMock, patch, AsyncMock
|
||||
|
||||
from praisonaiagents.eval.accuracy import AccuracyEvaluator
|
||||
from praisonaiagents.eval.performance import PerformanceEvaluator
|
||||
from praisonaiagents.eval.reliability import ReliabilityEvaluator
|
||||
from praisonaiagents.eval.criteria import CriteriaEvaluator
|
||||
from praisonaiagents.eval.results import (
|
||||
AccuracyResult,
|
||||
PerformanceResult,
|
||||
ReliabilityResult,
|
||||
CriteriaResult,
|
||||
)
|
||||
|
||||
|
||||
class TestAccuracyEvaluator:
|
||||
"""Tests for AccuracyEvaluator class."""
|
||||
|
||||
def test_init_with_func(self):
|
||||
"""Test initialization with function."""
|
||||
func = lambda x: "test output"
|
||||
evaluator = AccuracyEvaluator(
|
||||
func=func,
|
||||
input_text="test input",
|
||||
expected_output="test output"
|
||||
)
|
||||
assert evaluator.func == func
|
||||
assert evaluator.input_text == "test input"
|
||||
assert evaluator.expected_output == "test output"
|
||||
|
||||
def test_init_with_agent(self):
|
||||
"""Test initialization with agent."""
|
||||
agent = MagicMock()
|
||||
evaluator = AccuracyEvaluator(
|
||||
agent=agent,
|
||||
input_text="test",
|
||||
expected_output="expected"
|
||||
)
|
||||
assert evaluator.agent == agent
|
||||
|
||||
def test_init_requires_agent_or_func(self):
|
||||
"""Test that either agent or func must be provided."""
|
||||
with pytest.raises(ValueError, match="Either 'agent' or 'func' must be provided"):
|
||||
AccuracyEvaluator(
|
||||
input_text="test",
|
||||
expected_output="expected"
|
||||
)
|
||||
|
||||
def test_get_output_from_func(self):
|
||||
"""Test getting output from function."""
|
||||
func = lambda x: f"output for {x}"
|
||||
evaluator = AccuracyEvaluator(
|
||||
func=func,
|
||||
input_text="test",
|
||||
expected_output="expected"
|
||||
)
|
||||
output = evaluator._get_output()
|
||||
assert output == "output for test"
|
||||
|
||||
def test_get_output_from_agent_chat(self):
|
||||
"""Test getting output from agent with chat method."""
|
||||
agent = MagicMock()
|
||||
agent.chat.return_value = "agent response"
|
||||
|
||||
evaluator = AccuracyEvaluator(
|
||||
agent=agent,
|
||||
input_text="test",
|
||||
expected_output="expected"
|
||||
)
|
||||
output = evaluator._get_output()
|
||||
assert output == "agent response"
|
||||
agent.chat.assert_called_once_with("test")
|
||||
|
||||
def test_get_output_from_agent_start(self):
|
||||
"""Test getting output from agent with start method."""
|
||||
agent = MagicMock(spec=['start'])
|
||||
result = MagicMock()
|
||||
result.raw = "agent result"
|
||||
agent.start.return_value = result
|
||||
|
||||
evaluator = AccuracyEvaluator(
|
||||
agent=agent,
|
||||
input_text="test",
|
||||
expected_output="expected"
|
||||
)
|
||||
output = evaluator._get_output()
|
||||
assert output == "agent result"
|
||||
|
||||
def test_judge_output(self):
|
||||
"""Test LLM judging of output."""
|
||||
with patch('openai.OpenAI') as mock_openai_class:
|
||||
mock_client = MagicMock()
|
||||
mock_openai_class.return_value = mock_client
|
||||
|
||||
mock_response = MagicMock()
|
||||
mock_response.choices = [MagicMock()]
|
||||
mock_response.choices[0].message.content = "SCORE: 8\nREASONING: Good match"
|
||||
mock_client.chat.completions.create.return_value = mock_response
|
||||
|
||||
def test_func(x):
|
||||
return "4"
|
||||
|
||||
evaluator = AccuracyEvaluator(
|
||||
func=test_func,
|
||||
input_text="What is 2+2?",
|
||||
expected_output="4"
|
||||
)
|
||||
|
||||
score = evaluator._judge_output("4")
|
||||
assert score.score == 8.0
|
||||
assert "Good match" in score.reasoning
|
||||
|
||||
def test_run_returns_accuracy_result(self):
|
||||
"""Test that run returns AccuracyResult."""
|
||||
with patch('openai.OpenAI') as mock_openai_class:
|
||||
mock_client = MagicMock()
|
||||
mock_openai_class.return_value = mock_client
|
||||
|
||||
mock_response = MagicMock()
|
||||
mock_response.choices = [MagicMock()]
|
||||
mock_response.choices[0].message.content = "SCORE: 9\nREASONING: Excellent"
|
||||
mock_client.chat.completions.create.return_value = mock_response
|
||||
|
||||
def test_func(x):
|
||||
return "4"
|
||||
|
||||
evaluator = AccuracyEvaluator(
|
||||
func=test_func,
|
||||
input_text="What is 2+2?",
|
||||
expected_output="4",
|
||||
num_iterations=2
|
||||
)
|
||||
|
||||
result = evaluator.run()
|
||||
assert isinstance(result, AccuracyResult)
|
||||
assert len(result.evaluations) == 2
|
||||
|
||||
def test_evaluate_output(self):
|
||||
"""Test evaluate_output method for pre-generated outputs."""
|
||||
with patch('openai.OpenAI') as mock_openai_class:
|
||||
mock_client = MagicMock()
|
||||
mock_openai_class.return_value = mock_client
|
||||
|
||||
mock_response = MagicMock()
|
||||
mock_response.choices = [MagicMock()]
|
||||
mock_response.choices[0].message.content = "SCORE: 10\nREASONING: Perfect"
|
||||
mock_client.chat.completions.create.return_value = mock_response
|
||||
|
||||
def test_func(x):
|
||||
return "unused"
|
||||
|
||||
evaluator = AccuracyEvaluator(
|
||||
func=test_func,
|
||||
input_text="What is 2+2?",
|
||||
expected_output="4"
|
||||
)
|
||||
|
||||
result = evaluator.evaluate_output("4")
|
||||
assert isinstance(result, AccuracyResult)
|
||||
assert len(result.evaluations) == 1
|
||||
assert result.evaluations[0].score == 10.0
|
||||
|
||||
|
||||
class TestPerformanceEvaluator:
|
||||
"""Tests for PerformanceEvaluator class."""
|
||||
|
||||
def test_init_with_func(self):
|
||||
"""Test initialization with function."""
|
||||
func = lambda: time.sleep(0.01)
|
||||
evaluator = PerformanceEvaluator(
|
||||
func=func,
|
||||
num_iterations=5,
|
||||
warmup_runs=2
|
||||
)
|
||||
assert evaluator.func == func
|
||||
assert evaluator.num_iterations == 5
|
||||
assert evaluator.warmup_runs == 2
|
||||
|
||||
def test_init_with_agent(self):
|
||||
"""Test initialization with agent."""
|
||||
agent = MagicMock()
|
||||
evaluator = PerformanceEvaluator(
|
||||
agent=agent,
|
||||
input_text="test",
|
||||
num_iterations=3
|
||||
)
|
||||
assert evaluator.agent == agent
|
||||
assert evaluator.input_text == "test"
|
||||
|
||||
def test_init_requires_func_or_agent(self):
|
||||
"""Test that either func or agent must be provided."""
|
||||
with pytest.raises(ValueError, match="Either 'func' or 'agent' must be provided"):
|
||||
PerformanceEvaluator()
|
||||
|
||||
def test_run_measures_time(self):
|
||||
"""Test that run measures execution time."""
|
||||
call_count = 0
|
||||
def slow_func():
|
||||
nonlocal call_count
|
||||
call_count += 1
|
||||
time.sleep(0.01)
|
||||
|
||||
evaluator = PerformanceEvaluator(
|
||||
func=slow_func,
|
||||
num_iterations=3,
|
||||
warmup_runs=1,
|
||||
track_memory=False
|
||||
)
|
||||
|
||||
result = evaluator.run()
|
||||
assert isinstance(result, PerformanceResult)
|
||||
assert len(result.metrics) == 3
|
||||
assert all(m.run_time >= 0.01 for m in result.metrics)
|
||||
assert call_count == 4 # 1 warmup + 3 iterations
|
||||
|
||||
def test_run_with_agent(self):
|
||||
"""Test run with agent."""
|
||||
agent = MagicMock()
|
||||
agent.chat.return_value = "response"
|
||||
|
||||
evaluator = PerformanceEvaluator(
|
||||
agent=agent,
|
||||
input_text="test",
|
||||
num_iterations=2,
|
||||
warmup_runs=1,
|
||||
track_memory=False
|
||||
)
|
||||
|
||||
result = evaluator.run()
|
||||
assert isinstance(result, PerformanceResult)
|
||||
assert len(result.metrics) == 2
|
||||
assert agent.chat.call_count == 3 # 1 warmup + 2 iterations
|
||||
|
||||
|
||||
class TestReliabilityEvaluator:
|
||||
"""Tests for ReliabilityEvaluator class."""
|
||||
|
||||
def test_init(self):
|
||||
"""Test initialization."""
|
||||
agent = MagicMock()
|
||||
evaluator = ReliabilityEvaluator(
|
||||
agent=agent,
|
||||
input_text="test",
|
||||
expected_tools=["search", "calculate"]
|
||||
)
|
||||
assert evaluator.agent == agent
|
||||
assert evaluator.expected_tools == {"search", "calculate"}
|
||||
|
||||
def test_init_requires_agent(self):
|
||||
"""Test that agent must be provided."""
|
||||
with pytest.raises(ValueError, match="'agent' must be provided"):
|
||||
ReliabilityEvaluator(
|
||||
input_text="test",
|
||||
expected_tools=["search"]
|
||||
)
|
||||
|
||||
def test_evaluate_tool_calls(self):
|
||||
"""Test evaluate_tool_calls method."""
|
||||
agent = MagicMock()
|
||||
evaluator = ReliabilityEvaluator(
|
||||
agent=agent,
|
||||
input_text="test",
|
||||
expected_tools=["search", "calculate"],
|
||||
forbidden_tools=["delete"]
|
||||
)
|
||||
|
||||
result = evaluator.evaluate_tool_calls(["search", "calculate"])
|
||||
assert isinstance(result, ReliabilityResult)
|
||||
assert result.passed is True
|
||||
assert len(result.passed_calls) == 3 # 2 expected + 1 forbidden not called
|
||||
|
||||
def test_evaluate_tool_calls_with_failures(self):
|
||||
"""Test evaluate_tool_calls with missing tools."""
|
||||
agent = MagicMock()
|
||||
evaluator = ReliabilityEvaluator(
|
||||
agent=agent,
|
||||
input_text="test",
|
||||
expected_tools=["search", "calculate"]
|
||||
)
|
||||
|
||||
result = evaluator.evaluate_tool_calls(["search"]) # missing calculate
|
||||
assert result.passed is False
|
||||
assert len(result.failed_calls) == 1
|
||||
|
||||
def test_forbidden_tools(self):
|
||||
"""Test forbidden tools detection."""
|
||||
agent = MagicMock()
|
||||
evaluator = ReliabilityEvaluator(
|
||||
agent=agent,
|
||||
input_text="test",
|
||||
expected_tools=[],
|
||||
forbidden_tools=["delete", "drop"]
|
||||
)
|
||||
|
||||
result = evaluator.evaluate_tool_calls(["delete"]) # called forbidden tool
|
||||
assert result.passed is False
|
||||
assert len(result.failed_calls) == 1
|
||||
|
||||
|
||||
class TestCriteriaEvaluator:
|
||||
"""Tests for CriteriaEvaluator class."""
|
||||
|
||||
def test_init(self):
|
||||
"""Test initialization."""
|
||||
evaluator = CriteriaEvaluator(
|
||||
criteria="Response is helpful",
|
||||
func=lambda x: "test",
|
||||
input_text="test"
|
||||
)
|
||||
assert evaluator.criteria == "Response is helpful"
|
||||
assert evaluator.scoring_type == "numeric"
|
||||
assert evaluator.threshold == 7.0
|
||||
|
||||
def test_init_binary_scoring(self):
|
||||
"""Test initialization with binary scoring."""
|
||||
evaluator = CriteriaEvaluator(
|
||||
criteria="Response is helpful",
|
||||
func=lambda x: "test",
|
||||
scoring_type="binary"
|
||||
)
|
||||
assert evaluator.scoring_type == "binary"
|
||||
|
||||
def test_judge_output_numeric(self):
|
||||
"""Test numeric scoring."""
|
||||
with patch('openai.OpenAI') as mock_openai_class:
|
||||
mock_client = MagicMock()
|
||||
mock_openai_class.return_value = mock_client
|
||||
|
||||
mock_response = MagicMock()
|
||||
mock_response.choices = [MagicMock()]
|
||||
mock_response.choices[0].message.content = "SCORE: 8\nREASONING: Good"
|
||||
mock_client.chat.completions.create.return_value = mock_response
|
||||
|
||||
def test_func(x):
|
||||
return "test"
|
||||
|
||||
evaluator = CriteriaEvaluator(
|
||||
criteria="Be helpful",
|
||||
func=test_func,
|
||||
scoring_type="numeric",
|
||||
threshold=7.0
|
||||
)
|
||||
|
||||
score = evaluator._judge_output("test output")
|
||||
assert score.score == 8.0
|
||||
assert score.passed is True
|
||||
|
||||
def test_judge_output_binary(self):
|
||||
"""Test binary scoring."""
|
||||
with patch('openai.OpenAI') as mock_openai_class:
|
||||
mock_client = MagicMock()
|
||||
mock_openai_class.return_value = mock_client
|
||||
|
||||
mock_response = MagicMock()
|
||||
mock_response.choices = [MagicMock()]
|
||||
mock_response.choices[0].message.content = "RESULT: PASS\nREASONING: Meets criteria"
|
||||
mock_client.chat.completions.create.return_value = mock_response
|
||||
|
||||
def test_func(x):
|
||||
return "test"
|
||||
|
||||
evaluator = CriteriaEvaluator(
|
||||
criteria="Be helpful",
|
||||
func=test_func,
|
||||
scoring_type="binary"
|
||||
)
|
||||
|
||||
score = evaluator._judge_output("test output")
|
||||
assert score.score == 10.0
|
||||
assert score.passed is True
|
||||
|
||||
def test_on_fail_callback(self):
|
||||
"""Test on_fail callback is called."""
|
||||
with patch('openai.OpenAI') as mock_openai_class:
|
||||
mock_client = MagicMock()
|
||||
mock_openai_class.return_value = mock_client
|
||||
|
||||
mock_response = MagicMock()
|
||||
mock_response.choices = [MagicMock()]
|
||||
mock_response.choices[0].message.content = "SCORE: 3\nREASONING: Poor"
|
||||
mock_client.chat.completions.create.return_value = mock_response
|
||||
|
||||
on_fail_called = []
|
||||
def on_fail(score):
|
||||
on_fail_called.append(score)
|
||||
|
||||
def test_func(x):
|
||||
return "test"
|
||||
|
||||
evaluator = CriteriaEvaluator(
|
||||
criteria="Be helpful",
|
||||
func=test_func,
|
||||
threshold=7.0,
|
||||
on_fail=on_fail
|
||||
)
|
||||
|
||||
evaluator.run()
|
||||
assert len(on_fail_called) == 1
|
||||
assert on_fail_called[0].score == 3.0
|
||||
|
||||
def test_evaluate_output_criteria(self):
|
||||
"""Test evaluate_output method."""
|
||||
with patch('openai.OpenAI') as mock_openai_class:
|
||||
mock_client = MagicMock()
|
||||
mock_openai_class.return_value = mock_client
|
||||
|
||||
mock_response = MagicMock()
|
||||
mock_response.choices = [MagicMock()]
|
||||
mock_response.choices[0].message.content = "SCORE: 9\nREASONING: Great"
|
||||
mock_client.chat.completions.create.return_value = mock_response
|
||||
|
||||
def test_func(x):
|
||||
return "unused"
|
||||
|
||||
evaluator = CriteriaEvaluator(
|
||||
criteria="Be helpful",
|
||||
func=test_func
|
||||
)
|
||||
|
||||
result = evaluator.evaluate_output("pre-generated output")
|
||||
assert isinstance(result, CriteriaResult)
|
||||
assert len(result.evaluations) == 1
|
||||
@@ -0,0 +1,359 @@
|
||||
"""Unit tests for evaluation result dataclasses."""
|
||||
|
||||
import pytest
|
||||
import json
|
||||
from datetime import datetime
|
||||
|
||||
from praisonaiagents.eval.results import (
|
||||
EvaluationScore,
|
||||
AccuracyResult,
|
||||
PerformanceMetrics,
|
||||
PerformanceResult,
|
||||
ToolCallResult,
|
||||
ReliabilityResult,
|
||||
CriteriaScore,
|
||||
CriteriaResult,
|
||||
)
|
||||
|
||||
|
||||
class TestEvaluationScore:
|
||||
"""Tests for EvaluationScore dataclass."""
|
||||
|
||||
def test_creation(self):
|
||||
"""Test basic creation of EvaluationScore."""
|
||||
score = EvaluationScore(
|
||||
score=8.5,
|
||||
reasoning="Good response",
|
||||
input_text="What is 2+2?",
|
||||
output_text="4",
|
||||
expected_output="4"
|
||||
)
|
||||
assert score.score == 8.5
|
||||
assert score.reasoning == "Good response"
|
||||
assert score.input_text == "What is 2+2?"
|
||||
assert score.output_text == "4"
|
||||
assert score.expected_output == "4"
|
||||
|
||||
def test_timestamp_auto_generated(self):
|
||||
"""Test that timestamp is auto-generated."""
|
||||
score = EvaluationScore(
|
||||
score=8.0,
|
||||
reasoning="Test",
|
||||
input_text="test",
|
||||
output_text="test"
|
||||
)
|
||||
assert score.timestamp is not None
|
||||
datetime.fromisoformat(score.timestamp)
|
||||
|
||||
def test_metadata_default(self):
|
||||
"""Test that metadata defaults to empty dict."""
|
||||
score = EvaluationScore(
|
||||
score=8.0,
|
||||
reasoning="Test",
|
||||
input_text="test",
|
||||
output_text="test"
|
||||
)
|
||||
assert score.metadata == {}
|
||||
|
||||
|
||||
class TestAccuracyResult:
|
||||
"""Tests for AccuracyResult dataclass."""
|
||||
|
||||
def test_empty_result(self):
|
||||
"""Test empty AccuracyResult."""
|
||||
result = AccuracyResult()
|
||||
assert result.evaluations == []
|
||||
assert result.avg_score == 0.0
|
||||
assert result.min_score == 0.0
|
||||
assert result.max_score == 0.0
|
||||
assert result.std_dev == 0.0
|
||||
assert result.passed is False
|
||||
|
||||
def test_single_evaluation(self):
|
||||
"""Test AccuracyResult with single evaluation."""
|
||||
result = AccuracyResult()
|
||||
result.evaluations.append(EvaluationScore(
|
||||
score=8.0,
|
||||
reasoning="Good",
|
||||
input_text="test",
|
||||
output_text="test"
|
||||
))
|
||||
assert result.avg_score == 8.0
|
||||
assert result.min_score == 8.0
|
||||
assert result.max_score == 8.0
|
||||
assert result.std_dev == 0.0
|
||||
assert result.passed is True
|
||||
|
||||
def test_multiple_evaluations(self):
|
||||
"""Test AccuracyResult with multiple evaluations."""
|
||||
result = AccuracyResult()
|
||||
for score in [6.0, 8.0, 10.0]:
|
||||
result.evaluations.append(EvaluationScore(
|
||||
score=score,
|
||||
reasoning="Test",
|
||||
input_text="test",
|
||||
output_text="test"
|
||||
))
|
||||
assert result.avg_score == 8.0
|
||||
assert result.min_score == 6.0
|
||||
assert result.max_score == 10.0
|
||||
assert result.passed is True
|
||||
|
||||
def test_passed_threshold(self):
|
||||
"""Test passed property with different scores."""
|
||||
result = AccuracyResult()
|
||||
result.evaluations.append(EvaluationScore(
|
||||
score=6.0,
|
||||
reasoning="Below threshold",
|
||||
input_text="test",
|
||||
output_text="test"
|
||||
))
|
||||
assert result.passed is False
|
||||
|
||||
result.evaluations.append(EvaluationScore(
|
||||
score=8.0,
|
||||
reasoning="Above threshold",
|
||||
input_text="test",
|
||||
output_text="test"
|
||||
))
|
||||
assert result.passed is True
|
||||
|
||||
def test_to_dict(self):
|
||||
"""Test to_dict method."""
|
||||
result = AccuracyResult(eval_id="test123", name="test_eval")
|
||||
result.evaluations.append(EvaluationScore(
|
||||
score=8.0,
|
||||
reasoning="Good",
|
||||
input_text="test",
|
||||
output_text="test"
|
||||
))
|
||||
d = result.to_dict()
|
||||
assert d["eval_id"] == "test123"
|
||||
assert d["name"] == "test_eval"
|
||||
assert d["avg_score"] == 8.0
|
||||
assert d["num_evaluations"] == 1
|
||||
|
||||
def test_to_json(self):
|
||||
"""Test to_json method."""
|
||||
result = AccuracyResult(eval_id="test123")
|
||||
result.evaluations.append(EvaluationScore(
|
||||
score=8.0,
|
||||
reasoning="Good",
|
||||
input_text="test",
|
||||
output_text="test"
|
||||
))
|
||||
json_str = result.to_json()
|
||||
data = json.loads(json_str)
|
||||
assert data["eval_id"] == "test123"
|
||||
|
||||
|
||||
class TestPerformanceMetrics:
|
||||
"""Tests for PerformanceMetrics dataclass."""
|
||||
|
||||
def test_creation(self):
|
||||
"""Test basic creation."""
|
||||
metrics = PerformanceMetrics(
|
||||
run_time=0.5,
|
||||
memory_usage=10.5,
|
||||
iteration=1
|
||||
)
|
||||
assert metrics.run_time == 0.5
|
||||
assert metrics.memory_usage == 10.5
|
||||
assert metrics.iteration == 1
|
||||
|
||||
|
||||
class TestPerformanceResult:
|
||||
"""Tests for PerformanceResult dataclass."""
|
||||
|
||||
def test_empty_result(self):
|
||||
"""Test empty PerformanceResult."""
|
||||
result = PerformanceResult()
|
||||
assert result.metrics == []
|
||||
assert result.avg_run_time == 0.0
|
||||
assert result.min_run_time == 0.0
|
||||
assert result.max_run_time == 0.0
|
||||
assert result.avg_memory == 0.0
|
||||
|
||||
def test_with_metrics(self):
|
||||
"""Test PerformanceResult with metrics."""
|
||||
result = PerformanceResult()
|
||||
for i, (time, mem) in enumerate([(0.1, 5.0), (0.2, 10.0), (0.3, 15.0)]):
|
||||
result.metrics.append(PerformanceMetrics(
|
||||
run_time=time,
|
||||
memory_usage=mem,
|
||||
iteration=i + 1
|
||||
))
|
||||
assert result.avg_run_time == 0.2
|
||||
assert result.min_run_time == 0.1
|
||||
assert result.max_run_time == 0.3
|
||||
assert result.avg_memory == 10.0
|
||||
assert result.max_memory == 15.0
|
||||
|
||||
def test_median_and_p95(self):
|
||||
"""Test median and p95 calculations."""
|
||||
result = PerformanceResult()
|
||||
for i in range(100):
|
||||
result.metrics.append(PerformanceMetrics(
|
||||
run_time=float(i + 1) / 100,
|
||||
memory_usage=1.0,
|
||||
iteration=i + 1
|
||||
))
|
||||
assert 0.49 <= result.median_run_time <= 0.51
|
||||
assert result.p95_run_time >= 0.95
|
||||
|
||||
def test_to_dict(self):
|
||||
"""Test to_dict method."""
|
||||
result = PerformanceResult(warmup_runs=5, eval_id="perf123")
|
||||
result.metrics.append(PerformanceMetrics(
|
||||
run_time=0.5,
|
||||
memory_usage=10.0,
|
||||
iteration=1
|
||||
))
|
||||
d = result.to_dict()
|
||||
assert d["warmup_runs"] == 5
|
||||
assert d["eval_id"] == "perf123"
|
||||
assert d["num_iterations"] == 1
|
||||
|
||||
|
||||
class TestToolCallResult:
|
||||
"""Tests for ToolCallResult dataclass."""
|
||||
|
||||
def test_creation(self):
|
||||
"""Test basic creation."""
|
||||
tc = ToolCallResult(
|
||||
tool_name="search",
|
||||
expected=True,
|
||||
actual=True,
|
||||
passed=True
|
||||
)
|
||||
assert tc.tool_name == "search"
|
||||
assert tc.expected is True
|
||||
assert tc.actual is True
|
||||
assert tc.passed is True
|
||||
|
||||
|
||||
class TestReliabilityResult:
|
||||
"""Tests for ReliabilityResult dataclass."""
|
||||
|
||||
def test_empty_result(self):
|
||||
"""Test empty ReliabilityResult."""
|
||||
result = ReliabilityResult()
|
||||
assert result.tool_results == []
|
||||
assert result.pass_rate == 0.0
|
||||
assert result.status == "PASSED"
|
||||
assert result.passed is True
|
||||
|
||||
def test_with_tool_calls(self):
|
||||
"""Test ReliabilityResult with tool calls."""
|
||||
result = ReliabilityResult()
|
||||
result.tool_results.append(ToolCallResult(
|
||||
tool_name="search",
|
||||
expected=True,
|
||||
actual=True,
|
||||
passed=True
|
||||
))
|
||||
result.tool_results.append(ToolCallResult(
|
||||
tool_name="calculate",
|
||||
expected=True,
|
||||
actual=False,
|
||||
passed=False
|
||||
))
|
||||
assert len(result.passed_calls) == 1
|
||||
assert len(result.failed_calls) == 1
|
||||
assert result.pass_rate == 0.5
|
||||
assert result.status == "FAILED"
|
||||
assert result.passed is False
|
||||
|
||||
def test_all_passed(self):
|
||||
"""Test when all tool calls pass."""
|
||||
result = ReliabilityResult()
|
||||
for name in ["tool1", "tool2", "tool3"]:
|
||||
result.tool_results.append(ToolCallResult(
|
||||
tool_name=name,
|
||||
expected=True,
|
||||
actual=True,
|
||||
passed=True
|
||||
))
|
||||
assert result.pass_rate == 1.0
|
||||
assert result.passed is True
|
||||
|
||||
|
||||
class TestCriteriaScore:
|
||||
"""Tests for CriteriaScore dataclass."""
|
||||
|
||||
def test_creation(self):
|
||||
"""Test basic creation."""
|
||||
score = CriteriaScore(
|
||||
score=8.0,
|
||||
passed=True,
|
||||
reasoning="Meets criteria",
|
||||
output_text="test output",
|
||||
criteria="Be helpful"
|
||||
)
|
||||
assert score.score == 8.0
|
||||
assert score.passed is True
|
||||
assert score.reasoning == "Meets criteria"
|
||||
|
||||
|
||||
class TestCriteriaResult:
|
||||
"""Tests for CriteriaResult dataclass."""
|
||||
|
||||
def test_empty_result(self):
|
||||
"""Test empty CriteriaResult."""
|
||||
result = CriteriaResult(criteria="Be helpful")
|
||||
assert result.evaluations == []
|
||||
assert result.avg_score == 0.0
|
||||
assert result.pass_rate == 0.0
|
||||
|
||||
def test_numeric_scoring(self):
|
||||
"""Test numeric scoring mode."""
|
||||
result = CriteriaResult(
|
||||
criteria="Be helpful",
|
||||
scoring_type="numeric",
|
||||
threshold=7.0
|
||||
)
|
||||
result.evaluations.append(CriteriaScore(
|
||||
score=8.0,
|
||||
passed=True,
|
||||
reasoning="Good",
|
||||
output_text="test",
|
||||
criteria="Be helpful"
|
||||
))
|
||||
assert result.avg_score == 8.0
|
||||
assert result.passed is True
|
||||
|
||||
def test_binary_scoring(self):
|
||||
"""Test binary scoring mode."""
|
||||
result = CriteriaResult(
|
||||
criteria="Be helpful",
|
||||
scoring_type="binary"
|
||||
)
|
||||
result.evaluations.append(CriteriaScore(
|
||||
score=10.0,
|
||||
passed=True,
|
||||
reasoning="Pass",
|
||||
output_text="test",
|
||||
criteria="Be helpful"
|
||||
))
|
||||
result.evaluations.append(CriteriaScore(
|
||||
score=0.0,
|
||||
passed=False,
|
||||
reasoning="Fail",
|
||||
output_text="test",
|
||||
criteria="Be helpful"
|
||||
))
|
||||
assert result.pass_rate == 0.5
|
||||
assert result.passed is True
|
||||
|
||||
def test_to_dict(self):
|
||||
"""Test to_dict method."""
|
||||
result = CriteriaResult(
|
||||
criteria="Be helpful",
|
||||
scoring_type="numeric",
|
||||
threshold=7.0,
|
||||
eval_id="crit123"
|
||||
)
|
||||
d = result.to_dict()
|
||||
assert d["criteria"] == "Be helpful"
|
||||
assert d["scoring_type"] == "numeric"
|
||||
assert d["threshold"] == 7.0
|
||||
@@ -0,0 +1,222 @@
|
||||
"""Unit tests for evaluation utility functions."""
|
||||
|
||||
import json
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from praisonaiagents.eval.utils import (
|
||||
save_result_to_file,
|
||||
load_result_from_file,
|
||||
format_score,
|
||||
format_percentage,
|
||||
format_duration,
|
||||
format_memory,
|
||||
calculate_pass_rate,
|
||||
)
|
||||
|
||||
|
||||
class TestSaveResultToFile:
|
||||
"""Tests for save_result_to_file function."""
|
||||
|
||||
def test_save_with_to_dict(self):
|
||||
"""Test saving result with to_dict method."""
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
path = f"{tmpdir}/result.json"
|
||||
result = MagicMock()
|
||||
result.to_dict.return_value = {"score": 8.0, "passed": True}
|
||||
|
||||
success = save_result_to_file(result, path)
|
||||
|
||||
assert success is True
|
||||
assert Path(path).exists()
|
||||
with open(path) as f:
|
||||
data = json.load(f)
|
||||
assert data["score"] == 8.0
|
||||
|
||||
def test_save_with_dict_attr(self):
|
||||
"""Test saving result with __dict__ attribute."""
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
path = f"{tmpdir}/result.json"
|
||||
|
||||
class SimpleResult:
|
||||
def __init__(self):
|
||||
self.score = 9.0
|
||||
self.name = "test"
|
||||
|
||||
result = SimpleResult()
|
||||
success = save_result_to_file(result, path)
|
||||
|
||||
assert success is True
|
||||
with open(path) as f:
|
||||
data = json.load(f)
|
||||
assert data["score"] == 9.0
|
||||
|
||||
def test_save_with_placeholders(self):
|
||||
"""Test saving with path placeholders."""
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
path = f"{tmpdir}/{{name}}_{{eval_id}}.json"
|
||||
result = MagicMock()
|
||||
result.to_dict.return_value = {"score": 8.0}
|
||||
|
||||
success = save_result_to_file(
|
||||
result, path,
|
||||
eval_id="abc123",
|
||||
name="test_eval"
|
||||
)
|
||||
|
||||
assert success is True
|
||||
expected_path = f"{tmpdir}/test_eval_abc123.json"
|
||||
assert Path(expected_path).exists()
|
||||
|
||||
def test_save_creates_directories(self):
|
||||
"""Test that save creates parent directories."""
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
path = f"{tmpdir}/nested/dir/result.json"
|
||||
result = MagicMock()
|
||||
result.to_dict.return_value = {"score": 8.0}
|
||||
|
||||
success = save_result_to_file(result, path)
|
||||
|
||||
assert success is True
|
||||
assert Path(path).exists()
|
||||
|
||||
def test_save_failure_returns_false(self):
|
||||
"""Test that save failure returns False."""
|
||||
result = MagicMock()
|
||||
result.to_dict.side_effect = Exception("Test error")
|
||||
|
||||
success = save_result_to_file(result, "/invalid/path/result.json")
|
||||
|
||||
assert success is False
|
||||
|
||||
|
||||
class TestLoadResultFromFile:
|
||||
"""Tests for load_result_from_file function."""
|
||||
|
||||
def test_load_existing_file(self):
|
||||
"""Test loading existing result file."""
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
path = f"{tmpdir}/result.json"
|
||||
data = {"score": 8.0, "passed": True}
|
||||
with open(path, 'w') as f:
|
||||
json.dump(data, f)
|
||||
|
||||
loaded = load_result_from_file(path)
|
||||
|
||||
assert loaded == data
|
||||
|
||||
def test_load_nonexistent_file(self):
|
||||
"""Test loading nonexistent file returns None."""
|
||||
result = load_result_from_file("/nonexistent/path/result.json")
|
||||
assert result is None
|
||||
|
||||
def test_load_invalid_json(self):
|
||||
"""Test loading invalid JSON returns None."""
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
path = f"{tmpdir}/invalid.json"
|
||||
with open(path, 'w') as f:
|
||||
f.write("not valid json")
|
||||
|
||||
result = load_result_from_file(path)
|
||||
assert result is None
|
||||
|
||||
|
||||
class TestFormatScore:
|
||||
"""Tests for format_score function."""
|
||||
|
||||
def test_format_default_max(self):
|
||||
"""Test formatting with default max score."""
|
||||
assert format_score(8.5) == "8.5/10"
|
||||
|
||||
def test_format_custom_max(self):
|
||||
"""Test formatting with custom max score."""
|
||||
assert format_score(85.0, 100.0) == "85.0/100"
|
||||
|
||||
def test_format_zero(self):
|
||||
"""Test formatting zero score."""
|
||||
assert format_score(0.0) == "0.0/10"
|
||||
|
||||
|
||||
class TestFormatPercentage:
|
||||
"""Tests for format_percentage function."""
|
||||
|
||||
def test_format_full(self):
|
||||
"""Test formatting 100%."""
|
||||
assert format_percentage(1.0) == "100.0%"
|
||||
|
||||
def test_format_half(self):
|
||||
"""Test formatting 50%."""
|
||||
assert format_percentage(0.5) == "50.0%"
|
||||
|
||||
def test_format_zero(self):
|
||||
"""Test formatting 0%."""
|
||||
assert format_percentage(0.0) == "0.0%"
|
||||
|
||||
def test_format_decimal(self):
|
||||
"""Test formatting decimal percentage."""
|
||||
assert format_percentage(0.333) == "33.3%"
|
||||
|
||||
|
||||
class TestFormatDuration:
|
||||
"""Tests for format_duration function."""
|
||||
|
||||
def test_format_microseconds(self):
|
||||
"""Test formatting microseconds."""
|
||||
result = format_duration(0.0001)
|
||||
assert "μs" in result
|
||||
|
||||
def test_format_milliseconds(self):
|
||||
"""Test formatting milliseconds."""
|
||||
result = format_duration(0.5)
|
||||
assert "ms" in result
|
||||
|
||||
def test_format_seconds(self):
|
||||
"""Test formatting seconds."""
|
||||
result = format_duration(5.5)
|
||||
assert "s" in result
|
||||
assert "m" not in result
|
||||
|
||||
def test_format_minutes(self):
|
||||
"""Test formatting minutes."""
|
||||
result = format_duration(90.0)
|
||||
assert "m" in result
|
||||
|
||||
|
||||
class TestFormatMemory:
|
||||
"""Tests for format_memory function."""
|
||||
|
||||
def test_format_kilobytes(self):
|
||||
"""Test formatting kilobytes."""
|
||||
result = format_memory(0.5)
|
||||
assert "KB" in result
|
||||
|
||||
def test_format_megabytes(self):
|
||||
"""Test formatting megabytes."""
|
||||
result = format_memory(100.0)
|
||||
assert "MB" in result
|
||||
|
||||
def test_format_gigabytes(self):
|
||||
"""Test formatting gigabytes."""
|
||||
result = format_memory(2048.0)
|
||||
assert "GB" in result
|
||||
|
||||
|
||||
class TestCalculatePassRate:
|
||||
"""Tests for calculate_pass_rate function."""
|
||||
|
||||
def test_all_passed(self):
|
||||
"""Test 100% pass rate."""
|
||||
assert calculate_pass_rate(10, 10) == 1.0
|
||||
|
||||
def test_none_passed(self):
|
||||
"""Test 0% pass rate."""
|
||||
assert calculate_pass_rate(0, 10) == 0.0
|
||||
|
||||
def test_half_passed(self):
|
||||
"""Test 50% pass rate."""
|
||||
assert calculate_pass_rate(5, 10) == 0.5
|
||||
|
||||
def test_zero_total(self):
|
||||
"""Test with zero total."""
|
||||
assert calculate_pass_rate(0, 0) == 0.0
|
||||
@@ -0,0 +1 @@
|
||||
"""Unit tests for Agent Skills module."""
|
||||
@@ -0,0 +1,180 @@
|
||||
"""Tests for Agent Skills discovery."""
|
||||
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
|
||||
|
||||
class TestGetDefaultSkillDirs:
|
||||
"""Tests for get_default_skill_dirs function."""
|
||||
|
||||
def test_returns_list(self):
|
||||
"""Test that function returns a list."""
|
||||
from praisonaiagents.skills.discovery import get_default_skill_dirs
|
||||
|
||||
result = get_default_skill_dirs()
|
||||
|
||||
assert isinstance(result, list)
|
||||
|
||||
def test_returns_path_objects(self):
|
||||
"""Test that returned items are Path objects."""
|
||||
from praisonaiagents.skills.discovery import get_default_skill_dirs
|
||||
|
||||
result = get_default_skill_dirs()
|
||||
|
||||
for item in result:
|
||||
assert isinstance(item, Path)
|
||||
|
||||
|
||||
class TestDiscoverSkills:
|
||||
"""Tests for discover_skills function."""
|
||||
|
||||
def test_discover_empty_directory(self):
|
||||
"""Test discovering skills in empty directory."""
|
||||
from praisonaiagents.skills.discovery import discover_skills
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
result = discover_skills([tmpdir], include_defaults=False)
|
||||
|
||||
assert result == []
|
||||
|
||||
def test_discover_single_skill(self):
|
||||
"""Test discovering a single valid skill."""
|
||||
from praisonaiagents.skills.discovery import discover_skills
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
# Create a skill directory
|
||||
skill_dir = Path(tmpdir) / "test-skill"
|
||||
skill_dir.mkdir()
|
||||
(skill_dir / "SKILL.md").write_text("""---
|
||||
name: test-skill
|
||||
description: A test skill for discovery
|
||||
---
|
||||
|
||||
# Test Skill
|
||||
""")
|
||||
|
||||
result = discover_skills([tmpdir], include_defaults=False)
|
||||
|
||||
assert len(result) == 1
|
||||
assert result[0].name == "test-skill"
|
||||
assert result[0].description == "A test skill for discovery"
|
||||
|
||||
def test_discover_multiple_skills(self):
|
||||
"""Test discovering multiple skills."""
|
||||
from praisonaiagents.skills.discovery import discover_skills
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
# Create first skill
|
||||
skill1_dir = Path(tmpdir) / "skill-one"
|
||||
skill1_dir.mkdir()
|
||||
(skill1_dir / "SKILL.md").write_text("""---
|
||||
name: skill-one
|
||||
description: First skill
|
||||
---
|
||||
""")
|
||||
|
||||
# Create second skill
|
||||
skill2_dir = Path(tmpdir) / "skill-two"
|
||||
skill2_dir.mkdir()
|
||||
(skill2_dir / "SKILL.md").write_text("""---
|
||||
name: skill-two
|
||||
description: Second skill
|
||||
---
|
||||
""")
|
||||
|
||||
result = discover_skills([tmpdir], include_defaults=False)
|
||||
|
||||
assert len(result) == 2
|
||||
names = {s.name for s in result}
|
||||
assert "skill-one" in names
|
||||
assert "skill-two" in names
|
||||
|
||||
def test_discover_skips_invalid_skills(self):
|
||||
"""Test that invalid skills are skipped."""
|
||||
from praisonaiagents.skills.discovery import discover_skills
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
# Create valid skill
|
||||
valid_dir = Path(tmpdir) / "valid-skill"
|
||||
valid_dir.mkdir()
|
||||
(valid_dir / "SKILL.md").write_text("""---
|
||||
name: valid-skill
|
||||
description: A valid skill
|
||||
---
|
||||
""")
|
||||
|
||||
# Create invalid skill (no SKILL.md)
|
||||
invalid_dir = Path(tmpdir) / "invalid-skill"
|
||||
invalid_dir.mkdir()
|
||||
|
||||
result = discover_skills([tmpdir], include_defaults=False)
|
||||
|
||||
assert len(result) == 1
|
||||
assert result[0].name == "valid-skill"
|
||||
|
||||
def test_discover_from_multiple_directories(self):
|
||||
"""Test discovering from multiple directories."""
|
||||
from praisonaiagents.skills.discovery import discover_skills
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir1:
|
||||
with tempfile.TemporaryDirectory() as tmpdir2:
|
||||
# Create skill in first directory
|
||||
skill1_dir = Path(tmpdir1) / "skill-a"
|
||||
skill1_dir.mkdir()
|
||||
(skill1_dir / "SKILL.md").write_text("""---
|
||||
name: skill-a
|
||||
description: Skill A
|
||||
---
|
||||
""")
|
||||
|
||||
# Create skill in second directory
|
||||
skill2_dir = Path(tmpdir2) / "skill-b"
|
||||
skill2_dir.mkdir()
|
||||
(skill2_dir / "SKILL.md").write_text("""---
|
||||
name: skill-b
|
||||
description: Skill B
|
||||
---
|
||||
""")
|
||||
|
||||
result = discover_skills([tmpdir1, tmpdir2], include_defaults=False)
|
||||
|
||||
assert len(result) == 2
|
||||
|
||||
|
||||
class TestDiscoverSkill:
|
||||
"""Tests for discover_skill function (single skill)."""
|
||||
|
||||
def test_discover_valid_skill(self):
|
||||
"""Test discovering a single valid skill."""
|
||||
from praisonaiagents.skills.discovery import discover_skill
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
skill_dir = Path(tmpdir) / "my-skill"
|
||||
skill_dir.mkdir()
|
||||
(skill_dir / "SKILL.md").write_text("""---
|
||||
name: my-skill
|
||||
description: My skill
|
||||
---
|
||||
""")
|
||||
|
||||
result = discover_skill(str(skill_dir))
|
||||
|
||||
assert result is not None
|
||||
assert result.name == "my-skill"
|
||||
|
||||
def test_discover_nonexistent_path(self):
|
||||
"""Test discovering from nonexistent path."""
|
||||
from praisonaiagents.skills.discovery import discover_skill
|
||||
|
||||
result = discover_skill("/nonexistent/path")
|
||||
|
||||
assert result is None
|
||||
|
||||
def test_discover_directory_without_skill_md(self):
|
||||
"""Test discovering from directory without SKILL.md."""
|
||||
from praisonaiagents.skills.discovery import discover_skill
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
result = discover_skill(tmpdir)
|
||||
|
||||
assert result is None
|
||||
@@ -0,0 +1,243 @@
|
||||
"""Tests for Agent Skills loader."""
|
||||
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
|
||||
|
||||
class TestSkillLoader:
|
||||
"""Tests for SkillLoader class."""
|
||||
|
||||
def test_load_metadata_valid_skill(self):
|
||||
"""Test loading metadata from valid skill."""
|
||||
from praisonaiagents.skills.loader import SkillLoader
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
skill_dir = Path(tmpdir) / "test-skill"
|
||||
skill_dir.mkdir()
|
||||
(skill_dir / "SKILL.md").write_text("""---
|
||||
name: test-skill
|
||||
description: A test skill
|
||||
---
|
||||
|
||||
# Instructions
|
||||
""")
|
||||
|
||||
loader = SkillLoader()
|
||||
skill = loader.load_metadata(str(skill_dir))
|
||||
|
||||
assert skill is not None
|
||||
assert skill.properties.name == "test-skill"
|
||||
assert skill.instructions is None # Not activated yet
|
||||
|
||||
def test_load_metadata_nonexistent(self):
|
||||
"""Test loading from nonexistent path."""
|
||||
from praisonaiagents.skills.loader import SkillLoader
|
||||
|
||||
loader = SkillLoader()
|
||||
skill = loader.load_metadata("/nonexistent/path")
|
||||
|
||||
assert skill is None
|
||||
|
||||
def test_activate_skill(self):
|
||||
"""Test activating a skill loads instructions."""
|
||||
from praisonaiagents.skills.loader import SkillLoader
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
skill_dir = Path(tmpdir) / "test-skill"
|
||||
skill_dir.mkdir()
|
||||
(skill_dir / "SKILL.md").write_text("""---
|
||||
name: test-skill
|
||||
description: A test skill
|
||||
---
|
||||
|
||||
# Instructions
|
||||
|
||||
Follow these steps:
|
||||
1. Do this
|
||||
2. Do that
|
||||
""")
|
||||
|
||||
loader = SkillLoader()
|
||||
skill = loader.load_metadata(str(skill_dir))
|
||||
|
||||
assert not skill.is_activated
|
||||
|
||||
result = loader.activate(skill)
|
||||
|
||||
assert result is True
|
||||
assert skill.is_activated
|
||||
assert "Follow these steps" in skill.instructions
|
||||
|
||||
def test_activate_already_activated(self):
|
||||
"""Test activating already activated skill returns True."""
|
||||
from praisonaiagents.skills.loader import SkillLoader
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
skill_dir = Path(tmpdir) / "test-skill"
|
||||
skill_dir.mkdir()
|
||||
(skill_dir / "SKILL.md").write_text("""---
|
||||
name: test-skill
|
||||
description: A test skill
|
||||
---
|
||||
|
||||
Instructions.
|
||||
""")
|
||||
|
||||
loader = SkillLoader()
|
||||
skill = loader.load_metadata(str(skill_dir))
|
||||
loader.activate(skill)
|
||||
|
||||
# Activate again
|
||||
result = loader.activate(skill)
|
||||
|
||||
assert result is True
|
||||
|
||||
def test_load_scripts(self):
|
||||
"""Test loading scripts from skill."""
|
||||
from praisonaiagents.skills.loader import SkillLoader
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
skill_dir = Path(tmpdir) / "test-skill"
|
||||
skill_dir.mkdir()
|
||||
(skill_dir / "SKILL.md").write_text("""---
|
||||
name: test-skill
|
||||
description: A test skill
|
||||
---
|
||||
""")
|
||||
|
||||
# Create scripts directory
|
||||
scripts_dir = skill_dir / "scripts"
|
||||
scripts_dir.mkdir()
|
||||
(scripts_dir / "helper.py").write_text("print('hello')")
|
||||
(scripts_dir / "utils.sh").write_text("echo 'hello'")
|
||||
|
||||
loader = SkillLoader()
|
||||
skill = loader.load_metadata(str(skill_dir))
|
||||
scripts = loader.load_scripts(skill)
|
||||
|
||||
assert "helper.py" in scripts
|
||||
assert "utils.sh" in scripts
|
||||
assert "print('hello')" in scripts["helper.py"]
|
||||
|
||||
def test_load_references(self):
|
||||
"""Test loading references from skill."""
|
||||
from praisonaiagents.skills.loader import SkillLoader
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
skill_dir = Path(tmpdir) / "test-skill"
|
||||
skill_dir.mkdir()
|
||||
(skill_dir / "SKILL.md").write_text("""---
|
||||
name: test-skill
|
||||
description: A test skill
|
||||
---
|
||||
""")
|
||||
|
||||
# Create references directory
|
||||
refs_dir = skill_dir / "references"
|
||||
refs_dir.mkdir()
|
||||
(refs_dir / "api.md").write_text("# API Reference")
|
||||
|
||||
loader = SkillLoader()
|
||||
skill = loader.load_metadata(str(skill_dir))
|
||||
refs = loader.load_references(skill)
|
||||
|
||||
assert "api.md" in refs
|
||||
assert "# API Reference" in refs["api.md"]
|
||||
|
||||
def test_load_assets(self):
|
||||
"""Test loading assets from skill."""
|
||||
from praisonaiagents.skills.loader import SkillLoader
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
skill_dir = Path(tmpdir) / "test-skill"
|
||||
skill_dir.mkdir()
|
||||
(skill_dir / "SKILL.md").write_text("""---
|
||||
name: test-skill
|
||||
description: A test skill
|
||||
---
|
||||
""")
|
||||
|
||||
# Create assets directory
|
||||
assets_dir = skill_dir / "assets"
|
||||
assets_dir.mkdir()
|
||||
(assets_dir / "template.json").write_text("{}")
|
||||
|
||||
loader = SkillLoader()
|
||||
skill = loader.load_metadata(str(skill_dir))
|
||||
assets = loader.load_assets(skill)
|
||||
|
||||
assert "template.json" in assets
|
||||
# Path may be resolved differently on macOS (/var vs /private/var)
|
||||
assert assets["template.json"].endswith("test-skill/assets/template.json")
|
||||
|
||||
def test_class_method_load(self):
|
||||
"""Test convenience class method load."""
|
||||
from praisonaiagents.skills.loader import SkillLoader
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
skill_dir = Path(tmpdir) / "test-skill"
|
||||
skill_dir.mkdir()
|
||||
(skill_dir / "SKILL.md").write_text("""---
|
||||
name: test-skill
|
||||
description: A test skill
|
||||
---
|
||||
|
||||
Instructions here.
|
||||
""")
|
||||
|
||||
skill = SkillLoader.load(str(skill_dir), activate=True)
|
||||
|
||||
assert skill is not None
|
||||
assert skill.is_activated
|
||||
assert "Instructions here" in skill.instructions
|
||||
|
||||
|
||||
class TestLoadedSkill:
|
||||
"""Tests for LoadedSkill dataclass."""
|
||||
|
||||
def test_metadata_property(self):
|
||||
"""Test metadata property returns SkillMetadata."""
|
||||
from praisonaiagents.skills.loader import SkillLoader
|
||||
from praisonaiagents.skills.models import SkillMetadata
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
skill_dir = Path(tmpdir) / "test-skill"
|
||||
skill_dir.mkdir()
|
||||
(skill_dir / "SKILL.md").write_text("""---
|
||||
name: test-skill
|
||||
description: A test skill
|
||||
---
|
||||
""")
|
||||
|
||||
loader = SkillLoader()
|
||||
skill = loader.load_metadata(str(skill_dir))
|
||||
|
||||
meta = skill.metadata
|
||||
|
||||
assert isinstance(meta, SkillMetadata)
|
||||
assert meta.name == "test-skill"
|
||||
assert meta.description == "A test skill"
|
||||
|
||||
def test_is_activated_property(self):
|
||||
"""Test is_activated property."""
|
||||
from praisonaiagents.skills.loader import SkillLoader
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
skill_dir = Path(tmpdir) / "test-skill"
|
||||
skill_dir.mkdir()
|
||||
(skill_dir / "SKILL.md").write_text("""---
|
||||
name: test-skill
|
||||
description: A test skill
|
||||
---
|
||||
|
||||
Body.
|
||||
""")
|
||||
|
||||
loader = SkillLoader()
|
||||
skill = loader.load_metadata(str(skill_dir))
|
||||
|
||||
assert skill.is_activated is False
|
||||
|
||||
loader.activate(skill)
|
||||
|
||||
assert skill.is_activated is True
|
||||
@@ -0,0 +1,254 @@
|
||||
"""Tests for Agent Skills manager."""
|
||||
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
|
||||
|
||||
class TestSkillManager:
|
||||
"""Tests for SkillManager class."""
|
||||
|
||||
def test_init_empty(self):
|
||||
"""Test initializing empty manager."""
|
||||
from praisonaiagents.skills.manager import SkillManager
|
||||
|
||||
manager = SkillManager()
|
||||
|
||||
assert len(manager) == 0
|
||||
assert manager.skill_names == []
|
||||
|
||||
def test_discover_skills(self):
|
||||
"""Test discovering skills from directory."""
|
||||
from praisonaiagents.skills.manager import SkillManager
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
# Create a skill
|
||||
skill_dir = Path(tmpdir) / "test-skill"
|
||||
skill_dir.mkdir()
|
||||
(skill_dir / "SKILL.md").write_text("""---
|
||||
name: test-skill
|
||||
description: A test skill
|
||||
---
|
||||
""")
|
||||
|
||||
manager = SkillManager()
|
||||
count = manager.discover([tmpdir], include_defaults=False)
|
||||
|
||||
assert count == 1
|
||||
assert len(manager) == 1
|
||||
assert "test-skill" in manager
|
||||
|
||||
def test_add_skill(self):
|
||||
"""Test adding a single skill."""
|
||||
from praisonaiagents.skills.manager import SkillManager
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
skill_dir = Path(tmpdir) / "my-skill"
|
||||
skill_dir.mkdir()
|
||||
(skill_dir / "SKILL.md").write_text("""---
|
||||
name: my-skill
|
||||
description: My skill
|
||||
---
|
||||
""")
|
||||
|
||||
manager = SkillManager()
|
||||
skill = manager.add_skill(str(skill_dir))
|
||||
|
||||
assert skill is not None
|
||||
assert skill.properties.name == "my-skill"
|
||||
assert "my-skill" in manager
|
||||
|
||||
def test_get_skill(self):
|
||||
"""Test getting a skill by name."""
|
||||
from praisonaiagents.skills.manager import SkillManager
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
skill_dir = Path(tmpdir) / "test-skill"
|
||||
skill_dir.mkdir()
|
||||
(skill_dir / "SKILL.md").write_text("""---
|
||||
name: test-skill
|
||||
description: A test skill
|
||||
---
|
||||
""")
|
||||
|
||||
manager = SkillManager()
|
||||
manager.add_skill(str(skill_dir))
|
||||
|
||||
skill = manager.get_skill("test-skill")
|
||||
|
||||
assert skill is not None
|
||||
assert skill.properties.name == "test-skill"
|
||||
|
||||
def test_get_skill_not_found(self):
|
||||
"""Test getting nonexistent skill."""
|
||||
from praisonaiagents.skills.manager import SkillManager
|
||||
|
||||
manager = SkillManager()
|
||||
skill = manager.get_skill("nonexistent")
|
||||
|
||||
assert skill is None
|
||||
|
||||
def test_activate_by_name(self):
|
||||
"""Test activating skill by name."""
|
||||
from praisonaiagents.skills.manager import SkillManager
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
skill_dir = Path(tmpdir) / "test-skill"
|
||||
skill_dir.mkdir()
|
||||
(skill_dir / "SKILL.md").write_text("""---
|
||||
name: test-skill
|
||||
description: A test skill
|
||||
---
|
||||
|
||||
# Instructions
|
||||
""")
|
||||
|
||||
manager = SkillManager()
|
||||
manager.add_skill(str(skill_dir))
|
||||
|
||||
result = manager.activate_by_name("test-skill")
|
||||
|
||||
assert result is True
|
||||
skill = manager.get_skill("test-skill")
|
||||
assert skill.is_activated
|
||||
|
||||
def test_get_available_skills(self):
|
||||
"""Test getting available skills metadata."""
|
||||
from praisonaiagents.skills.manager import SkillManager
|
||||
from praisonaiagents.skills.models import SkillMetadata
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
skill_dir = Path(tmpdir) / "test-skill"
|
||||
skill_dir.mkdir()
|
||||
(skill_dir / "SKILL.md").write_text("""---
|
||||
name: test-skill
|
||||
description: A test skill
|
||||
---
|
||||
""")
|
||||
|
||||
manager = SkillManager()
|
||||
manager.add_skill(str(skill_dir))
|
||||
|
||||
skills = manager.get_available_skills()
|
||||
|
||||
assert len(skills) == 1
|
||||
assert isinstance(skills[0], SkillMetadata)
|
||||
assert skills[0].name == "test-skill"
|
||||
|
||||
def test_to_prompt(self):
|
||||
"""Test generating prompt XML."""
|
||||
from praisonaiagents.skills.manager import SkillManager
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
skill_dir = Path(tmpdir) / "test-skill"
|
||||
skill_dir.mkdir()
|
||||
(skill_dir / "SKILL.md").write_text("""---
|
||||
name: test-skill
|
||||
description: A test skill
|
||||
---
|
||||
""")
|
||||
|
||||
manager = SkillManager()
|
||||
manager.add_skill(str(skill_dir))
|
||||
|
||||
prompt = manager.to_prompt()
|
||||
|
||||
assert "<available_skills>" in prompt
|
||||
assert "test-skill" in prompt
|
||||
assert "A test skill" in prompt
|
||||
|
||||
def test_get_instructions(self):
|
||||
"""Test getting instructions for a skill."""
|
||||
from praisonaiagents.skills.manager import SkillManager
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
skill_dir = Path(tmpdir) / "test-skill"
|
||||
skill_dir.mkdir()
|
||||
(skill_dir / "SKILL.md").write_text("""---
|
||||
name: test-skill
|
||||
description: A test skill
|
||||
---
|
||||
|
||||
# Instructions
|
||||
|
||||
Do this and that.
|
||||
""")
|
||||
|
||||
manager = SkillManager()
|
||||
manager.add_skill(str(skill_dir))
|
||||
|
||||
instructions = manager.get_instructions("test-skill")
|
||||
|
||||
assert instructions is not None
|
||||
assert "Do this and that" in instructions
|
||||
|
||||
def test_load_resources(self):
|
||||
"""Test loading resources for a skill."""
|
||||
from praisonaiagents.skills.manager import SkillManager
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
skill_dir = Path(tmpdir) / "test-skill"
|
||||
skill_dir.mkdir()
|
||||
(skill_dir / "SKILL.md").write_text("""---
|
||||
name: test-skill
|
||||
description: A test skill
|
||||
---
|
||||
""")
|
||||
|
||||
# Create scripts directory
|
||||
scripts_dir = skill_dir / "scripts"
|
||||
scripts_dir.mkdir()
|
||||
(scripts_dir / "helper.py").write_text("print('hello')")
|
||||
|
||||
manager = SkillManager()
|
||||
manager.add_skill(str(skill_dir))
|
||||
|
||||
result = manager.load_resources("test-skill")
|
||||
|
||||
assert result is True
|
||||
skill = manager.get_skill("test-skill")
|
||||
assert skill.resources_loaded
|
||||
|
||||
def test_clear(self):
|
||||
"""Test clearing all skills."""
|
||||
from praisonaiagents.skills.manager import SkillManager
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
skill_dir = Path(tmpdir) / "test-skill"
|
||||
skill_dir.mkdir()
|
||||
(skill_dir / "SKILL.md").write_text("""---
|
||||
name: test-skill
|
||||
description: A test skill
|
||||
---
|
||||
""")
|
||||
|
||||
manager = SkillManager()
|
||||
manager.add_skill(str(skill_dir))
|
||||
assert len(manager) == 1
|
||||
|
||||
manager.clear()
|
||||
|
||||
assert len(manager) == 0
|
||||
|
||||
def test_iteration(self):
|
||||
"""Test iterating over skills."""
|
||||
from praisonaiagents.skills.manager import SkillManager
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
# Create two skills
|
||||
for name in ["skill-a", "skill-b"]:
|
||||
skill_dir = Path(tmpdir) / name
|
||||
skill_dir.mkdir()
|
||||
(skill_dir / "SKILL.md").write_text(f"""---
|
||||
name: {name}
|
||||
description: Skill {name}
|
||||
---
|
||||
""")
|
||||
|
||||
manager = SkillManager()
|
||||
manager.discover([tmpdir], include_defaults=False)
|
||||
|
||||
names = [s.properties.name for s in manager]
|
||||
|
||||
assert len(names) == 2
|
||||
assert "skill-a" in names
|
||||
assert "skill-b" in names
|
||||
@@ -0,0 +1,184 @@
|
||||
"""Tests for Agent Skills models."""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
class TestSkillProperties:
|
||||
"""Tests for SkillProperties dataclass."""
|
||||
|
||||
def test_skill_properties_required_fields(self):
|
||||
"""Test that SkillProperties requires name and description."""
|
||||
from praisonaiagents.skills.models import SkillProperties
|
||||
|
||||
skill = SkillProperties(
|
||||
name="test-skill",
|
||||
description="A test skill for unit testing"
|
||||
)
|
||||
|
||||
assert skill.name == "test-skill"
|
||||
assert skill.description == "A test skill for unit testing"
|
||||
|
||||
def test_skill_properties_optional_fields_default_none(self):
|
||||
"""Test that optional fields default to None."""
|
||||
from praisonaiagents.skills.models import SkillProperties
|
||||
|
||||
skill = SkillProperties(
|
||||
name="test-skill",
|
||||
description="A test skill"
|
||||
)
|
||||
|
||||
assert skill.license is None
|
||||
assert skill.compatibility is None
|
||||
assert skill.allowed_tools is None
|
||||
assert skill.path is None
|
||||
|
||||
def test_skill_properties_metadata_defaults_empty_dict(self):
|
||||
"""Test that metadata defaults to empty dict."""
|
||||
from praisonaiagents.skills.models import SkillProperties
|
||||
|
||||
skill = SkillProperties(
|
||||
name="test-skill",
|
||||
description="A test skill"
|
||||
)
|
||||
|
||||
assert skill.metadata == {}
|
||||
|
||||
def test_skill_properties_all_fields(self):
|
||||
"""Test SkillProperties with all fields populated."""
|
||||
from praisonaiagents.skills.models import SkillProperties
|
||||
|
||||
skill = SkillProperties(
|
||||
name="pdf-processing",
|
||||
description="Extract text and tables from PDF files",
|
||||
license="Apache-2.0",
|
||||
compatibility="Requires pypdf package",
|
||||
allowed_tools="Read Grep Bash",
|
||||
metadata={"author": "test-org", "version": "1.0"},
|
||||
path=Path("/path/to/skill")
|
||||
)
|
||||
|
||||
assert skill.name == "pdf-processing"
|
||||
assert skill.description == "Extract text and tables from PDF files"
|
||||
assert skill.license == "Apache-2.0"
|
||||
assert skill.compatibility == "Requires pypdf package"
|
||||
assert skill.allowed_tools == "Read Grep Bash"
|
||||
assert skill.metadata == {"author": "test-org", "version": "1.0"}
|
||||
assert skill.path == Path("/path/to/skill")
|
||||
|
||||
def test_skill_properties_to_dict(self):
|
||||
"""Test to_dict method excludes None values."""
|
||||
from praisonaiagents.skills.models import SkillProperties
|
||||
|
||||
skill = SkillProperties(
|
||||
name="test-skill",
|
||||
description="A test skill",
|
||||
license="MIT"
|
||||
)
|
||||
|
||||
result = skill.to_dict()
|
||||
|
||||
assert result["name"] == "test-skill"
|
||||
assert result["description"] == "A test skill"
|
||||
assert result["license"] == "MIT"
|
||||
assert "compatibility" not in result
|
||||
assert "allowed-tools" not in result
|
||||
assert "metadata" not in result # Empty metadata excluded
|
||||
|
||||
def test_skill_properties_to_dict_with_metadata(self):
|
||||
"""Test to_dict includes non-empty metadata."""
|
||||
from praisonaiagents.skills.models import SkillProperties
|
||||
|
||||
skill = SkillProperties(
|
||||
name="test-skill",
|
||||
description="A test skill",
|
||||
metadata={"version": "1.0"}
|
||||
)
|
||||
|
||||
result = skill.to_dict()
|
||||
|
||||
assert result["metadata"] == {"version": "1.0"}
|
||||
|
||||
def test_skill_properties_to_dict_allowed_tools_key(self):
|
||||
"""Test to_dict uses 'allowed-tools' key (hyphenated)."""
|
||||
from praisonaiagents.skills.models import SkillProperties
|
||||
|
||||
skill = SkillProperties(
|
||||
name="test-skill",
|
||||
description="A test skill",
|
||||
allowed_tools="Read Write"
|
||||
)
|
||||
|
||||
result = skill.to_dict()
|
||||
|
||||
assert "allowed-tools" in result
|
||||
assert result["allowed-tools"] == "Read Write"
|
||||
|
||||
|
||||
class TestSkillMetadata:
|
||||
"""Tests for SkillMetadata dataclass (lightweight for system prompt)."""
|
||||
|
||||
def test_skill_metadata_required_fields(self):
|
||||
"""Test SkillMetadata with required fields."""
|
||||
from praisonaiagents.skills.models import SkillMetadata
|
||||
|
||||
meta = SkillMetadata(
|
||||
name="test-skill",
|
||||
description="A test skill",
|
||||
location="/path/to/skill/SKILL.md"
|
||||
)
|
||||
|
||||
assert meta.name == "test-skill"
|
||||
assert meta.description == "A test skill"
|
||||
assert meta.location == "/path/to/skill/SKILL.md"
|
||||
|
||||
def test_skill_metadata_from_properties(self):
|
||||
"""Test creating SkillMetadata from SkillProperties."""
|
||||
from praisonaiagents.skills.models import SkillProperties, SkillMetadata
|
||||
|
||||
props = SkillProperties(
|
||||
name="pdf-skill",
|
||||
description="Process PDF files",
|
||||
path=Path("/skills/pdf-skill")
|
||||
)
|
||||
|
||||
meta = SkillMetadata.from_properties(props)
|
||||
|
||||
assert meta.name == "pdf-skill"
|
||||
assert meta.description == "Process PDF files"
|
||||
assert "/skills/pdf-skill" in meta.location
|
||||
|
||||
|
||||
class TestParseError:
|
||||
"""Tests for ParseError exception."""
|
||||
|
||||
def test_parse_error_message(self):
|
||||
"""Test ParseError stores message correctly."""
|
||||
from praisonaiagents.skills.models import ParseError
|
||||
|
||||
error = ParseError("Invalid YAML frontmatter")
|
||||
|
||||
assert str(error) == "Invalid YAML frontmatter"
|
||||
|
||||
def test_parse_error_is_exception(self):
|
||||
"""Test ParseError is an Exception subclass."""
|
||||
from praisonaiagents.skills.models import ParseError
|
||||
|
||||
assert issubclass(ParseError, Exception)
|
||||
|
||||
|
||||
class TestValidationError:
|
||||
"""Tests for ValidationError exception."""
|
||||
|
||||
def test_validation_error_message(self):
|
||||
"""Test ValidationError stores message correctly."""
|
||||
from praisonaiagents.skills.models import ValidationError
|
||||
|
||||
error = ValidationError("Missing required field: name")
|
||||
|
||||
assert str(error) == "Missing required field: name"
|
||||
|
||||
def test_validation_error_is_exception(self):
|
||||
"""Test ValidationError is an Exception subclass."""
|
||||
from praisonaiagents.skills.models import ValidationError
|
||||
|
||||
assert issubclass(ValidationError, Exception)
|
||||
@@ -0,0 +1,288 @@
|
||||
"""Tests for Agent Skills parser."""
|
||||
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
|
||||
|
||||
class TestParseFrontmatter:
|
||||
"""Tests for parse_frontmatter function."""
|
||||
|
||||
def test_parse_valid_frontmatter(self):
|
||||
"""Test parsing valid YAML frontmatter."""
|
||||
from praisonaiagents.skills.parser import parse_frontmatter
|
||||
|
||||
content = """---
|
||||
name: test-skill
|
||||
description: A test skill for unit testing
|
||||
---
|
||||
|
||||
# Test Skill
|
||||
|
||||
Instructions go here.
|
||||
"""
|
||||
metadata, body = parse_frontmatter(content)
|
||||
|
||||
assert metadata["name"] == "test-skill"
|
||||
assert metadata["description"] == "A test skill for unit testing"
|
||||
assert "# Test Skill" in body
|
||||
|
||||
def test_parse_frontmatter_with_optional_fields(self):
|
||||
"""Test parsing frontmatter with all optional fields."""
|
||||
from praisonaiagents.skills.parser import parse_frontmatter
|
||||
|
||||
content = """---
|
||||
name: pdf-processing
|
||||
description: Extract text from PDFs
|
||||
license: Apache-2.0
|
||||
compatibility: Requires pypdf
|
||||
allowed-tools: Read Grep
|
||||
metadata:
|
||||
author: test-org
|
||||
version: "1.0"
|
||||
---
|
||||
|
||||
Body content.
|
||||
"""
|
||||
metadata, body = parse_frontmatter(content)
|
||||
|
||||
assert metadata["name"] == "pdf-processing"
|
||||
assert metadata["license"] == "Apache-2.0"
|
||||
assert metadata["compatibility"] == "Requires pypdf"
|
||||
assert metadata["allowed-tools"] == "Read Grep"
|
||||
assert metadata["metadata"]["author"] == "test-org"
|
||||
assert metadata["metadata"]["version"] == "1.0"
|
||||
|
||||
def test_parse_frontmatter_missing_opening_delimiter(self):
|
||||
"""Test error when frontmatter doesn't start with ---."""
|
||||
from praisonaiagents.skills.parser import parse_frontmatter
|
||||
from praisonaiagents.skills.models import ParseError
|
||||
|
||||
content = """name: test-skill
|
||||
description: Missing opening delimiter
|
||||
---
|
||||
|
||||
Body content.
|
||||
"""
|
||||
try:
|
||||
parse_frontmatter(content)
|
||||
assert False, "Should have raised ParseError"
|
||||
except ParseError as e:
|
||||
assert "must start with" in str(e).lower() or "---" in str(e)
|
||||
|
||||
def test_parse_frontmatter_missing_closing_delimiter(self):
|
||||
"""Test error when frontmatter is not closed."""
|
||||
from praisonaiagents.skills.parser import parse_frontmatter
|
||||
from praisonaiagents.skills.models import ParseError
|
||||
|
||||
content = """---
|
||||
name: test-skill
|
||||
description: Missing closing delimiter
|
||||
|
||||
Body content without closing delimiter.
|
||||
"""
|
||||
try:
|
||||
parse_frontmatter(content)
|
||||
assert False, "Should have raised ParseError"
|
||||
except ParseError as e:
|
||||
assert "closed" in str(e).lower() or "---" in str(e)
|
||||
|
||||
def test_parse_frontmatter_invalid_yaml(self):
|
||||
"""Test error when YAML is invalid."""
|
||||
from praisonaiagents.skills.parser import parse_frontmatter
|
||||
from praisonaiagents.skills.models import ParseError
|
||||
|
||||
content = """---
|
||||
name: test-skill
|
||||
description: [invalid yaml
|
||||
- missing bracket
|
||||
---
|
||||
|
||||
Body content.
|
||||
"""
|
||||
try:
|
||||
parse_frontmatter(content)
|
||||
assert False, "Should have raised ParseError"
|
||||
except ParseError as e:
|
||||
assert "yaml" in str(e).lower() or "invalid" in str(e).lower()
|
||||
|
||||
def test_parse_frontmatter_empty_body(self):
|
||||
"""Test parsing when body is empty."""
|
||||
from praisonaiagents.skills.parser import parse_frontmatter
|
||||
|
||||
content = """---
|
||||
name: test-skill
|
||||
description: A skill with no body
|
||||
---
|
||||
"""
|
||||
metadata, body = parse_frontmatter(content)
|
||||
|
||||
assert metadata["name"] == "test-skill"
|
||||
assert body.strip() == ""
|
||||
|
||||
|
||||
class TestFindSkillMd:
|
||||
"""Tests for find_skill_md function."""
|
||||
|
||||
def test_find_skill_md_uppercase(self):
|
||||
"""Test finding SKILL.md (uppercase)."""
|
||||
from praisonaiagents.skills.parser import find_skill_md
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
skill_dir = Path(tmpdir)
|
||||
skill_md = skill_dir / "SKILL.md"
|
||||
skill_md.write_text("---\nname: test\ndescription: test\n---\n")
|
||||
|
||||
result = find_skill_md(skill_dir)
|
||||
|
||||
assert result is not None
|
||||
assert result.name == "SKILL.md"
|
||||
|
||||
def test_find_skill_md_lowercase(self):
|
||||
"""Test finding skill.md (lowercase) as fallback."""
|
||||
from praisonaiagents.skills.parser import find_skill_md
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
skill_dir = Path(tmpdir)
|
||||
skill_md = skill_dir / "skill.md"
|
||||
skill_md.write_text("---\nname: test\ndescription: test\n---\n")
|
||||
|
||||
result = find_skill_md(skill_dir)
|
||||
|
||||
assert result is not None
|
||||
# On case-insensitive filesystems (macOS), SKILL.md may match skill.md
|
||||
assert result.name.lower() == "skill.md"
|
||||
|
||||
def test_find_skill_md_prefers_uppercase(self):
|
||||
"""Test that SKILL.md is preferred over skill.md."""
|
||||
from praisonaiagents.skills.parser import find_skill_md
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
skill_dir = Path(tmpdir)
|
||||
(skill_dir / "SKILL.md").write_text("uppercase")
|
||||
(skill_dir / "skill.md").write_text("lowercase")
|
||||
|
||||
result = find_skill_md(skill_dir)
|
||||
|
||||
assert result is not None
|
||||
assert result.name == "SKILL.md"
|
||||
|
||||
def test_find_skill_md_not_found(self):
|
||||
"""Test returning None when no SKILL.md exists."""
|
||||
from praisonaiagents.skills.parser import find_skill_md
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
skill_dir = Path(tmpdir)
|
||||
|
||||
result = find_skill_md(skill_dir)
|
||||
|
||||
assert result is None
|
||||
|
||||
|
||||
class TestReadProperties:
|
||||
"""Tests for read_properties function."""
|
||||
|
||||
def test_read_properties_valid_skill(self):
|
||||
"""Test reading properties from a valid skill directory."""
|
||||
from praisonaiagents.skills.parser import read_properties
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
skill_dir = Path(tmpdir) / "test-skill"
|
||||
skill_dir.mkdir()
|
||||
(skill_dir / "SKILL.md").write_text("""---
|
||||
name: test-skill
|
||||
description: A test skill for unit testing
|
||||
---
|
||||
|
||||
# Test Skill
|
||||
""")
|
||||
|
||||
props = read_properties(skill_dir)
|
||||
|
||||
assert props.name == "test-skill"
|
||||
assert props.description == "A test skill for unit testing"
|
||||
|
||||
def test_read_properties_with_all_fields(self):
|
||||
"""Test reading properties with all optional fields."""
|
||||
from praisonaiagents.skills.parser import read_properties
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
skill_dir = Path(tmpdir) / "pdf-skill"
|
||||
skill_dir.mkdir()
|
||||
(skill_dir / "SKILL.md").write_text("""---
|
||||
name: pdf-skill
|
||||
description: Process PDF files
|
||||
license: MIT
|
||||
compatibility: Requires pypdf
|
||||
allowed-tools: Read Write
|
||||
metadata:
|
||||
author: test
|
||||
version: "2.0"
|
||||
---
|
||||
|
||||
Instructions.
|
||||
""")
|
||||
|
||||
props = read_properties(skill_dir)
|
||||
|
||||
assert props.name == "pdf-skill"
|
||||
assert props.license == "MIT"
|
||||
assert props.compatibility == "Requires pypdf"
|
||||
assert props.allowed_tools == "Read Write"
|
||||
assert props.metadata["author"] == "test"
|
||||
|
||||
def test_read_properties_missing_skill_md(self):
|
||||
"""Test error when SKILL.md is missing."""
|
||||
from praisonaiagents.skills.parser import read_properties
|
||||
from praisonaiagents.skills.models import ParseError
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
skill_dir = Path(tmpdir) / "empty-skill"
|
||||
skill_dir.mkdir()
|
||||
|
||||
try:
|
||||
read_properties(skill_dir)
|
||||
assert False, "Should have raised ParseError"
|
||||
except ParseError as e:
|
||||
assert "not found" in str(e).lower() or "skill.md" in str(e).lower()
|
||||
|
||||
def test_read_properties_missing_name(self):
|
||||
"""Test error when name field is missing."""
|
||||
from praisonaiagents.skills.parser import read_properties
|
||||
from praisonaiagents.skills.models import ValidationError
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
skill_dir = Path(tmpdir) / "no-name"
|
||||
skill_dir.mkdir()
|
||||
(skill_dir / "SKILL.md").write_text("""---
|
||||
description: Missing name field
|
||||
---
|
||||
|
||||
Body.
|
||||
""")
|
||||
|
||||
try:
|
||||
read_properties(skill_dir)
|
||||
assert False, "Should have raised ValidationError"
|
||||
except ValidationError as e:
|
||||
assert "name" in str(e).lower()
|
||||
|
||||
def test_read_properties_missing_description(self):
|
||||
"""Test error when description field is missing."""
|
||||
from praisonaiagents.skills.parser import read_properties
|
||||
from praisonaiagents.skills.models import ValidationError
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
skill_dir = Path(tmpdir) / "no-desc"
|
||||
skill_dir.mkdir()
|
||||
(skill_dir / "SKILL.md").write_text("""---
|
||||
name: no-desc
|
||||
---
|
||||
|
||||
Body.
|
||||
""")
|
||||
|
||||
try:
|
||||
read_properties(skill_dir)
|
||||
assert False, "Should have raised ValidationError"
|
||||
except ValidationError as e:
|
||||
assert "description" in str(e).lower()
|
||||
@@ -0,0 +1,180 @@
|
||||
"""Tests for Agent Skills prompt generation."""
|
||||
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
|
||||
|
||||
class TestToPrompt:
|
||||
"""Tests for to_prompt function (XML generation)."""
|
||||
|
||||
def test_to_prompt_empty_list(self):
|
||||
"""Test generating prompt with no skills."""
|
||||
from praisonaiagents.skills.prompt import to_prompt
|
||||
|
||||
result = to_prompt([])
|
||||
|
||||
assert "<available_skills>" in result
|
||||
assert "</available_skills>" in result
|
||||
|
||||
def test_to_prompt_single_skill(self):
|
||||
"""Test generating prompt with one skill."""
|
||||
from praisonaiagents.skills.prompt import to_prompt
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
skill_dir = Path(tmpdir) / "test-skill"
|
||||
skill_dir.mkdir()
|
||||
(skill_dir / "SKILL.md").write_text("""---
|
||||
name: test-skill
|
||||
description: A test skill for prompt generation
|
||||
---
|
||||
|
||||
# Test Skill
|
||||
""")
|
||||
|
||||
result = to_prompt([skill_dir])
|
||||
|
||||
assert "<available_skills>" in result
|
||||
assert "</available_skills>" in result
|
||||
assert "<skill>" in result
|
||||
assert "</skill>" in result
|
||||
assert "<name>" in result
|
||||
assert "test-skill" in result
|
||||
assert "<description>" in result
|
||||
assert "A test skill for prompt generation" in result
|
||||
assert "<location>" in result
|
||||
|
||||
def test_to_prompt_multiple_skills(self):
|
||||
"""Test generating prompt with multiple skills."""
|
||||
from praisonaiagents.skills.prompt import to_prompt
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
# Create first skill
|
||||
skill1_dir = Path(tmpdir) / "skill-one"
|
||||
skill1_dir.mkdir()
|
||||
(skill1_dir / "SKILL.md").write_text("""---
|
||||
name: skill-one
|
||||
description: First skill
|
||||
---
|
||||
""")
|
||||
|
||||
# Create second skill
|
||||
skill2_dir = Path(tmpdir) / "skill-two"
|
||||
skill2_dir.mkdir()
|
||||
(skill2_dir / "SKILL.md").write_text("""---
|
||||
name: skill-two
|
||||
description: Second skill
|
||||
---
|
||||
""")
|
||||
|
||||
result = to_prompt([skill1_dir, skill2_dir])
|
||||
|
||||
assert result.count("<skill>") == 2
|
||||
assert result.count("</skill>") == 2
|
||||
assert "skill-one" in result
|
||||
assert "skill-two" in result
|
||||
assert "First skill" in result
|
||||
assert "Second skill" in result
|
||||
|
||||
def test_to_prompt_escapes_html(self):
|
||||
"""Test that special characters are HTML escaped."""
|
||||
from praisonaiagents.skills.prompt import to_prompt
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
skill_dir = Path(tmpdir) / "escape-test"
|
||||
skill_dir.mkdir()
|
||||
(skill_dir / "SKILL.md").write_text("""---
|
||||
name: escape-test
|
||||
description: Test <script> & "quotes" handling
|
||||
---
|
||||
""")
|
||||
|
||||
result = to_prompt([skill_dir])
|
||||
|
||||
# Should escape < > & "
|
||||
assert "<script>" not in result or "<script>" in result
|
||||
assert "&" in result or "& " not in result
|
||||
|
||||
def test_to_prompt_includes_location(self):
|
||||
"""Test that location points to SKILL.md file."""
|
||||
from praisonaiagents.skills.prompt import to_prompt
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
skill_dir = Path(tmpdir) / "location-test"
|
||||
skill_dir.mkdir()
|
||||
(skill_dir / "SKILL.md").write_text("""---
|
||||
name: location-test
|
||||
description: Test location field
|
||||
---
|
||||
""")
|
||||
|
||||
result = to_prompt([skill_dir])
|
||||
|
||||
assert "<location>" in result
|
||||
assert "SKILL.md" in result
|
||||
assert "</location>" in result
|
||||
|
||||
|
||||
class TestGenerateSkillsXml:
|
||||
"""Tests for generate_skills_xml helper function."""
|
||||
|
||||
def test_generate_skills_xml_from_metadata_list(self):
|
||||
"""Test generating XML from SkillMetadata list."""
|
||||
from praisonaiagents.skills.prompt import generate_skills_xml
|
||||
from praisonaiagents.skills.models import SkillMetadata
|
||||
|
||||
skills = [
|
||||
SkillMetadata(
|
||||
name="pdf-skill",
|
||||
description="Process PDF files",
|
||||
location="/path/to/pdf-skill/SKILL.md"
|
||||
),
|
||||
SkillMetadata(
|
||||
name="excel-skill",
|
||||
description="Process Excel files",
|
||||
location="/path/to/excel-skill/SKILL.md"
|
||||
)
|
||||
]
|
||||
|
||||
result = generate_skills_xml(skills)
|
||||
|
||||
assert "<available_skills>" in result
|
||||
assert "pdf-skill" in result
|
||||
assert "excel-skill" in result
|
||||
assert "Process PDF files" in result
|
||||
assert "Process Excel files" in result
|
||||
|
||||
def test_generate_skills_xml_empty_list(self):
|
||||
"""Test generating XML with empty list."""
|
||||
from praisonaiagents.skills.prompt import generate_skills_xml
|
||||
|
||||
result = generate_skills_xml([])
|
||||
|
||||
assert "<available_skills>" in result
|
||||
assert "</available_skills>" in result
|
||||
assert "<skill>" not in result
|
||||
|
||||
|
||||
class TestFormatSkillForPrompt:
|
||||
"""Tests for format_skill_for_prompt helper."""
|
||||
|
||||
def test_format_single_skill(self):
|
||||
"""Test formatting a single skill for prompt."""
|
||||
from praisonaiagents.skills.prompt import format_skill_for_prompt
|
||||
from praisonaiagents.skills.models import SkillMetadata
|
||||
|
||||
skill = SkillMetadata(
|
||||
name="test-skill",
|
||||
description="A test skill",
|
||||
location="/path/to/SKILL.md"
|
||||
)
|
||||
|
||||
result = format_skill_for_prompt(skill)
|
||||
|
||||
assert "<skill>" in result
|
||||
assert "</skill>" in result
|
||||
assert "<name>" in result
|
||||
assert "test-skill" in result
|
||||
assert "<description>" in result
|
||||
assert "A test skill" in result
|
||||
assert "<location>" in result
|
||||
assert "/path/to/SKILL.md" in result
|
||||
@@ -0,0 +1,258 @@
|
||||
"""Tests for Agent Skills validator."""
|
||||
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
|
||||
|
||||
class TestValidateName:
|
||||
"""Tests for skill name validation."""
|
||||
|
||||
def test_valid_name_simple(self):
|
||||
"""Test valid simple skill name."""
|
||||
from praisonaiagents.skills.validator import _validate_name
|
||||
|
||||
errors = _validate_name("test-skill", None)
|
||||
|
||||
assert errors == []
|
||||
|
||||
def test_valid_name_with_numbers(self):
|
||||
"""Test valid name with numbers."""
|
||||
from praisonaiagents.skills.validator import _validate_name
|
||||
|
||||
errors = _validate_name("skill-v2", None)
|
||||
|
||||
assert errors == []
|
||||
|
||||
def test_invalid_name_uppercase(self):
|
||||
"""Test that uppercase letters are rejected."""
|
||||
from praisonaiagents.skills.validator import _validate_name
|
||||
|
||||
errors = _validate_name("Test-Skill", None)
|
||||
|
||||
assert len(errors) > 0
|
||||
assert any("lowercase" in e.lower() for e in errors)
|
||||
|
||||
def test_invalid_name_starts_with_hyphen(self):
|
||||
"""Test that names starting with hyphen are rejected."""
|
||||
from praisonaiagents.skills.validator import _validate_name
|
||||
|
||||
errors = _validate_name("-skill", None)
|
||||
|
||||
assert len(errors) > 0
|
||||
assert any("hyphen" in e.lower() for e in errors)
|
||||
|
||||
def test_invalid_name_ends_with_hyphen(self):
|
||||
"""Test that names ending with hyphen are rejected."""
|
||||
from praisonaiagents.skills.validator import _validate_name
|
||||
|
||||
errors = _validate_name("skill-", None)
|
||||
|
||||
assert len(errors) > 0
|
||||
assert any("hyphen" in e.lower() for e in errors)
|
||||
|
||||
def test_invalid_name_consecutive_hyphens(self):
|
||||
"""Test that consecutive hyphens are rejected."""
|
||||
from praisonaiagents.skills.validator import _validate_name
|
||||
|
||||
errors = _validate_name("test--skill", None)
|
||||
|
||||
assert len(errors) > 0
|
||||
assert any("consecutive" in e.lower() for e in errors)
|
||||
|
||||
def test_invalid_name_too_long(self):
|
||||
"""Test that names over 64 characters are rejected."""
|
||||
from praisonaiagents.skills.validator import _validate_name
|
||||
|
||||
long_name = "a" * 65
|
||||
errors = _validate_name(long_name, None)
|
||||
|
||||
assert len(errors) > 0
|
||||
assert any("64" in e or "limit" in e.lower() for e in errors)
|
||||
|
||||
def test_invalid_name_special_characters(self):
|
||||
"""Test that special characters are rejected."""
|
||||
from praisonaiagents.skills.validator import _validate_name
|
||||
|
||||
errors = _validate_name("test_skill", None)
|
||||
|
||||
assert len(errors) > 0
|
||||
assert any("invalid" in e.lower() or "character" in e.lower() for e in errors)
|
||||
|
||||
def test_name_must_match_directory(self):
|
||||
"""Test that name must match parent directory name."""
|
||||
from praisonaiagents.skills.validator import _validate_name
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
skill_dir = Path(tmpdir) / "actual-name"
|
||||
skill_dir.mkdir()
|
||||
|
||||
errors = _validate_name("different-name", skill_dir)
|
||||
|
||||
assert len(errors) > 0
|
||||
assert any("match" in e.lower() for e in errors)
|
||||
|
||||
|
||||
class TestValidateDescription:
|
||||
"""Tests for skill description validation."""
|
||||
|
||||
def test_valid_description(self):
|
||||
"""Test valid description."""
|
||||
from praisonaiagents.skills.validator import _validate_description
|
||||
|
||||
errors = _validate_description("A helpful skill that does things")
|
||||
|
||||
assert errors == []
|
||||
|
||||
def test_invalid_description_empty(self):
|
||||
"""Test that empty description is rejected."""
|
||||
from praisonaiagents.skills.validator import _validate_description
|
||||
|
||||
errors = _validate_description("")
|
||||
|
||||
assert len(errors) > 0
|
||||
assert any("empty" in e.lower() or "non-empty" in e.lower() for e in errors)
|
||||
|
||||
def test_invalid_description_too_long(self):
|
||||
"""Test that descriptions over 1024 characters are rejected."""
|
||||
from praisonaiagents.skills.validator import _validate_description
|
||||
|
||||
long_desc = "a" * 1025
|
||||
errors = _validate_description(long_desc)
|
||||
|
||||
assert len(errors) > 0
|
||||
assert any("1024" in e or "limit" in e.lower() for e in errors)
|
||||
|
||||
|
||||
class TestValidateCompatibility:
|
||||
"""Tests for skill compatibility validation."""
|
||||
|
||||
def test_valid_compatibility(self):
|
||||
"""Test valid compatibility string."""
|
||||
from praisonaiagents.skills.validator import _validate_compatibility
|
||||
|
||||
errors = _validate_compatibility("Requires Python 3.11+")
|
||||
|
||||
assert errors == []
|
||||
|
||||
def test_invalid_compatibility_too_long(self):
|
||||
"""Test that compatibility over 500 characters is rejected."""
|
||||
from praisonaiagents.skills.validator import _validate_compatibility
|
||||
|
||||
long_compat = "a" * 501
|
||||
errors = _validate_compatibility(long_compat)
|
||||
|
||||
assert len(errors) > 0
|
||||
assert any("500" in e or "limit" in e.lower() for e in errors)
|
||||
|
||||
|
||||
class TestValidateMetadata:
|
||||
"""Tests for full metadata validation."""
|
||||
|
||||
def test_valid_metadata(self):
|
||||
"""Test valid complete metadata."""
|
||||
from praisonaiagents.skills.validator import validate_metadata
|
||||
|
||||
metadata = {
|
||||
"name": "test-skill",
|
||||
"description": "A test skill"
|
||||
}
|
||||
|
||||
errors = validate_metadata(metadata)
|
||||
|
||||
assert errors == []
|
||||
|
||||
def test_missing_name(self):
|
||||
"""Test error when name is missing."""
|
||||
from praisonaiagents.skills.validator import validate_metadata
|
||||
|
||||
metadata = {
|
||||
"description": "A test skill"
|
||||
}
|
||||
|
||||
errors = validate_metadata(metadata)
|
||||
|
||||
assert len(errors) > 0
|
||||
assert any("name" in e.lower() for e in errors)
|
||||
|
||||
def test_missing_description(self):
|
||||
"""Test error when description is missing."""
|
||||
from praisonaiagents.skills.validator import validate_metadata
|
||||
|
||||
metadata = {
|
||||
"name": "test-skill"
|
||||
}
|
||||
|
||||
errors = validate_metadata(metadata)
|
||||
|
||||
assert len(errors) > 0
|
||||
assert any("description" in e.lower() for e in errors)
|
||||
|
||||
def test_unexpected_fields_rejected(self):
|
||||
"""Test that unexpected fields are rejected."""
|
||||
from praisonaiagents.skills.validator import validate_metadata
|
||||
|
||||
metadata = {
|
||||
"name": "test-skill",
|
||||
"description": "A test skill",
|
||||
"unknown-field": "value"
|
||||
}
|
||||
|
||||
errors = validate_metadata(metadata)
|
||||
|
||||
assert len(errors) > 0
|
||||
assert any("unexpected" in e.lower() or "unknown-field" in e for e in errors)
|
||||
|
||||
|
||||
class TestValidate:
|
||||
"""Tests for full skill directory validation."""
|
||||
|
||||
def test_validate_valid_skill(self):
|
||||
"""Test validating a valid skill directory."""
|
||||
from praisonaiagents.skills.validator import validate
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
skill_dir = Path(tmpdir) / "test-skill"
|
||||
skill_dir.mkdir()
|
||||
(skill_dir / "SKILL.md").write_text("""---
|
||||
name: test-skill
|
||||
description: A test skill for validation
|
||||
---
|
||||
|
||||
# Test Skill
|
||||
""")
|
||||
|
||||
errors = validate(skill_dir)
|
||||
|
||||
assert errors == []
|
||||
|
||||
def test_validate_missing_skill_md(self):
|
||||
"""Test error when SKILL.md is missing."""
|
||||
from praisonaiagents.skills.validator import validate
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
skill_dir = Path(tmpdir) / "empty-skill"
|
||||
skill_dir.mkdir()
|
||||
|
||||
errors = validate(skill_dir)
|
||||
|
||||
assert len(errors) > 0
|
||||
assert any("skill.md" in e.lower() for e in errors)
|
||||
|
||||
def test_validate_nonexistent_directory(self):
|
||||
"""Test error when directory doesn't exist."""
|
||||
from praisonaiagents.skills.validator import validate
|
||||
|
||||
errors = validate(Path("/nonexistent/path"))
|
||||
|
||||
assert len(errors) > 0
|
||||
assert any("exist" in e.lower() for e in errors)
|
||||
|
||||
def test_validate_not_a_directory(self):
|
||||
"""Test error when path is not a directory."""
|
||||
from praisonaiagents.skills.validator import validate
|
||||
|
||||
with tempfile.NamedTemporaryFile() as tmpfile:
|
||||
errors = validate(Path(tmpfile.name))
|
||||
|
||||
assert len(errors) > 0
|
||||
assert any("directory" in e.lower() for e in errors)
|
||||
Generated
+1
-1
@@ -2422,7 +2422,7 @@ wheels = [
|
||||
|
||||
[[package]]
|
||||
name = "praisonaiagents"
|
||||
version = "0.1.17"
|
||||
version = "0.1.24"
|
||||
source = { editable = "." }
|
||||
dependencies = [
|
||||
{ name = "aiohttp" },
|
||||
|
||||
+121
-69
@@ -171,12 +171,13 @@ npm install praisonai
|
||||
|
||||
| Feature | Code | Docs |
|
||||
|---------|:----:|:----:|
|
||||
| 🚀 Single Agent | [Example](examples/python/agents/single-agent.py) | [📖](https://docs.praison.ai/agents/single) |
|
||||
| 🤝 Multi Agents | [Example](examples/python/general/mini_agents_example.py) | [📖](https://docs.praison.ai/concepts/agents) |
|
||||
| 🤖 Auto Agents | [Example](examples/python/general/auto_agents_example.py) | [📖](https://docs.praison.ai/features/autoagents) |
|
||||
| 🔄 Self Reflection AI Agents | [Example](examples/python/concepts/self-reflection-details.py) | [📖](https://docs.praison.ai/features/selfreflection) |
|
||||
| 🧠 Reasoning AI Agents | [Example](examples/python/concepts/reasoning-extraction.py) | [📖](https://docs.praison.ai/features/reasoning) |
|
||||
| 👁️ Multi Modal AI Agents | [Example](examples/python/general/multimodal.py) | [📖](https://docs.praison.ai/features/multimodal) |
|
||||
| **🤖 Core Agents** | | |
|
||||
| ↳ Single Agent | [Example](examples/python/agents/single-agent.py) | [📖](https://docs.praison.ai/agents/single) |
|
||||
| ↳ Multi Agents | [Example](examples/python/general/mini_agents_example.py) | [📖](https://docs.praison.ai/concepts/agents) |
|
||||
| ↳ Auto Agents | [Example](examples/python/general/auto_agents_example.py) | [📖](https://docs.praison.ai/features/autoagents) |
|
||||
| ↳ Self Reflection AI Agents | [Example](examples/python/concepts/self-reflection-details.py) | [📖](https://docs.praison.ai/features/selfreflection) |
|
||||
| ↳ Reasoning AI Agents | [Example](examples/python/concepts/reasoning-extraction.py) | [📖](https://docs.praison.ai/features/reasoning) |
|
||||
| ↳ Multi Modal AI Agents | [Example](examples/python/general/multimodal.py) | [📖](https://docs.praison.ai/features/multimodal) |
|
||||
| **🔄 Workflows** | | |
|
||||
| ↳ Simple Workflow | [Example](examples/python/workflows/simple_workflow.py) | [📖](https://docs.praison.ai/features/workflows) |
|
||||
| ↳ Workflow with Agents | [Example](examples/python/workflows/workflow_with_agents.py) | [📖](https://docs.praison.ai/features/workflows) |
|
||||
@@ -188,10 +189,7 @@ npm install praisonai
|
||||
| ↳ Workflow Branching | [Example](examples/python/workflows/workflow_branching.py) | [📖](https://docs.praison.ai/features/workflows) |
|
||||
| ↳ Workflow Early Stop | [Example](examples/python/workflows/workflow_early_stop.py) | [📖](https://docs.praison.ai/features/workflows) |
|
||||
| ↳ Workflow Checkpoints | [Example](examples/python/workflows/workflow_checkpoints.py) | [📖](https://docs.praison.ai/features/workflows) |
|
||||
| 📚 Add Custom Knowledge | [Example](examples/python/concepts/knowledge-agents.py) | [📖](https://docs.praison.ai/features/knowledge) |
|
||||
| 🧠 Memory (Short & Long Term) | [Example](examples/python/general/memory_example.py) | [📖](https://docs.praison.ai/concepts/memory) |
|
||||
| 📄 Chat with PDF Agents | [Example](examples/python/concepts/chat-with-pdf.py) | [📖](https://docs.praison.ai/features/chat-with-pdf) |
|
||||
| **💻 Code** | | |
|
||||
| **💻 Code & Development** | | |
|
||||
| ↳ Code Interpreter Agents | [Example](examples/python/agents/code-agent.py) | [📖](https://docs.praison.ai/features/codeagent) |
|
||||
| ↳ AI Code Editing Tools | [Example](examples/python/code/code_editing_example.py) | [📖](https://docs.praison.ai/code/editing) |
|
||||
| ↳ External Agents (All) | [Example](examples/python/code/external_agents_example.py) | [📖](https://docs.praison.ai/code/external-agents) |
|
||||
@@ -199,65 +197,78 @@ npm install praisonai
|
||||
| ↳ Gemini CLI | [Example](examples/python/code/gemini_cli_example.py) | [📖](https://docs.praison.ai/code/gemini-cli) |
|
||||
| ↳ Codex CLI | [Example](examples/python/code/codex_cli_example.py) | [📖](https://docs.praison.ai/code/codex-cli) |
|
||||
| ↳ Cursor CLI | [Example](examples/python/code/cursor_cli_example.py) | [📖](https://docs.praison.ai/code/cursor-cli) |
|
||||
| 📚 RAG Agents | [Example](examples/python/concepts/rag-agents.py) | [📖](https://docs.praison.ai/features/rag) |
|
||||
| 🤔 Async & Parallel Processing | [Example](examples/python/general/async_example.py) | [📖](https://docs.praison.ai/features/async) |
|
||||
| 🔢 Math Agents | [Example](examples/python/agents/math-agent.py) | [📖](https://docs.praison.ai/features/mathagent) |
|
||||
| 🎯 Structured Output Agents | [Example](examples/python/general/structured_agents_example.py) | [📖](https://docs.praison.ai/features/structured) |
|
||||
| 🔗 LangChain Integrated Agents | [Example](examples/python/general/langchain_example.py) | [📖](https://docs.praison.ai/features/langchain) |
|
||||
| 📞 Callback Agents | [Example](examples/python/general/advanced-callback-systems.py) | [📖](https://docs.praison.ai/features/callbacks) |
|
||||
| 🛠️ 100+ Custom Tools | [Example](examples/python/general/tools_example.py) | [📖](https://docs.praison.ai/tools/tools) |
|
||||
| 📄 YAML Configuration | [Example](examples/cookbooks/yaml/secondary_market_research_agents.yaml) | [📖](https://docs.praison.ai/developers/agents-playbook) |
|
||||
| 💯 100+ LLM Support | [Example](examples/python/providers/openai/openai_gpt4_example.py) | [📖](https://docs.praison.ai/models) |
|
||||
| 🔬 Deep Research Agents | [Example](examples/python/agents/research-agent.py) | [📖](https://docs.praison.ai/agents/deep-research) |
|
||||
| 🔄 Query Rewriter Agent | [Example](#5-query-rewriter-agent) | [📖](https://docs.praison.ai/agents/query-rewriter) |
|
||||
| 🌐 Native Web Search | [Example](examples/python/agents/websearch-agent.py) | [📖](https://docs.praison.ai/agents/websearch) |
|
||||
| 📥 Web Fetch (Anthropic) | [Example](#web-search-web-fetch--prompt-caching) | [📖](https://docs.praison.ai/features/model-capabilities) |
|
||||
| 💾 Prompt Caching | [Example](#web-search-web-fetch--prompt-caching) | [📖](https://docs.praison.ai/features/model-capabilities) |
|
||||
| 🧠 Claude Memory Tool | [Example](#claude-memory-tool-cli) | [📖](https://docs.praison.ai/features/claude-memory-tool) |
|
||||
| 💾 File-Based Memory | [Example](examples/python/general/memory_example.py) | [📖](https://docs.praison.ai/concepts/memory) |
|
||||
| 🔍 Built-in Search Tools | [Example](examples/python/agents/websearch-agent.py) | [📖](https://docs.praison.ai/tools/tavily) |
|
||||
| 🔎 Unified Web Search | [Example](src/praisonai-agents/examples/web_search_example.py) | [📖](https://docs.praison.ai/tools/web-search) |
|
||||
| 📋 Planning Mode | [Example](examples/python/agents/planning-agent.py) | [📖](https://docs.praison.ai/features/planning-mode) |
|
||||
| 🔧 Planning Tools | [Example](#3-agent-with-planning-mode) | [📖](https://docs.praison.ai/features/planning-mode) |
|
||||
| 🧠 Planning Reasoning | [Example](#3-agent-with-planning-mode) | [📖](https://docs.praison.ai/features/planning-mode) |
|
||||
| 🔌 MCP Transports | [Example](examples/python/mcp/mcp-transports-overview.py) | [📖](https://docs.praison.ai/mcp/transports) |
|
||||
| 🌐 WebSocket MCP | [Example](examples/python/mcp/websocket-mcp.py) | [📖](https://docs.praison.ai/mcp/sse-transport) |
|
||||
| 🔐 MCP Security | [Example](examples/python/mcp/mcp-security.py) | [📖](https://docs.praison.ai/mcp/transports) |
|
||||
| 🔄 MCP Resumability | [Example](examples/python/mcp/mcp-resumability.py) | [📖](https://docs.praison.ai/mcp/sse-transport) |
|
||||
| ⚡ Fast Context | [Example](examples/context/00_agent_fast_context_basic.py) | [📖](https://docs.praison.ai/features/fast-context) |
|
||||
| 🖼️ Image Generation Agent | [Example](examples/python/image/image-agent.py) | [📖](https://docs.praison.ai/features/image-generation) |
|
||||
| 📷 Image to Text Agent | [Example](examples/python/agents/image-to-text-agent.py) | [📖](https://docs.praison.ai/agents/image-to-text) |
|
||||
| 🎬 Video Agent | [Example](examples/python/agents/video-agent.py) | [📖](https://docs.praison.ai/agents/video) |
|
||||
| 📊 Data Analyst Agent | [Example](examples/python/agents/data-analyst-agent.py) | [📖](https://docs.praison.ai/agents/data-analyst) |
|
||||
| 💰 Finance Agent | [Example](examples/python/agents/finance-agent.py) | [📖](https://docs.praison.ai/agents/finance) |
|
||||
| 🛒 Shopping Agent | [Example](examples/python/agents/shopping-agent.py) | [📖](https://docs.praison.ai/agents/shopping) |
|
||||
| ⭐ Recommendation Agent | [Example](examples/python/agents/recommendation-agent.py) | [📖](https://docs.praison.ai/agents/recommendation) |
|
||||
| 📖 Wikipedia Agent | [Example](examples/python/agents/wikipedia-agent.py) | [📖](https://docs.praison.ai/agents/wikipedia) |
|
||||
| 💻 Programming Agent | [Example](examples/python/agents/programming-agent.py) | [📖](https://docs.praison.ai/agents/programming) |
|
||||
| 📝 Markdown Agent | [Example](examples/python/agents/markdown-agent.py) | [📖](https://docs.praison.ai/agents/markdown) |
|
||||
| 📝 Prompt Expander Agent | [Example](#prompt-expansion) | [📖](https://docs.praison.ai/agents/prompt-expander) |
|
||||
| 🔀 Model Router | [Example](examples/python/agents/router-agent-cost-optimization.py) | [📖](https://docs.praison.ai/features/model-router) |
|
||||
| ⛓️ Prompt Chaining | [Example](examples/python/general/prompt_chaining.py) | [📖](https://docs.praison.ai/features/promptchaining) |
|
||||
| 🔍 Evaluator Optimiser | [Example](examples/python/general/evaluator-optimiser.py) | [📖](https://docs.praison.ai/features/evaluator-optimiser) |
|
||||
| 👷 Orchestrator Workers | [Example](examples/python/general/orchestrator-workers.py) | [📖](https://docs.praison.ai/features/orchestrator-worker) |
|
||||
| ⚡ Parallelisation | [Example](examples/python/general/parallelisation.py) | [📖](https://docs.praison.ai/features/parallelisation) |
|
||||
| 🔁 Repetitive Agents | [Example](examples/python/concepts/repetitive-agents.py) | [📖](https://docs.praison.ai/features/repetitive) |
|
||||
| 🤝 Agent Handoffs | [Example](examples/python/handoff/handoff_basic.py) | [📖](https://docs.praison.ai/features/handoffs) |
|
||||
| 🛡️ Guardrails | [Example](examples/python/guardrails/comprehensive-guardrails-example.py) | [📖](https://docs.praison.ai/features/guardrails) |
|
||||
| 💬 Sessions Management | [Example](examples/python/sessions/comprehensive-session-management.py) | [📖](https://docs.praison.ai/features/sessions) |
|
||||
| ✅ Human Approval | [Example](examples/python/general/human_approval_example.py) | [📖](https://docs.praison.ai/features/approval) |
|
||||
| 🔄 Stateful Agents | [Example](examples/python/stateful/workflow-state-example.py) | [📖](https://docs.praison.ai/features/stateful-agents) |
|
||||
| 🤖 Autonomous Workflow | [Example](examples/python/general/autonomous-agent.py) | [📖](https://docs.praison.ai/features/autonomous-workflow) |
|
||||
| 📜 Rules & Instructions | [Example](#6-rules--instructions) | [📖](https://docs.praison.ai/features/rules) |
|
||||
| 🪝 Hooks | [Example](#9-hooks) | [📖](https://docs.praison.ai/features/hooks) |
|
||||
| 📈 Telemetry | [Example](examples/python/telemetry/production-telemetry-example.py) | [📖](https://docs.praison.ai/features/telemetry) |
|
||||
| 📹 Camera Integration | [Example](examples/python/camera/) | [📖](https://docs.praison.ai/features/camera-integration) |
|
||||
| 📄 Project Docs (.praison/docs/) | [Example](#docs-cli) | [📖](https://docs.praison.ai/docs/cli/docs) |
|
||||
| 🔌 MCP Config Management | [Example](#mcp-config-cli) | [📖](https://docs.praison.ai/docs/cli/mcp) |
|
||||
| 💬 AI Commit Messages | [Example](#ai-commit-cli) | [📖](https://docs.praison.ai/docs/cli/commit) |
|
||||
| @ @Mentions in Prompts | [Example](#mentions-in-prompts) | [📖](https://docs.praison.ai/docs/cli/mentions) |
|
||||
| 💾 Auto-Save Sessions | [Example](#session-management-python) | [📖](https://docs.praison.ai/docs/cli/session) |
|
||||
| 📜 History in Context | [Example](#session-management-python) | [📖](https://docs.praison.ai/docs/cli/session) |
|
||||
| **🧠 Memory & Knowledge** | | |
|
||||
| ↳ Memory (Short & Long Term) | [Example](examples/python/general/memory_example.py) | [📖](https://docs.praison.ai/concepts/memory) |
|
||||
| ↳ File-Based Memory | [Example](examples/python/general/memory_example.py) | [📖](https://docs.praison.ai/concepts/memory) |
|
||||
| ↳ Claude Memory Tool | [Example](#claude-memory-tool-cli) | [📖](https://docs.praison.ai/features/claude-memory-tool) |
|
||||
| ↳ Add Custom Knowledge | [Example](examples/python/concepts/knowledge-agents.py) | [📖](https://docs.praison.ai/features/knowledge) |
|
||||
| ↳ RAG Agents | [Example](examples/python/concepts/rag-agents.py) | [📖](https://docs.praison.ai/features/rag) |
|
||||
| ↳ Chat with PDF Agents | [Example](examples/python/concepts/chat-with-pdf.py) | [📖](https://docs.praison.ai/features/chat-with-pdf) |
|
||||
| **🔬 Research & Intelligence** | | |
|
||||
| ↳ Deep Research Agents | [Example](examples/python/agents/research-agent.py) | [📖](https://docs.praison.ai/agents/deep-research) |
|
||||
| ↳ Query Rewriter Agent | [Example](#5-query-rewriter-agent) | [📖](https://docs.praison.ai/agents/query-rewriter) |
|
||||
| ↳ Native Web Search | [Example](examples/python/agents/websearch-agent.py) | [📖](https://docs.praison.ai/agents/websearch) |
|
||||
| ↳ Built-in Search Tools | [Example](examples/python/agents/websearch-agent.py) | [📖](https://docs.praison.ai/tools/tavily) |
|
||||
| ↳ Unified Web Search | [Example](src/praisonai-agents/examples/web_search_example.py) | [📖](https://docs.praison.ai/tools/web-search) |
|
||||
| ↳ Web Fetch (Anthropic) | [Example](#web-search-web-fetch--prompt-caching) | [📖](https://docs.praison.ai/features/model-capabilities) |
|
||||
| **📋 Planning & Execution** | | |
|
||||
| ↳ Planning Mode | [Example](examples/python/agents/planning-agent.py) | [📖](https://docs.praison.ai/features/planning-mode) |
|
||||
| ↳ Planning Tools | [Example](#3-agent-with-planning-mode) | [📖](https://docs.praison.ai/features/planning-mode) |
|
||||
| ↳ Planning Reasoning | [Example](#3-agent-with-planning-mode) | [📖](https://docs.praison.ai/features/planning-mode) |
|
||||
| ↳ Prompt Chaining | [Example](examples/python/general/prompt_chaining.py) | [📖](https://docs.praison.ai/features/promptchaining) |
|
||||
| ↳ Evaluator Optimiser | [Example](examples/python/general/evaluator-optimiser.py) | [📖](https://docs.praison.ai/features/evaluator-optimiser) |
|
||||
| ↳ Orchestrator Workers | [Example](examples/python/general/orchestrator-workers.py) | [📖](https://docs.praison.ai/features/orchestrator-worker) |
|
||||
| **👥 Specialized Agents** | | |
|
||||
| ↳ Data Analyst Agent | [Example](examples/python/agents/data-analyst-agent.py) | [📖](https://docs.praison.ai/agents/data-analyst) |
|
||||
| ↳ Finance Agent | [Example](examples/python/agents/finance-agent.py) | [📖](https://docs.praison.ai/agents/finance) |
|
||||
| ↳ Shopping Agent | [Example](examples/python/agents/shopping-agent.py) | [📖](https://docs.praison.ai/agents/shopping) |
|
||||
| ↳ Recommendation Agent | [Example](examples/python/agents/recommendation-agent.py) | [📖](https://docs.praison.ai/agents/recommendation) |
|
||||
| ↳ Wikipedia Agent | [Example](examples/python/agents/wikipedia-agent.py) | [📖](https://docs.praison.ai/agents/wikipedia) |
|
||||
| ↳ Programming Agent | [Example](examples/python/agents/programming-agent.py) | [📖](https://docs.praison.ai/agents/programming) |
|
||||
| ↳ Math Agents | [Example](examples/python/agents/math-agent.py) | [📖](https://docs.praison.ai/features/mathagent) |
|
||||
| ↳ Markdown Agent | [Example](examples/python/agents/markdown-agent.py) | [📖](https://docs.praison.ai/agents/markdown) |
|
||||
| ↳ Prompt Expander Agent | [Example](#prompt-expansion) | [📖](https://docs.praison.ai/agents/prompt-expander) |
|
||||
| **🎨 Media & Multimodal** | | |
|
||||
| ↳ Image Generation Agent | [Example](examples/python/image/image-agent.py) | [📖](https://docs.praison.ai/features/image-generation) |
|
||||
| ↳ Image to Text Agent | [Example](examples/python/agents/image-to-text-agent.py) | [📖](https://docs.praison.ai/agents/image-to-text) |
|
||||
| ↳ Video Agent | [Example](examples/python/agents/video-agent.py) | [📖](https://docs.praison.ai/agents/video) |
|
||||
| ↳ Camera Integration | [Example](examples/python/camera/) | [📖](https://docs.praison.ai/features/camera-integration) |
|
||||
| **🔌 Protocols & Integration** | | |
|
||||
| ↳ MCP Transports | [Example](examples/python/mcp/mcp-transports-overview.py) | [📖](https://docs.praison.ai/mcp/transports) |
|
||||
| ↳ WebSocket MCP | [Example](examples/python/mcp/websocket-mcp.py) | [📖](https://docs.praison.ai/mcp/sse-transport) |
|
||||
| ↳ MCP Security | [Example](examples/python/mcp/mcp-security.py) | [📖](https://docs.praison.ai/mcp/transports) |
|
||||
| ↳ MCP Resumability | [Example](examples/python/mcp/mcp-resumability.py) | [📖](https://docs.praison.ai/mcp/sse-transport) |
|
||||
| ↳ MCP Config Management | [Example](#mcp-config-cli) | [📖](https://docs.praison.ai/docs/cli/mcp) |
|
||||
| ↳ LangChain Integrated Agents | [Example](examples/python/general/langchain_example.py) | [📖](https://docs.praison.ai/features/langchain) |
|
||||
| **🛡️ Safety & Control** | | |
|
||||
| ↳ Guardrails | [Example](examples/python/guardrails/comprehensive-guardrails-example.py) | [📖](https://docs.praison.ai/features/guardrails) |
|
||||
| ↳ Human Approval | [Example](examples/python/general/human_approval_example.py) | [📖](https://docs.praison.ai/features/approval) |
|
||||
| ↳ Rules & Instructions | [Example](#6-rules--instructions) | [📖](https://docs.praison.ai/features/rules) |
|
||||
| **⚙️ Advanced Features** | | |
|
||||
| ↳ Async & Parallel Processing | [Example](examples/python/general/async_example.py) | [📖](https://docs.praison.ai/features/async) |
|
||||
| ↳ Parallelisation | [Example](examples/python/general/parallelisation.py) | [📖](https://docs.praison.ai/features/parallelisation) |
|
||||
| ↳ Repetitive Agents | [Example](examples/python/concepts/repetitive-agents.py) | [📖](https://docs.praison.ai/features/repetitive) |
|
||||
| ↳ Agent Handoffs | [Example](examples/python/handoff/handoff_basic.py) | [📖](https://docs.praison.ai/features/handoffs) |
|
||||
| ↳ Stateful Agents | [Example](examples/python/stateful/workflow-state-example.py) | [📖](https://docs.praison.ai/features/stateful-agents) |
|
||||
| ↳ Autonomous Workflow | [Example](examples/python/general/autonomous-agent.py) | [📖](https://docs.praison.ai/features/autonomous-workflow) |
|
||||
| ↳ Structured Output Agents | [Example](examples/python/general/structured_agents_example.py) | [📖](https://docs.praison.ai/features/structured) |
|
||||
| ↳ Model Router | [Example](examples/python/agents/router-agent-cost-optimization.py) | [📖](https://docs.praison.ai/features/model-router) |
|
||||
| ↳ Prompt Caching | [Example](#web-search-web-fetch--prompt-caching) | [📖](https://docs.praison.ai/features/model-capabilities) |
|
||||
| ↳ Fast Context | [Example](examples/context/00_agent_fast_context_basic.py) | [📖](https://docs.praison.ai/features/fast-context) |
|
||||
| **🛠️ Tools & Configuration** | | |
|
||||
| ↳ 100+ Custom Tools | [Example](examples/python/general/tools_example.py) | [📖](https://docs.praison.ai/tools/tools) |
|
||||
| ↳ YAML Configuration | [Example](examples/cookbooks/yaml/secondary_market_research_agents.yaml) | [📖](https://docs.praison.ai/developers/agents-playbook) |
|
||||
| ↳ 100+ LLM Support | [Example](examples/python/providers/openai/openai_gpt4_example.py) | [📖](https://docs.praison.ai/models) |
|
||||
| ↳ Callback Agents | [Example](examples/python/general/advanced-callback-systems.py) | [📖](https://docs.praison.ai/features/callbacks) |
|
||||
| ↳ Hooks | [Example](#9-hooks) | [📖](https://docs.praison.ai/features/hooks) |
|
||||
| **📊 Monitoring & Management** | | |
|
||||
| ↳ Sessions Management | [Example](examples/python/sessions/comprehensive-session-management.py) | [📖](https://docs.praison.ai/features/sessions) |
|
||||
| ↳ Auto-Save Sessions | [Example](#session-management-python) | [📖](https://docs.praison.ai/docs/cli/session) |
|
||||
| ↳ History in Context | [Example](#session-management-python) | [📖](https://docs.praison.ai/docs/cli/session) |
|
||||
| ↳ Telemetry | [Example](examples/python/telemetry/production-telemetry-example.py) | [📖](https://docs.praison.ai/features/telemetry) |
|
||||
| ↳ Project Docs (.praison/docs/) | [Example](#docs-cli) | [📖](https://docs.praison.ai/docs/cli/docs) |
|
||||
| ↳ AI Commit Messages | [Example](#ai-commit-cli) | [📖](https://docs.praison.ai/docs/cli/commit) |
|
||||
| ↳ @Mentions in Prompts | [Example](#mentions-in-prompts) | [📖](https://docs.praison.ai/docs/cli/mentions) |
|
||||
| **🖥️ CLI Features** | | |
|
||||
| ↳ Slash Commands | [Example](examples/python/cli/slash_commands_example.py) | [📖](https://docs.praison.ai/docs/cli/slash-commands) |
|
||||
| ↳ Autonomy Modes | [Example](examples/python/cli/autonomy_modes_example.py) | [📖](https://docs.praison.ai/docs/cli/autonomy-modes) |
|
||||
@@ -266,6 +277,17 @@ npm install praisonai
|
||||
| ↳ Interactive TUI | [Example](examples/python/cli/interactive_tui_example.py) | [📖](https://docs.praison.ai/docs/cli/interactive-tui) |
|
||||
| ↳ Git Integration | [Example](examples/python/cli/git_integration_example.py) | [📖](https://docs.praison.ai/docs/cli/git-integration) |
|
||||
| ↳ Sandbox Execution | [Example](examples/python/cli/sandbox_execution_example.py) | [📖](https://docs.praison.ai/docs/cli/sandbox-execution) |
|
||||
| ↳ CLI Compare | [Example](examples/compare/cli_compare_basic.py) | [📖](https://docs.praison.ai/docs/cli/compare) |
|
||||
| **🧪 Evaluation** | | |
|
||||
| ↳ Accuracy Evaluation | [Example](examples/eval/accuracy_example.py) | [📖](https://docs.praison.ai/docs/cli/eval) |
|
||||
| ↳ Performance Evaluation | [Example](examples/eval/performance_example.py) | [📖](https://docs.praison.ai/docs/cli/eval) |
|
||||
| ↳ Reliability Evaluation | [Example](examples/eval/reliability_example.py) | [📖](https://docs.praison.ai/docs/cli/eval) |
|
||||
| ↳ Criteria Evaluation | [Example](examples/eval/criteria_example.py) | [📖](https://docs.praison.ai/docs/cli/eval) |
|
||||
| **🎯 Agent Skills** | | |
|
||||
| ↳ Skills Management | [Example](examples/skills/basic_skill_usage.py) | [📖](https://docs.praison.ai/features/skills) |
|
||||
| ↳ Custom Skills | [Example](examples/skills/custom_skill_example.py) | [📖](https://docs.praison.ai/features/skills) |
|
||||
| **⏰ 24/7 Scheduling** | | |
|
||||
| ↳ Agent Scheduler | [Example](examples/python/scheduled_agents/news_checker_live.py) | [📖](https://docs.praison.ai/docs/cli/scheduler) |
|
||||
|
||||
---
|
||||
|
||||
@@ -1058,6 +1080,20 @@ praisonai "Complex analysis task" --planning --planning-reasoning
|
||||
praisonai "Task" --planning --auto-approve-plan
|
||||
```
|
||||
|
||||
### Tool Approval CLI:
|
||||
```bash
|
||||
# Auto-approve ALL tool executions (use with caution!)
|
||||
praisonai "run ls command" --trust
|
||||
|
||||
# Auto-approve tools up to a risk level (prompt for higher)
|
||||
# Levels: low, medium, high, critical
|
||||
praisonai "write to file" --approve-level high # Prompts for critical tools only
|
||||
praisonai "task" --approve-level medium # Prompts for high and critical
|
||||
|
||||
# Default behavior (no flags): prompts for all dangerous tools
|
||||
praisonai "run shell command" # Will prompt for approval
|
||||
```
|
||||
|
||||
### Memory CLI:
|
||||
```bash
|
||||
# Enable memory for agent (persists across sessions)
|
||||
@@ -1228,6 +1264,21 @@ praisonai "Analyze this data" --metrics
|
||||
praisonai "Complex task" --metrics --planning
|
||||
```
|
||||
|
||||
### Scheduler CLI:
|
||||
|
||||
```bash
|
||||
praisonai schedule start <name> "task" --interval hourly
|
||||
praisonai schedule list
|
||||
praisonai schedule logs <name> [--follow]
|
||||
praisonai schedule stop <name>
|
||||
praisonai schedule restart <name>
|
||||
praisonai schedule delete <name>
|
||||
praisonai schedule describe <name>
|
||||
praisonai schedule save <name> [file.yaml]
|
||||
praisonai schedule "task" --interval hourly # foreground mode
|
||||
praisonai schedule agents.yaml # foreground mode
|
||||
```
|
||||
|
||||
### Image Processing CLI:
|
||||
```bash
|
||||
# Process images with vision-based tasks
|
||||
@@ -2399,6 +2450,7 @@ PraisonAI provides zero-dependency persistent memory for agents. For detailed ex
|
||||
- 🤝 **Agent Handoffs** - Transfer context between specialised agents
|
||||
- 🛡️ **Guardrails** - Input/output validation and safety checks
|
||||
- ✅ **Human Approval** - Require human confirmation for critical actions
|
||||
- 🔐 **Tool Approval CLI** - `--trust` (auto-approve all) and `--approve-level` (risk-based approval)
|
||||
- 💬 **Sessions Management** - Isolated conversation contexts
|
||||
- 🔄 **Stateful Agents** - Maintain state across interactions
|
||||
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
# PraisonAI CLI - Benchmark Results
|
||||
|
||||
**Generated:** 2025-12-19 16:04:40
|
||||
|
||||
## Import Times
|
||||
|
||||
| Module | Time (ms) | Target | Status |
|
||||
|--------|-----------|--------|--------|
|
||||
| praisonai | 4.11 | <10 | ✅ |
|
||||
| message_queue | 6.87 | <50 | ✅ |
|
||||
| at_mentions | 0.58 | <50 | ✅ |
|
||||
| profiler | 0.90 | <10 | ✅ |
|
||||
|
||||
## Instantiation Times
|
||||
|
||||
| Class | Time (μs) | Target | Status |
|
||||
|-------|-----------|--------|--------|
|
||||
| MessageQueue | 0.26 | <100 | ✅ |
|
||||
| StateManager | 0.30 | <100 | ✅ |
|
||||
| FileSearchService | 8.42 | <100 | ✅ |
|
||||
|
||||
## Lazy Loading
|
||||
|
||||
| Operation | Time (ms) |
|
||||
|-----------|----------|
|
||||
| PraisonAI lazy load | 3886.47 |
|
||||
|
||||
## How to Reproduce
|
||||
|
||||
```bash
|
||||
cd praisonai
|
||||
python benchmarks/simple_benchmark.py
|
||||
```
|
||||
@@ -0,0 +1,231 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
PraisonAI CLI - Performance Benchmark
|
||||
|
||||
Measures import times and module loading performance.
|
||||
|
||||
Usage:
|
||||
python benchmarks/simple_benchmark.py
|
||||
python benchmarks/simple_benchmark.py --no-save
|
||||
"""
|
||||
|
||||
import time
|
||||
import sys
|
||||
import os
|
||||
import argparse
|
||||
from datetime import datetime
|
||||
|
||||
|
||||
ITERATIONS = 100
|
||||
|
||||
|
||||
def measure_import(module_name: str) -> float:
|
||||
"""Measure import time in milliseconds."""
|
||||
# Clear from cache if present
|
||||
modules_to_remove = [k for k in sys.modules.keys() if k.startswith(module_name)]
|
||||
for mod in modules_to_remove:
|
||||
del sys.modules[mod]
|
||||
|
||||
start = time.perf_counter()
|
||||
__import__(module_name)
|
||||
return (time.perf_counter() - start) * 1000
|
||||
|
||||
|
||||
def measure_instantiation(create_fn, iterations=ITERATIONS) -> float:
|
||||
"""Measure average instantiation time in microseconds."""
|
||||
times = []
|
||||
for _ in range(iterations):
|
||||
start = time.perf_counter()
|
||||
create_fn()
|
||||
times.append((time.perf_counter() - start) * 1_000_000)
|
||||
return sum(times) / len(times)
|
||||
|
||||
|
||||
def run_benchmark():
|
||||
"""Run the benchmark."""
|
||||
results = {}
|
||||
|
||||
print('=' * 60)
|
||||
print('PraisonAI CLI - Performance Benchmark')
|
||||
print('=' * 60)
|
||||
print(f'\nIterations: {ITERATIONS}')
|
||||
print('Metrics: Import time (ms), Instantiation time (μs)\n')
|
||||
|
||||
# 1. Package import time
|
||||
print("1. Testing package imports...")
|
||||
|
||||
# Clear cache
|
||||
modules_to_remove = [k for k in sys.modules.keys() if 'praisonai' in k]
|
||||
for mod in modules_to_remove:
|
||||
del sys.modules[mod]
|
||||
|
||||
start = time.perf_counter()
|
||||
import praisonai
|
||||
results['praisonai_import_ms'] = (time.perf_counter() - start) * 1000
|
||||
print(f" import praisonai: {results['praisonai_import_ms']:.2f}ms")
|
||||
|
||||
# 2. CLI features import
|
||||
print("\n2. Testing CLI features imports...")
|
||||
|
||||
start = time.perf_counter()
|
||||
from praisonai.cli.features.message_queue import MessageQueue
|
||||
results['message_queue_import_ms'] = (time.perf_counter() - start) * 1000
|
||||
print(f" message_queue: {results['message_queue_import_ms']:.2f}ms")
|
||||
|
||||
start = time.perf_counter()
|
||||
from praisonai.cli.features.at_mentions import CombinedCompleter
|
||||
results['at_mentions_import_ms'] = (time.perf_counter() - start) * 1000
|
||||
print(f" at_mentions: {results['at_mentions_import_ms']:.2f}ms")
|
||||
|
||||
# 3. Profiler import (should be fast, standalone)
|
||||
print("\n3. Testing profiler import...")
|
||||
start = time.perf_counter()
|
||||
from praisonai.profiler import Profiler
|
||||
results['profiler_import_ms'] = (time.perf_counter() - start) * 1000
|
||||
print(f" profiler: {results['profiler_import_ms']:.2f}ms")
|
||||
|
||||
# 4. Instantiation benchmarks
|
||||
print(f"\n4. Testing instantiation ({ITERATIONS} iterations)...")
|
||||
|
||||
from praisonai.cli.features.message_queue import MessageQueue, StateManager
|
||||
from praisonai.cli.features.at_mentions import FileSearchService
|
||||
|
||||
results['MessageQueue_us'] = measure_instantiation(lambda: MessageQueue())
|
||||
print(f" MessageQueue: {results['MessageQueue_us']:.2f}μs")
|
||||
|
||||
results['StateManager_us'] = measure_instantiation(lambda: StateManager())
|
||||
print(f" StateManager: {results['StateManager_us']:.2f}μs")
|
||||
|
||||
results['FileSearchService_us'] = measure_instantiation(
|
||||
lambda: FileSearchService(root_dir=".")
|
||||
)
|
||||
print(f" FileSearchService: {results['FileSearchService_us']:.2f}μs")
|
||||
|
||||
# 5. PraisonAI class (lazy load)
|
||||
print("\n5. Testing PraisonAI class access (lazy load)...")
|
||||
start = time.perf_counter()
|
||||
PraisonAI = praisonai.PraisonAI
|
||||
results['PraisonAI_lazy_ms'] = (time.perf_counter() - start) * 1000
|
||||
print(f" PraisonAI lazy load: {results['PraisonAI_lazy_ms']:.2f}ms")
|
||||
|
||||
return results
|
||||
|
||||
|
||||
def print_summary(results: dict):
|
||||
"""Print benchmark summary."""
|
||||
print('\n' + '=' * 60)
|
||||
print('SUMMARY')
|
||||
print('=' * 60)
|
||||
|
||||
print('\nImport Times (target: <50ms for features, <10ms for package):')
|
||||
print(f" {'Module':<25} {'Time':>10} {'Status':>10}")
|
||||
print(f" {'-'*25} {'-'*10} {'-'*10}")
|
||||
|
||||
import_checks = [
|
||||
('praisonai', results['praisonai_import_ms'], 10),
|
||||
('message_queue', results['message_queue_import_ms'], 50),
|
||||
('at_mentions', results['at_mentions_import_ms'], 50),
|
||||
('profiler', results['profiler_import_ms'], 10),
|
||||
]
|
||||
|
||||
all_pass = True
|
||||
for name, time_ms, threshold in import_checks:
|
||||
status = '✅ PASS' if time_ms < threshold else '❌ SLOW'
|
||||
if time_ms >= threshold:
|
||||
all_pass = False
|
||||
print(f" {name:<25} {time_ms:>8.2f}ms {status:>10}")
|
||||
|
||||
print('\nInstantiation Times (target: <100μs):')
|
||||
print(f" {'Class':<25} {'Time':>10} {'Status':>10}")
|
||||
print(f" {'-'*25} {'-'*10} {'-'*10}")
|
||||
|
||||
inst_checks = [
|
||||
('MessageQueue', results['MessageQueue_us'], 100),
|
||||
('StateManager', results['StateManager_us'], 100),
|
||||
('FileSearchService', results['FileSearchService_us'], 100),
|
||||
]
|
||||
|
||||
for name, time_us, threshold in inst_checks:
|
||||
status = '✅ PASS' if time_us < threshold else '❌ SLOW'
|
||||
if time_us >= threshold:
|
||||
all_pass = False
|
||||
print(f" {name:<25} {time_us:>8.2f}μs {status:>10}")
|
||||
|
||||
print('\n' + '=' * 60)
|
||||
if all_pass:
|
||||
print('✅ All benchmarks PASSED')
|
||||
else:
|
||||
print('❌ Some benchmarks FAILED')
|
||||
print('=' * 60)
|
||||
|
||||
return all_pass
|
||||
|
||||
|
||||
def save_results(results: dict):
|
||||
"""Save benchmark results to markdown file."""
|
||||
filepath = os.path.join(os.path.dirname(__file__), 'BENCHMARK_RESULTS.md')
|
||||
|
||||
with open(filepath, 'w') as f:
|
||||
f.write('# PraisonAI CLI - Benchmark Results\n\n')
|
||||
f.write(f'**Generated:** {datetime.now().strftime("%Y-%m-%d %H:%M:%S")}\n\n')
|
||||
|
||||
f.write('## Import Times\n\n')
|
||||
f.write('| Module | Time (ms) | Target | Status |\n')
|
||||
f.write('|--------|-----------|--------|--------|\n')
|
||||
|
||||
import_data = [
|
||||
('praisonai', results['praisonai_import_ms'], 10),
|
||||
('message_queue', results['message_queue_import_ms'], 50),
|
||||
('at_mentions', results['at_mentions_import_ms'], 50),
|
||||
('profiler', results['profiler_import_ms'], 10),
|
||||
]
|
||||
|
||||
for name, time_ms, threshold in import_data:
|
||||
status = '✅' if time_ms < threshold else '❌'
|
||||
f.write(f'| {name} | {time_ms:.2f} | <{threshold} | {status} |\n')
|
||||
|
||||
f.write('\n## Instantiation Times\n\n')
|
||||
f.write('| Class | Time (μs) | Target | Status |\n')
|
||||
f.write('|-------|-----------|--------|--------|\n')
|
||||
|
||||
inst_data = [
|
||||
('MessageQueue', results['MessageQueue_us'], 100),
|
||||
('StateManager', results['StateManager_us'], 100),
|
||||
('FileSearchService', results['FileSearchService_us'], 100),
|
||||
]
|
||||
|
||||
for name, time_us, threshold in inst_data:
|
||||
status = '✅' if time_us < threshold else '❌'
|
||||
f.write(f'| {name} | {time_us:.2f} | <{threshold} | {status} |\n')
|
||||
|
||||
f.write('\n## Lazy Loading\n\n')
|
||||
f.write(f'| Operation | Time (ms) |\n')
|
||||
f.write(f'|-----------|----------|\n')
|
||||
f.write(f'| PraisonAI lazy load | {results["PraisonAI_lazy_ms"]:.2f} |\n')
|
||||
|
||||
f.write('\n## How to Reproduce\n\n')
|
||||
f.write('```bash\n')
|
||||
f.write('cd praisonai\n')
|
||||
f.write('python benchmarks/simple_benchmark.py\n')
|
||||
f.write('```\n')
|
||||
|
||||
print(f'\nResults saved to: {filepath}')
|
||||
return filepath
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description='PraisonAI CLI Performance Benchmark')
|
||||
parser.add_argument('--no-save', action='store_true', help='Do not save results to file')
|
||||
args = parser.parse_args()
|
||||
|
||||
results = run_benchmark()
|
||||
all_pass = print_summary(results)
|
||||
|
||||
if not args.no_save:
|
||||
save_results(results)
|
||||
|
||||
return 0 if all_pass else 1
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
@@ -3,8 +3,8 @@ class Praisonai < Formula
|
||||
|
||||
desc "AI tools for various AI applications"
|
||||
homepage "https://github.com/MervinPraison/PraisonAI"
|
||||
url "https://github.com/MervinPraison/PraisonAI/archive/refs/tags/v2.3.52.tar.gz"
|
||||
sha256 `curl -sL https://github.com/MervinPraison/PraisonAI/archive/refs/tags/v2.3.52.tar.gz | shasum -a 256`.split.first
|
||||
url "https://github.com/MervinPraison/PraisonAI/archive/refs/tags/v2.3.77.tar.gz"
|
||||
sha256 `curl -sL https://github.com/MervinPraison/PraisonAI/archive/refs/tags/v2.3.77.tar.gz | shasum -a 256`.split.first
|
||||
license "MIT"
|
||||
|
||||
depends_on "[email protected]"
|
||||
|
||||
@@ -6,30 +6,29 @@ logging.getLogger('crewai.cli.config').setLevel(logging.ERROR)
|
||||
import os
|
||||
os.environ["OTEL_SDK_DISABLED"] = "true"
|
||||
os.environ["EC_TELEMETRY"] = "false"
|
||||
from .cli import PraisonAI
|
||||
|
||||
# Version is lightweight, import directly
|
||||
from .version import __version__
|
||||
|
||||
# Re-export all classes from praisonaiagents to enable:
|
||||
# from praisonai import Agent, Task, PraisonAIAgents
|
||||
try:
|
||||
import praisonaiagents
|
||||
# Import all symbols from praisonaiagents using * import
|
||||
from praisonaiagents import *
|
||||
except ImportError:
|
||||
# If praisonaiagents is not available, these imports will fail gracefully
|
||||
pass
|
||||
|
||||
# Define __all__ to include both PraisonAI core classes and praisonaiagents exports
|
||||
# Define __all__ for lazy loading
|
||||
__all__ = [
|
||||
# Core PraisonAI classes
|
||||
'PraisonAI',
|
||||
'__version__',
|
||||
]
|
||||
|
||||
# Dynamically extend __all__ with praisonaiagents exports
|
||||
try:
|
||||
import praisonaiagents
|
||||
__all__.extend(praisonaiagents.__all__)
|
||||
except (ImportError, AttributeError):
|
||||
# If praisonaiagents is not available or doesn't have __all__, fail gracefully
|
||||
pass
|
||||
# Lazy loading for heavy imports
|
||||
def __getattr__(name):
|
||||
"""Lazy load heavy modules to improve import time."""
|
||||
if name == 'PraisonAI':
|
||||
from .cli import PraisonAI
|
||||
return PraisonAI
|
||||
|
||||
# Try praisonaiagents exports
|
||||
try:
|
||||
import praisonaiagents
|
||||
if hasattr(praisonaiagents, name):
|
||||
return getattr(praisonaiagents, name)
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
||||
@@ -0,0 +1,337 @@
|
||||
"""
|
||||
Agent Scheduler for PraisonAI - Run agents periodically 24/7.
|
||||
|
||||
This module provides scheduling capabilities for running PraisonAI agents
|
||||
at regular intervals, enabling 24/7 autonomous agent operations.
|
||||
|
||||
Example:
|
||||
# Run news checker every hour
|
||||
from praisonai.agent_scheduler import AgentScheduler
|
||||
from praisonai_agents import Agent
|
||||
|
||||
agent = Agent(
|
||||
name="NewsChecker",
|
||||
instructions="Check latest AI news and summarize",
|
||||
tools=[search_tool]
|
||||
)
|
||||
|
||||
scheduler = AgentScheduler(agent, task="Check latest AI news")
|
||||
scheduler.start(schedule_expr="hourly")
|
||||
"""
|
||||
|
||||
import threading
|
||||
import time
|
||||
import logging
|
||||
from datetime import datetime
|
||||
from typing import Optional, Dict, Any, Callable
|
||||
from abc import ABC, abstractmethod
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ScheduleParser:
|
||||
"""Parse schedule expressions into intervals."""
|
||||
|
||||
@staticmethod
|
||||
def parse(schedule_expr: str) -> int:
|
||||
"""
|
||||
Parse schedule expression and return interval in seconds.
|
||||
|
||||
Supported formats:
|
||||
- "daily" -> 86400 seconds
|
||||
- "hourly" -> 3600 seconds
|
||||
- "*/30m" -> 1800 seconds (every 30 minutes)
|
||||
- "*/1h" -> 3600 seconds (every 1 hour)
|
||||
- "60" -> 60 seconds (plain number)
|
||||
|
||||
Args:
|
||||
schedule_expr: Schedule expression string
|
||||
|
||||
Returns:
|
||||
Interval in seconds
|
||||
|
||||
Raises:
|
||||
ValueError: If schedule format is not supported
|
||||
"""
|
||||
schedule_expr = schedule_expr.strip().lower()
|
||||
|
||||
if schedule_expr == "daily":
|
||||
return 86400
|
||||
elif schedule_expr == "hourly":
|
||||
return 3600
|
||||
elif schedule_expr.isdigit():
|
||||
return int(schedule_expr)
|
||||
elif schedule_expr.startswith("*/"):
|
||||
interval_part = schedule_expr[2:]
|
||||
if interval_part.endswith("m"):
|
||||
minutes = int(interval_part[:-1])
|
||||
return minutes * 60
|
||||
elif interval_part.endswith("h"):
|
||||
hours = int(interval_part[:-1])
|
||||
return hours * 3600
|
||||
elif interval_part.endswith("s"):
|
||||
return int(interval_part[:-1])
|
||||
else:
|
||||
return int(interval_part)
|
||||
else:
|
||||
raise ValueError(f"Unsupported schedule format: {schedule_expr}")
|
||||
|
||||
|
||||
class AgentExecutorInterface(ABC):
|
||||
"""Abstract interface for agent execution."""
|
||||
|
||||
@abstractmethod
|
||||
def execute(self, task: str) -> Any:
|
||||
"""Execute the agent with given task."""
|
||||
pass
|
||||
|
||||
|
||||
class PraisonAgentExecutor(AgentExecutorInterface):
|
||||
"""Executor for PraisonAI agents."""
|
||||
|
||||
def __init__(self, agent):
|
||||
"""
|
||||
Initialize executor with a PraisonAI agent.
|
||||
|
||||
Args:
|
||||
agent: PraisonAI Agent instance
|
||||
"""
|
||||
self.agent = agent
|
||||
|
||||
def execute(self, task: str) -> Any:
|
||||
"""
|
||||
Execute the agent with given task.
|
||||
|
||||
Args:
|
||||
task: Task description for the agent
|
||||
|
||||
Returns:
|
||||
Agent execution result
|
||||
"""
|
||||
try:
|
||||
result = self.agent.start(task)
|
||||
return result
|
||||
except Exception as e:
|
||||
logger.error(f"Agent execution failed: {e}")
|
||||
raise
|
||||
|
||||
|
||||
class AgentScheduler:
|
||||
"""
|
||||
Scheduler for running PraisonAI agents periodically.
|
||||
|
||||
Features:
|
||||
- Interval-based scheduling (hourly, daily, custom)
|
||||
- Thread-safe operation
|
||||
- Automatic retry on failure
|
||||
- Execution logging and monitoring
|
||||
- Graceful shutdown
|
||||
|
||||
Example:
|
||||
scheduler = AgentScheduler(agent, task="Check news")
|
||||
scheduler.start(schedule_expr="hourly", max_retries=3)
|
||||
# Agent runs every hour automatically
|
||||
scheduler.stop() # Stop when needed
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
agent,
|
||||
task: str,
|
||||
config: Optional[Dict[str, Any]] = None,
|
||||
on_success: Optional[Callable] = None,
|
||||
on_failure: Optional[Callable] = None
|
||||
):
|
||||
"""
|
||||
Initialize agent scheduler.
|
||||
|
||||
Args:
|
||||
agent: PraisonAI Agent instance
|
||||
task: Task description to execute
|
||||
config: Optional configuration dict
|
||||
on_success: Callback function on successful execution
|
||||
on_failure: Callback function on failed execution
|
||||
"""
|
||||
self.agent = agent
|
||||
self.task = task
|
||||
self.config = config or {}
|
||||
self.on_success = on_success
|
||||
self.on_failure = on_failure
|
||||
|
||||
self.is_running = False
|
||||
self._stop_event = threading.Event()
|
||||
self._thread = None
|
||||
self._executor = PraisonAgentExecutor(agent)
|
||||
self._execution_count = 0
|
||||
self._success_count = 0
|
||||
self._failure_count = 0
|
||||
|
||||
def start(
|
||||
self,
|
||||
schedule_expr: str,
|
||||
max_retries: int = 3,
|
||||
run_immediately: bool = False
|
||||
) -> bool:
|
||||
"""
|
||||
Start scheduled agent execution.
|
||||
|
||||
Args:
|
||||
schedule_expr: Schedule expression (e.g., "hourly", "*/1h", "3600")
|
||||
max_retries: Maximum retry attempts on failure
|
||||
run_immediately: If True, run agent immediately before starting schedule
|
||||
|
||||
Returns:
|
||||
True if scheduler started successfully
|
||||
"""
|
||||
if self.is_running:
|
||||
logger.warning("Scheduler is already running")
|
||||
return False
|
||||
|
||||
try:
|
||||
interval = ScheduleParser.parse(schedule_expr)
|
||||
self.is_running = True
|
||||
self._stop_event.clear()
|
||||
|
||||
logger.info(f"Starting agent scheduler: {self.agent.name if hasattr(self.agent, 'name') else 'Agent'}")
|
||||
logger.info(f"Task: {self.task}")
|
||||
logger.info(f"Schedule: {schedule_expr} ({interval}s interval)")
|
||||
logger.info(f"Max retries: {max_retries}")
|
||||
|
||||
# Run immediately if requested
|
||||
if run_immediately:
|
||||
logger.info("Running agent immediately before starting schedule...")
|
||||
self._execute_with_retry(max_retries)
|
||||
|
||||
self._thread = threading.Thread(
|
||||
target=self._run_schedule,
|
||||
args=(interval, max_retries),
|
||||
daemon=True
|
||||
)
|
||||
self._thread.start()
|
||||
|
||||
logger.info("Agent scheduler started successfully")
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to start scheduler: {e}")
|
||||
self.is_running = False
|
||||
return False
|
||||
|
||||
def stop(self) -> bool:
|
||||
"""
|
||||
Stop the scheduler gracefully.
|
||||
|
||||
Returns:
|
||||
True if stopped successfully
|
||||
"""
|
||||
if not self.is_running:
|
||||
logger.info("Scheduler is not running")
|
||||
return True
|
||||
|
||||
logger.info("Stopping agent scheduler...")
|
||||
self._stop_event.set()
|
||||
|
||||
if self._thread and self._thread.is_alive():
|
||||
self._thread.join(timeout=10)
|
||||
|
||||
self.is_running = False
|
||||
logger.info("Agent scheduler stopped")
|
||||
logger.info(f"Execution stats - Total: {self._execution_count}, Success: {self._success_count}, Failed: {self._failure_count}")
|
||||
return True
|
||||
|
||||
def get_stats(self) -> Dict[str, Any]:
|
||||
"""
|
||||
Get execution statistics.
|
||||
|
||||
Returns:
|
||||
Dictionary with execution stats
|
||||
"""
|
||||
return {
|
||||
"is_running": self.is_running,
|
||||
"total_executions": self._execution_count,
|
||||
"successful_executions": self._success_count,
|
||||
"failed_executions": self._failure_count,
|
||||
"success_rate": (self._success_count / self._execution_count * 100) if self._execution_count > 0 else 0
|
||||
}
|
||||
|
||||
def _run_schedule(self, interval: int, max_retries: int):
|
||||
"""Internal method to run scheduled agent executions."""
|
||||
while not self._stop_event.is_set():
|
||||
logger.info(f"[{datetime.now().strftime('%Y-%m-%d %H:%M:%S')}] Starting scheduled agent execution")
|
||||
|
||||
self._execute_with_retry(max_retries)
|
||||
|
||||
# Wait for next scheduled time
|
||||
logger.info(f"Next execution in {interval} seconds ({interval/3600:.1f} hours)")
|
||||
self._stop_event.wait(interval)
|
||||
|
||||
def _execute_with_retry(self, max_retries: int):
|
||||
"""Execute agent with retry logic."""
|
||||
self._execution_count += 1
|
||||
success = False
|
||||
|
||||
for attempt in range(max_retries):
|
||||
try:
|
||||
logger.info(f"Attempt {attempt + 1}/{max_retries}")
|
||||
result = self._executor.execute(self.task)
|
||||
|
||||
logger.info(f"Agent execution successful on attempt {attempt + 1}")
|
||||
logger.info(f"Result: {result}")
|
||||
|
||||
self._success_count += 1
|
||||
success = True
|
||||
|
||||
if self.on_success:
|
||||
self.on_success(result)
|
||||
|
||||
break
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Agent execution failed on attempt {attempt + 1}: {e}")
|
||||
|
||||
if attempt < max_retries - 1:
|
||||
wait_time = 30 * (attempt + 1) # Exponential backoff
|
||||
logger.info(f"Waiting {wait_time}s before retry...")
|
||||
time.sleep(wait_time)
|
||||
|
||||
if not success:
|
||||
self._failure_count += 1
|
||||
logger.error(f"Agent execution failed after {max_retries} attempts")
|
||||
|
||||
if self.on_failure:
|
||||
self.on_failure(f"Failed after {max_retries} attempts")
|
||||
|
||||
def execute_once(self) -> Any:
|
||||
"""
|
||||
Execute agent immediately (one-time execution).
|
||||
|
||||
Returns:
|
||||
Agent execution result
|
||||
"""
|
||||
logger.info("Executing agent once")
|
||||
try:
|
||||
result = self._executor.execute(self.task)
|
||||
logger.info(f"One-time execution successful: {result}")
|
||||
return result
|
||||
except Exception as e:
|
||||
logger.error(f"One-time execution failed: {e}")
|
||||
raise
|
||||
|
||||
|
||||
def create_agent_scheduler(
|
||||
agent,
|
||||
task: str,
|
||||
config: Optional[Dict[str, Any]] = None
|
||||
) -> AgentScheduler:
|
||||
"""
|
||||
Factory function to create agent scheduler.
|
||||
|
||||
Args:
|
||||
agent: PraisonAI Agent instance
|
||||
task: Task description
|
||||
config: Optional configuration
|
||||
|
||||
Returns:
|
||||
Configured AgentScheduler instance
|
||||
"""
|
||||
return AgentScheduler(agent, task, config)
|
||||
@@ -8,12 +8,19 @@ Structure:
|
||||
- features/: Feature handlers for CLI flags and commands
|
||||
"""
|
||||
|
||||
from .main import PraisonAI
|
||||
|
||||
__all__ = ["PraisonAI"]
|
||||
|
||||
|
||||
def __getattr__(name):
|
||||
"""Lazy load PraisonAI to avoid slow imports."""
|
||||
if name == "PraisonAI":
|
||||
from .main import PraisonAI
|
||||
return PraisonAI
|
||||
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
||||
|
||||
|
||||
def main():
|
||||
"""CLI entry point function."""
|
||||
from .main import PraisonAI
|
||||
praison_ai = PraisonAI()
|
||||
praison_ai.main()
|
||||
@@ -26,6 +26,9 @@ Features:
|
||||
- autonomy_mode: Autonomy levels (suggest, auto-edit, full-auto)
|
||||
- cost_tracker: Real-time cost and token tracking
|
||||
- message_queue: Message queuing while agent is processing
|
||||
- at_mentions: @ mention autocomplete for files and directories
|
||||
- compare: CLI command comparison feature
|
||||
- eval: Agent evaluation framework (accuracy, performance, reliability, criteria)
|
||||
"""
|
||||
|
||||
# Type hints available for IDE support
|
||||
@@ -58,6 +61,16 @@ __all__ = [
|
||||
'GitIntegrationHandler',
|
||||
'SandboxExecutorHandler',
|
||||
'MessageQueueHandler',
|
||||
# @ mention autocomplete
|
||||
'AtMentionCompleter',
|
||||
'CombinedCompleter',
|
||||
'FileSearchService',
|
||||
# Evaluation
|
||||
'EvalHandler',
|
||||
# Compare feature
|
||||
'CompareHandler',
|
||||
# Agent Skills
|
||||
'SkillsHandler',
|
||||
]
|
||||
|
||||
def __getattr__(name):
|
||||
@@ -138,4 +151,23 @@ def __getattr__(name):
|
||||
elif name == 'MessageQueueHandler':
|
||||
from .message_queue import MessageQueueHandler
|
||||
return MessageQueueHandler
|
||||
# @ mention autocomplete
|
||||
elif name == 'AtMentionCompleter':
|
||||
from .at_mentions import AtMentionCompleter
|
||||
return AtMentionCompleter
|
||||
elif name == 'CombinedCompleter':
|
||||
from .at_mentions import CombinedCompleter
|
||||
return CombinedCompleter
|
||||
elif name == 'FileSearchService':
|
||||
from .at_mentions import FileSearchService
|
||||
return FileSearchService
|
||||
elif name == 'CompareHandler':
|
||||
from .compare import CompareHandler
|
||||
return CompareHandler
|
||||
elif name == 'EvalHandler':
|
||||
from .eval import EvalHandler
|
||||
return EvalHandler
|
||||
elif name == 'SkillsHandler':
|
||||
from .skills import SkillsHandler
|
||||
return SkillsHandler
|
||||
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
||||
@@ -0,0 +1,734 @@
|
||||
"""
|
||||
Agent Scheduler CLI Feature.
|
||||
|
||||
Provides CLI commands for scheduling agents to run 24/7 at regular intervals.
|
||||
"""
|
||||
|
||||
import logging
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class AgentSchedulerHandler:
|
||||
"""Handler for agent scheduler CLI commands."""
|
||||
|
||||
@staticmethod
|
||||
def handle_daemon_command(subcommand: str, args, unknown_args=None) -> int:
|
||||
"""
|
||||
Handle daemon management commands (start, list, stop, logs, restart).
|
||||
|
||||
Args:
|
||||
subcommand: Command to execute (start, list, stop, logs, restart)
|
||||
args: Parsed command-line arguments
|
||||
unknown_args: Additional arguments
|
||||
|
||||
Returns:
|
||||
Exit code
|
||||
"""
|
||||
from praisonai.scheduler.state_manager import SchedulerStateManager
|
||||
from praisonai.scheduler.daemon_manager import DaemonManager
|
||||
|
||||
state_manager = SchedulerStateManager()
|
||||
daemon_manager = DaemonManager()
|
||||
|
||||
if subcommand == "start":
|
||||
return AgentSchedulerHandler._handle_start(args, unknown_args, state_manager, daemon_manager)
|
||||
elif subcommand == "list":
|
||||
return AgentSchedulerHandler._handle_list(state_manager)
|
||||
elif subcommand == "stop":
|
||||
return AgentSchedulerHandler._handle_stop(unknown_args, state_manager, daemon_manager)
|
||||
elif subcommand == "logs":
|
||||
return AgentSchedulerHandler._handle_logs(unknown_args, daemon_manager)
|
||||
elif subcommand == "restart":
|
||||
return AgentSchedulerHandler._handle_restart(unknown_args, state_manager, daemon_manager)
|
||||
elif subcommand == "delete":
|
||||
return AgentSchedulerHandler._handle_delete(unknown_args, state_manager)
|
||||
elif subcommand == "describe":
|
||||
return AgentSchedulerHandler._handle_describe(unknown_args, state_manager, daemon_manager)
|
||||
elif subcommand == "save":
|
||||
return AgentSchedulerHandler._handle_save(unknown_args, state_manager)
|
||||
elif subcommand == "stop-all":
|
||||
return AgentSchedulerHandler._handle_stop_all(state_manager, daemon_manager)
|
||||
elif subcommand == "stats":
|
||||
return AgentSchedulerHandler._handle_stats(state_manager, unknown_args, daemon_manager)
|
||||
else:
|
||||
print(f"❌ Unknown subcommand: {subcommand}")
|
||||
print("\nAvailable commands:")
|
||||
print(" start, list, stop, logs, restart, delete, describe, save, stop-all, stats")
|
||||
return 1
|
||||
|
||||
@staticmethod
|
||||
def _handle_start(args, unknown_args, state_manager, daemon_manager) -> int:
|
||||
"""Handle 'schedule start' command."""
|
||||
from datetime import datetime
|
||||
|
||||
if not unknown_args:
|
||||
print("❌ Error: Please provide scheduler name and task")
|
||||
print("\nUsage:")
|
||||
print(' praisonai schedule start <name> "Your task" --interval hourly')
|
||||
print("\nExample:")
|
||||
print(' praisonai schedule start news-checker "Check AI news" --interval hourly')
|
||||
return 1
|
||||
|
||||
# Parse arguments
|
||||
name = unknown_args[0]
|
||||
task = unknown_args[1] if len(unknown_args) > 1 else None
|
||||
|
||||
if not task:
|
||||
print("❌ Error: Please provide a task")
|
||||
return 1
|
||||
|
||||
# Get options
|
||||
interval = getattr(args, 'schedule_interval', None) or 'hourly'
|
||||
max_retries = getattr(args, 'schedule_max_retries', None) or 3
|
||||
timeout = getattr(args, 'timeout', None)
|
||||
max_cost = getattr(args, 'max_cost', None)
|
||||
|
||||
# Check if name already exists
|
||||
existing = state_manager.load_state(name)
|
||||
if existing and daemon_manager.get_status(existing.get('pid', 0))['is_alive']:
|
||||
print(f"❌ Error: Scheduler '{name}' is already running (PID: {existing['pid']})")
|
||||
print(f" Use 'praisonai schedule stop {name}' to stop it first")
|
||||
return 1
|
||||
|
||||
# Start daemon
|
||||
print(f"🚀 Starting scheduler '{name}'...")
|
||||
print(f" Task: {task}")
|
||||
print(f" Interval: {interval}")
|
||||
if timeout:
|
||||
print(f" Timeout: {timeout}s")
|
||||
if max_cost:
|
||||
print(f" Budget: ${max_cost}")
|
||||
|
||||
pid = daemon_manager.start_scheduler_daemon(
|
||||
name=name,
|
||||
task=task,
|
||||
interval=interval,
|
||||
max_cost=max_cost,
|
||||
timeout=timeout,
|
||||
max_retries=max_retries
|
||||
)
|
||||
|
||||
# Save state
|
||||
state = {
|
||||
"name": name,
|
||||
"pid": pid,
|
||||
"task": task,
|
||||
"interval": interval,
|
||||
"timeout": timeout,
|
||||
"max_cost": max_cost,
|
||||
"max_retries": max_retries,
|
||||
"status": "running",
|
||||
"started_at": datetime.now().isoformat(),
|
||||
"executions": 0,
|
||||
"cost": 0.0
|
||||
}
|
||||
state_manager.save_state(name, state)
|
||||
|
||||
print(f"\n✅ Scheduler '{name}' started successfully!")
|
||||
print(f" PID: {pid}")
|
||||
print(f" Logs: ~/.praisonai/logs/{name}.log")
|
||||
print(f"\nManage:")
|
||||
print(f" praisonai schedule list")
|
||||
print(f" praisonai schedule describe {name}")
|
||||
print(f" praisonai schedule logs {name} --follow")
|
||||
print(f" praisonai schedule stop {name}")
|
||||
|
||||
return 0
|
||||
|
||||
@staticmethod
|
||||
def _handle_list(state_manager) -> int:
|
||||
"""Handle 'schedule list' command."""
|
||||
states = state_manager.list_all()
|
||||
|
||||
if not states:
|
||||
print("No schedulers running.")
|
||||
print("\nStart one with:")
|
||||
print(' praisonai schedule start <name> "Your task" --interval hourly')
|
||||
return 0
|
||||
|
||||
# Clean up dead processes
|
||||
state_manager.cleanup_dead_processes()
|
||||
states = state_manager.list_all()
|
||||
|
||||
# Display table
|
||||
print(f"\n{'Name':<20} {'Status':<10} {'PID':<8} {'Interval':<12} {'Task':<40}")
|
||||
print("=" * 90)
|
||||
|
||||
for state in states:
|
||||
name = state.get('name', 'unknown')[:20]
|
||||
pid = state.get('pid', 0)
|
||||
status = "running" if state_manager.is_process_alive(pid) else "stopped"
|
||||
interval = state.get('interval', 'unknown')[:12]
|
||||
task = state.get('task', '')[:40]
|
||||
|
||||
status_icon = "🟢" if status == "running" else "🔴"
|
||||
print(f"{name:<20} {status_icon} {status:<8} {pid:<8} {interval:<12} {task:<40}")
|
||||
|
||||
print(f"\nTotal: {len(states)} scheduler(s)")
|
||||
return 0
|
||||
|
||||
@staticmethod
|
||||
def _handle_stats(state_manager, unknown_args=None, daemon_manager=None) -> int:
|
||||
"""Handle 'schedule stats' command - show aggregate or individual statistics."""
|
||||
# If a name is provided, show individual stats (alias for describe)
|
||||
if unknown_args and len(unknown_args) > 0:
|
||||
name = unknown_args[0]
|
||||
return AgentSchedulerHandler._handle_describe([name], state_manager, daemon_manager)
|
||||
|
||||
# Otherwise show aggregate stats
|
||||
states = state_manager.list_all()
|
||||
|
||||
if not states:
|
||||
print("No schedulers found.")
|
||||
print("\nStart one with:")
|
||||
print(' praisonai schedule start <name> "Your task" --interval hourly')
|
||||
return 0
|
||||
|
||||
# Calculate aggregate stats
|
||||
total_schedulers = len(states)
|
||||
running = sum(1 for s in states if s.get('status') == 'running')
|
||||
stopped = total_schedulers - running
|
||||
total_executions = sum(s.get('executions', 0) for s in states)
|
||||
total_cost = sum(s.get('cost', 0.0) for s in states)
|
||||
|
||||
print(f"\n{'='*60}")
|
||||
print(f"📊 Aggregate Scheduler Statistics")
|
||||
print(f"{'='*60}")
|
||||
print(f"Total Schedulers: {total_schedulers}")
|
||||
print(f" 🟢 Running: {running}")
|
||||
print(f" 🔴 Stopped: {stopped}")
|
||||
print(f"\nTotal Executions: {total_executions}")
|
||||
print(f"Total Cost: ${total_cost:.4f}")
|
||||
|
||||
if total_executions > 0:
|
||||
avg_cost = total_cost / total_executions
|
||||
print(f"Avg Cost/Execution: ${avg_cost:.4f}")
|
||||
|
||||
print(f"{'='*60}\n")
|
||||
|
||||
# Show per-scheduler breakdown
|
||||
print("Per-Scheduler Breakdown:")
|
||||
print(f"{'Name':<20} {'Status':<10} {'Executions':<12} {'Cost':<12}")
|
||||
print("="*60)
|
||||
|
||||
for state in sorted(states, key=lambda x: x.get('executions', 0), reverse=True):
|
||||
name = state['name'][:20]
|
||||
status = state.get('status', 'unknown')
|
||||
executions = state.get('executions', 0)
|
||||
cost = state.get('cost', 0.0)
|
||||
|
||||
status_icon = "🟢" if status == "running" else "🔴"
|
||||
print(f"{name:<20} {status_icon} {status:<8} {executions:<12} ${cost:<11.4f}")
|
||||
|
||||
return 0
|
||||
|
||||
@staticmethod
|
||||
def _handle_stop_all(state_manager, daemon_manager) -> int:
|
||||
"""Handle 'schedule stop-all' command."""
|
||||
states = state_manager.list_all()
|
||||
|
||||
if not states:
|
||||
print("No schedulers running.")
|
||||
return 0
|
||||
|
||||
print(f"\n🛑 Stopping all schedulers ({len(states)} total)...\n")
|
||||
|
||||
stopped = 0
|
||||
failed = 0
|
||||
|
||||
for state in states:
|
||||
name = state['name']
|
||||
pid = state['pid']
|
||||
|
||||
try:
|
||||
if daemon_manager.stop_daemon(pid):
|
||||
state_manager.delete_state(name)
|
||||
print(f"✅ Stopped '{name}' (PID: {pid})")
|
||||
stopped += 1
|
||||
else:
|
||||
print(f"❌ Failed to stop '{name}' (PID: {pid})")
|
||||
failed += 1
|
||||
except Exception as e:
|
||||
print(f"❌ Error stopping '{name}': {e}")
|
||||
failed += 1
|
||||
|
||||
print(f"\n📊 Summary: {stopped} stopped, {failed} failed")
|
||||
return 0 if failed == 0 else 1
|
||||
|
||||
@staticmethod
|
||||
def _handle_stop(unknown_args, state_manager, daemon_manager) -> int:
|
||||
"""Handle 'schedule stop' command."""
|
||||
if not unknown_args:
|
||||
print("❌ Error: Please provide scheduler name")
|
||||
print("\nUsage: praisonai schedule stop <name>")
|
||||
return 1
|
||||
|
||||
name = unknown_args[0]
|
||||
state = state_manager.load_state(name)
|
||||
|
||||
if not state:
|
||||
print(f"❌ Error: Scheduler '{name}' not found")
|
||||
print("\nList schedulers with: praisonai schedule list")
|
||||
return 1
|
||||
|
||||
pid = state.get('pid')
|
||||
if not pid:
|
||||
print(f"❌ Error: No PID found for scheduler '{name}'")
|
||||
return 1
|
||||
|
||||
print(f"🛑 Stopping scheduler '{name}' (PID: {pid})...")
|
||||
|
||||
success = daemon_manager.stop_daemon(pid)
|
||||
|
||||
if success:
|
||||
state['status'] = 'stopped'
|
||||
state_manager.save_state(name, state)
|
||||
print(f"✅ Scheduler '{name}' stopped successfully")
|
||||
return 0
|
||||
else:
|
||||
print(f"❌ Failed to stop scheduler '{name}'")
|
||||
return 1
|
||||
|
||||
@staticmethod
|
||||
def _handle_logs(unknown_args, daemon_manager) -> int:
|
||||
"""Handle 'schedule logs' command."""
|
||||
if not unknown_args:
|
||||
print("❌ Error: Please provide scheduler name")
|
||||
print("\nUsage: praisonai schedule logs <name> [-f]")
|
||||
return 1
|
||||
|
||||
name = unknown_args[0]
|
||||
follow = '-f' in unknown_args or '--follow' in unknown_args
|
||||
|
||||
if follow:
|
||||
print(f"📋 Following logs for '{name}' (Ctrl+C to stop)...")
|
||||
import subprocess
|
||||
log_file = daemon_manager.log_dir / f"{name}.log"
|
||||
if not log_file.exists():
|
||||
print(f"❌ Error: Log file not found for '{name}'")
|
||||
return 1
|
||||
|
||||
try:
|
||||
subprocess.run(['tail', '-f', str(log_file)])
|
||||
except KeyboardInterrupt:
|
||||
print("\n✅ Stopped following logs")
|
||||
return 0
|
||||
else:
|
||||
logs = daemon_manager.read_logs(name, lines=50)
|
||||
if logs:
|
||||
print(f"📋 Last 50 lines of logs for '{name}':\n")
|
||||
print(logs)
|
||||
return 0
|
||||
else:
|
||||
print(f"❌ Error: No logs found for '{name}'")
|
||||
return 1
|
||||
|
||||
@staticmethod
|
||||
def _handle_restart(unknown_args, state_manager, daemon_manager) -> int:
|
||||
"""Handle 'schedule restart' command."""
|
||||
from datetime import datetime
|
||||
import time
|
||||
|
||||
if not unknown_args:
|
||||
print("❌ Error: Please provide scheduler name")
|
||||
print("\nUsage: praisonai schedule restart <name>")
|
||||
return 1
|
||||
|
||||
name = unknown_args[0]
|
||||
state = state_manager.load_state(name)
|
||||
|
||||
if not state:
|
||||
print(f"❌ Error: Scheduler '{name}' not found")
|
||||
return 1
|
||||
|
||||
print(f"🔄 Restarting scheduler '{name}'...")
|
||||
|
||||
# Stop if running
|
||||
pid = state.get('pid')
|
||||
if pid and state_manager.is_process_alive(pid):
|
||||
daemon_manager.stop_daemon(pid)
|
||||
time.sleep(1)
|
||||
|
||||
# Start again
|
||||
new_pid = daemon_manager.start_scheduler_daemon(
|
||||
name=name,
|
||||
task=state['task'],
|
||||
interval=state['interval'],
|
||||
max_cost=state.get('max_cost'),
|
||||
timeout=state.get('timeout'),
|
||||
max_retries=state.get('max_retries', 3)
|
||||
)
|
||||
|
||||
state['pid'] = new_pid
|
||||
state['status'] = 'running'
|
||||
state['started_at'] = datetime.now().isoformat()
|
||||
state_manager.save_state(name, state)
|
||||
|
||||
print(f"✅ Scheduler '{name}' restarted (PID: {new_pid})")
|
||||
return 0
|
||||
|
||||
@staticmethod
|
||||
def _handle_delete(unknown_args, state_manager) -> int:
|
||||
"""Handle 'schedule delete' command."""
|
||||
if not unknown_args:
|
||||
print("❌ Error: Please provide scheduler name")
|
||||
print("\nUsage: praisonai schedule delete <name>")
|
||||
return 1
|
||||
|
||||
name = unknown_args[0]
|
||||
|
||||
if state_manager.delete_state(name):
|
||||
print(f"✅ Scheduler '{name}' deleted from list")
|
||||
return 0
|
||||
else:
|
||||
print(f"❌ Error: Scheduler '{name}' not found")
|
||||
return 1
|
||||
|
||||
@staticmethod
|
||||
def _handle_describe(unknown_args, state_manager, daemon_manager) -> int:
|
||||
"""Handle 'schedule describe' command."""
|
||||
from datetime import datetime
|
||||
|
||||
if not unknown_args:
|
||||
print("❌ Error: Please provide scheduler name")
|
||||
print("\nUsage: praisonai schedule describe <name>")
|
||||
return 1
|
||||
|
||||
name = unknown_args[0]
|
||||
state = state_manager.load_state(name)
|
||||
|
||||
if not state:
|
||||
print(f"❌ Error: Scheduler '{name}' not found")
|
||||
print("\nList schedulers with: praisonai schedule list")
|
||||
return 1
|
||||
|
||||
# Get process status
|
||||
pid = state.get('pid', 0)
|
||||
is_alive = state_manager.is_process_alive(pid)
|
||||
status = "🟢 running" if is_alive else "🔴 stopped"
|
||||
|
||||
# Calculate uptime
|
||||
started_at = state.get('started_at')
|
||||
uptime = "N/A"
|
||||
if started_at:
|
||||
try:
|
||||
start_time = datetime.fromisoformat(started_at)
|
||||
uptime_delta = datetime.now() - start_time
|
||||
hours = int(uptime_delta.total_seconds() // 3600)
|
||||
minutes = int((uptime_delta.total_seconds() % 3600) // 60)
|
||||
uptime = f"{hours}h {minutes}m"
|
||||
except:
|
||||
pass
|
||||
|
||||
# Display detailed info
|
||||
print(f"\n{'='*60}")
|
||||
print(f"📋 Scheduler Details: {name}")
|
||||
print(f"{'='*60}")
|
||||
print(f"Status: {status}")
|
||||
print(f"PID: {pid}")
|
||||
print(f"Uptime: {uptime}")
|
||||
print(f"Task: {state.get('task', 'N/A')}")
|
||||
print(f"Interval: {state.get('interval', 'N/A')}")
|
||||
print(f"Max Retries: {state.get('max_retries', 'N/A')}")
|
||||
|
||||
if state.get('timeout'):
|
||||
print(f"Timeout: {state['timeout']}s")
|
||||
|
||||
if state.get('max_cost'):
|
||||
print(f"Budget: ${state['max_cost']}")
|
||||
|
||||
print(f"Executions: {state.get('executions', 0)}")
|
||||
print(f"Total Cost: ${state.get('cost', 0.0):.4f}")
|
||||
print(f"Started: {started_at or 'N/A'}")
|
||||
|
||||
# Log file location
|
||||
log_file = daemon_manager.log_dir / f"{name}.log"
|
||||
print(f"Logs: {log_file}")
|
||||
|
||||
print(f"{'='*60}\n")
|
||||
|
||||
return 0
|
||||
|
||||
@staticmethod
|
||||
def _handle_save(unknown_args, state_manager) -> int:
|
||||
"""Handle 'schedule save' command."""
|
||||
import yaml
|
||||
|
||||
if not unknown_args:
|
||||
print("❌ Error: Please provide scheduler name")
|
||||
print("\nUsage: praisonai schedule save <name> [output.yaml]")
|
||||
return 1
|
||||
|
||||
name = unknown_args[0]
|
||||
output_file = unknown_args[1] if len(unknown_args) > 1 else f"{name}.yaml"
|
||||
|
||||
state = state_manager.load_state(name)
|
||||
|
||||
if not state:
|
||||
print(f"❌ Error: Scheduler '{name}' not found")
|
||||
return 1
|
||||
|
||||
# Create YAML config from state
|
||||
yaml_config = {
|
||||
'framework': 'praisonai',
|
||||
'agents': [{
|
||||
'name': name,
|
||||
'role': 'Task Executor',
|
||||
'goal': state.get('task', ''),
|
||||
'instructions': state.get('task', ''),
|
||||
'verbose': True
|
||||
}],
|
||||
'task': state.get('task', ''),
|
||||
'schedule': {
|
||||
'interval': state.get('interval', 'hourly'),
|
||||
'max_retries': state.get('max_retries', 3),
|
||||
'run_immediately': True
|
||||
}
|
||||
}
|
||||
|
||||
# Add optional fields
|
||||
if state.get('timeout'):
|
||||
yaml_config['schedule']['timeout'] = state['timeout']
|
||||
|
||||
if state.get('max_cost'):
|
||||
yaml_config['schedule']['max_cost'] = state['max_cost']
|
||||
|
||||
# Write to file
|
||||
try:
|
||||
with open(output_file, 'w') as f:
|
||||
yaml.dump(yaml_config, f, default_flow_style=False, sort_keys=False)
|
||||
|
||||
print(f"✅ Configuration saved to: {output_file}")
|
||||
print(f"\nRun with:")
|
||||
print(f" praisonai schedule {output_file}")
|
||||
return 0
|
||||
except Exception as e:
|
||||
print(f"❌ Error saving configuration: {e}")
|
||||
return 1
|
||||
|
||||
@staticmethod
|
||||
def handle_schedule_command(args, unknown_args, daemon_mode=False) -> int:
|
||||
"""
|
||||
Handle the schedule command for running agents periodically.
|
||||
|
||||
Supports two modes:
|
||||
1. YAML mode: praisonai schedule agents.yaml
|
||||
2. Prompt mode: praisonai schedule "Your task here" --interval hourly
|
||||
|
||||
Args:
|
||||
args: Parsed command-line arguments
|
||||
unknown_args: Additional arguments after the command
|
||||
|
||||
Returns:
|
||||
Exit code (0 for success, 1 for error)
|
||||
"""
|
||||
try:
|
||||
from praisonai.scheduler import AgentScheduler
|
||||
from praisonaiagents import Agent
|
||||
except ImportError:
|
||||
print("Error: praisonai.scheduler or praisonaiagents module not found")
|
||||
print("Please ensure PraisonAI is properly installed")
|
||||
print("pip install praisonai praisonaiagents")
|
||||
return 1
|
||||
|
||||
# Check if first arg is a YAML file or a prompt
|
||||
first_arg = unknown_args[0] if unknown_args else None
|
||||
is_yaml_mode = first_arg and (first_arg.endswith('.yaml') or first_arg.endswith('.yml'))
|
||||
|
||||
# Get overrides from CLI
|
||||
interval_override = getattr(args, 'schedule_interval', None) or 'hourly'
|
||||
max_retries_override = getattr(args, 'schedule_max_retries', None) or 3
|
||||
timeout_override = getattr(args, 'timeout', None)
|
||||
max_cost_override = getattr(args, 'max_cost', None)
|
||||
verbose = getattr(args, 'verbose', False)
|
||||
|
||||
# Set up logging - only show logs if verbose
|
||||
if verbose:
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format='[%(asctime)s] %(levelname)s - %(message)s',
|
||||
datefmt='%Y-%m-%d %H:%M:%S'
|
||||
)
|
||||
else:
|
||||
# Suppress scheduler logs in non-verbose mode
|
||||
logging.getLogger('praisonai.scheduler').setLevel(logging.WARNING)
|
||||
|
||||
try:
|
||||
if is_yaml_mode:
|
||||
# YAML mode: Load from agents.yaml
|
||||
yaml_path = first_arg
|
||||
print(f"🤖 Loading agent configuration from: {yaml_path}")
|
||||
scheduler = AgentScheduler.from_yaml(
|
||||
yaml_path=yaml_path,
|
||||
interval_override=interval_override,
|
||||
max_retries_override=max_retries_override,
|
||||
timeout_override=timeout_override,
|
||||
max_cost_override=max_cost_override
|
||||
)
|
||||
else:
|
||||
# Prompt mode: Create agent from direct prompt
|
||||
if not first_arg:
|
||||
print("❌ Error: Please provide either a YAML file or a task prompt")
|
||||
print("\nExamples:")
|
||||
print(" praisonai schedule agents.yaml")
|
||||
print(' praisonai schedule "Check news every hour" --interval hourly')
|
||||
return 1
|
||||
|
||||
task = first_arg
|
||||
|
||||
# Create agent
|
||||
agent = Agent(
|
||||
name="Scheduled Agent",
|
||||
role="Task Executor",
|
||||
goal=task,
|
||||
instructions=task,
|
||||
verbose=True # Enable verbose to see output in logs
|
||||
)
|
||||
|
||||
# Create scheduler
|
||||
scheduler = AgentScheduler(
|
||||
agent=agent,
|
||||
task=task,
|
||||
timeout=timeout_override,
|
||||
max_cost=max_cost_override
|
||||
)
|
||||
|
||||
# Get configuration
|
||||
interval = interval_override
|
||||
max_retries = max_retries_override
|
||||
|
||||
if is_yaml_mode:
|
||||
schedule_config = scheduler._yaml_schedule_config
|
||||
interval = interval_override or schedule_config.get('interval', 'hourly')
|
||||
max_retries = max_retries_override or schedule_config.get('max_retries', 3)
|
||||
|
||||
print(f"Schedule: {interval}")
|
||||
print(f"Max Retries: {max_retries}")
|
||||
if scheduler.timeout:
|
||||
print(f"Timeout: {scheduler.timeout}s")
|
||||
if scheduler.max_cost:
|
||||
print(f"Budget: ${scheduler.max_cost}")
|
||||
print(f"{'='*60}\n")
|
||||
|
||||
# Start scheduler
|
||||
if is_yaml_mode:
|
||||
scheduler.start_from_yaml_config()
|
||||
else:
|
||||
scheduler.start(schedule_expr=interval, max_retries=max_retries, run_immediately=True)
|
||||
|
||||
# If daemon mode, block to keep process alive
|
||||
if daemon_mode:
|
||||
import time
|
||||
try:
|
||||
while scheduler.is_running:
|
||||
time.sleep(1)
|
||||
except KeyboardInterrupt:
|
||||
scheduler.stop()
|
||||
else:
|
||||
# Foreground mode - wait for Ctrl+C
|
||||
print("⏰ Starting scheduler... (Press Ctrl+C to stop)\n")
|
||||
import time
|
||||
try:
|
||||
while scheduler.is_running:
|
||||
time.sleep(1)
|
||||
except KeyboardInterrupt:
|
||||
print("\n\n🛑 Stopping scheduler...")
|
||||
scheduler.stop()
|
||||
|
||||
# Display final stats
|
||||
stats = scheduler.get_stats()
|
||||
print("\n📊 Final Statistics:")
|
||||
print(f" Total Executions: {stats['total_executions']}")
|
||||
print(f" Successful: {stats['successful_executions']}")
|
||||
print(f" Failed: {stats['failed_executions']}")
|
||||
print(f" Success Rate: {stats['success_rate']:.1f}%")
|
||||
print("\n✅ Agent stopped successfully\n")
|
||||
|
||||
return 0
|
||||
|
||||
except FileNotFoundError as e:
|
||||
print(f"❌ Error: {e}")
|
||||
print(f"\nMake sure {yaml_path} exists in the current directory.")
|
||||
print("\nExample agents.yaml structure:")
|
||||
print("""
|
||||
framework: praisonai
|
||||
|
||||
agents:
|
||||
- name: "AI News Monitor"
|
||||
role: "News Analyst"
|
||||
instructions: "Search and summarize AI news"
|
||||
tools:
|
||||
- search_tool
|
||||
|
||||
task: "Search for latest AI news"
|
||||
|
||||
schedule:
|
||||
interval: "hourly"
|
||||
max_retries: 3
|
||||
run_immediately: true
|
||||
""")
|
||||
return 1
|
||||
|
||||
except ValueError as e:
|
||||
print(f"❌ Configuration Error: {e}")
|
||||
return 1
|
||||
|
||||
except Exception as e:
|
||||
print(f"❌ Error: {e}")
|
||||
if verbose:
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
return 1
|
||||
|
||||
@staticmethod
|
||||
def add_schedule_arguments(subparsers):
|
||||
"""
|
||||
Add schedule subcommand and arguments to parser.
|
||||
|
||||
Args:
|
||||
subparsers: Subparsers object from argparse
|
||||
"""
|
||||
schedule_parser = subparsers.add_parser(
|
||||
'schedule',
|
||||
help='Schedule an agent to run continuously at regular intervals'
|
||||
)
|
||||
|
||||
schedule_parser.add_argument(
|
||||
'schedule_yaml',
|
||||
nargs='?',
|
||||
default='agents.yaml',
|
||||
help='Path to agents.yaml file (default: agents.yaml)'
|
||||
)
|
||||
|
||||
schedule_parser.add_argument(
|
||||
'--interval',
|
||||
dest='schedule_interval',
|
||||
type=str,
|
||||
help='Override schedule interval (e.g., "hourly", "*/30m", "daily")'
|
||||
)
|
||||
|
||||
schedule_parser.add_argument(
|
||||
'--max-retries',
|
||||
dest='schedule_max_retries',
|
||||
type=int,
|
||||
help='Override maximum retry attempts (default: from YAML or 3)'
|
||||
)
|
||||
|
||||
schedule_parser.add_argument(
|
||||
'--verbose', '-v',
|
||||
action='store_true',
|
||||
help='Enable verbose logging'
|
||||
)
|
||||
|
||||
schedule_parser.set_defaults(func=AgentSchedulerHandler.handle_schedule_command)
|
||||
|
||||
|
||||
def setup_scheduler_cli(subparsers):
|
||||
"""
|
||||
Setup scheduler CLI commands.
|
||||
|
||||
Args:
|
||||
subparsers: Subparsers object from argparse
|
||||
"""
|
||||
AgentSchedulerHandler.add_schedule_arguments(subparsers)
|
||||
@@ -0,0 +1,729 @@
|
||||
"""
|
||||
Agent Scheduler CLI Feature.
|
||||
|
||||
Provides CLI commands for scheduling agents to run 24/7 at regular intervals.
|
||||
"""
|
||||
|
||||
import logging
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class AgentSchedulerHandler:
|
||||
"""Handler for agent scheduler CLI commands."""
|
||||
|
||||
@staticmethod
|
||||
def handle_daemon_command(subcommand: str, args, unknown_args=None) -> int:
|
||||
"""
|
||||
Handle daemon management commands (start, list, stop, logs, restart).
|
||||
|
||||
Args:
|
||||
subcommand: Command to execute (start, list, stop, logs, restart)
|
||||
args: Parsed command-line arguments
|
||||
unknown_args: Additional arguments
|
||||
|
||||
Returns:
|
||||
Exit code
|
||||
"""
|
||||
from praisonai.scheduler.state_manager import SchedulerStateManager
|
||||
from praisonai.scheduler.daemon_manager import DaemonManager
|
||||
|
||||
state_manager = SchedulerStateManager()
|
||||
daemon_manager = DaemonManager()
|
||||
|
||||
if subcommand == "start":
|
||||
return AgentSchedulerHandler._handle_start(args, unknown_args, state_manager, daemon_manager)
|
||||
elif subcommand == "list":
|
||||
return AgentSchedulerHandler._handle_list(state_manager)
|
||||
elif subcommand == "stop":
|
||||
return AgentSchedulerHandler._handle_stop(unknown_args, state_manager, daemon_manager)
|
||||
elif subcommand == "logs":
|
||||
return AgentSchedulerHandler._handle_logs(unknown_args, daemon_manager)
|
||||
elif subcommand == "restart":
|
||||
return AgentSchedulerHandler._handle_restart(unknown_args, state_manager, daemon_manager)
|
||||
elif subcommand == "delete":
|
||||
return AgentSchedulerHandler._handle_delete(unknown_args, state_manager)
|
||||
elif subcommand == "describe":
|
||||
return AgentSchedulerHandler._handle_describe(unknown_args, state_manager, daemon_manager)
|
||||
elif subcommand == "save":
|
||||
return AgentSchedulerHandler._handle_save(unknown_args, state_manager)
|
||||
elif subcommand == "stop-all":
|
||||
return AgentSchedulerHandler._handle_stop_all(state_manager, daemon_manager)
|
||||
elif subcommand == "stats":
|
||||
return AgentSchedulerHandler._handle_stats(state_manager, unknown_args, daemon_manager)
|
||||
else:
|
||||
print(f"❌ Unknown subcommand: {subcommand}")
|
||||
print("\nAvailable commands:")
|
||||
print(" start, list, stop, logs, restart, delete, describe, save, stop-all, stats")
|
||||
return 1
|
||||
|
||||
@staticmethod
|
||||
def _handle_start(args, unknown_args, state_manager, daemon_manager) -> int:
|
||||
"""Handle 'schedule start' command."""
|
||||
from datetime import datetime
|
||||
|
||||
if not unknown_args:
|
||||
print("❌ Error: Please provide scheduler name and task")
|
||||
print("\nUsage:")
|
||||
print(' praisonai schedule start <name> "Your task" --interval hourly')
|
||||
print("\nExample:")
|
||||
print(' praisonai schedule start news-checker "Check AI news" --interval hourly')
|
||||
return 1
|
||||
|
||||
# Parse arguments
|
||||
name = unknown_args[0]
|
||||
task = unknown_args[1] if len(unknown_args) > 1 else None
|
||||
|
||||
if not task:
|
||||
print("❌ Error: Please provide a task")
|
||||
return 1
|
||||
|
||||
# Get options
|
||||
interval = getattr(args, 'schedule_interval', None) or 'hourly'
|
||||
max_retries = getattr(args, 'schedule_max_retries', None) or 3
|
||||
timeout = getattr(args, 'timeout', None)
|
||||
max_cost = getattr(args, 'max_cost', None)
|
||||
|
||||
# Check if name already exists
|
||||
existing = state_manager.load_state(name)
|
||||
if existing and daemon_manager.get_status(existing.get('pid', 0))['is_alive']:
|
||||
print(f"❌ Error: Scheduler '{name}' is already running (PID: {existing['pid']})")
|
||||
print(f" Use 'praisonai schedule stop {name}' to stop it first")
|
||||
return 1
|
||||
|
||||
# Start daemon
|
||||
print(f"🚀 Starting scheduler '{name}'...")
|
||||
print(f" Task: {task}")
|
||||
print(f" Interval: {interval}")
|
||||
if timeout:
|
||||
print(f" Timeout: {timeout}s")
|
||||
if max_cost:
|
||||
print(f" Budget: ${max_cost}")
|
||||
|
||||
pid = daemon_manager.start_scheduler_daemon(
|
||||
name=name,
|
||||
task=task,
|
||||
interval=interval,
|
||||
max_cost=max_cost,
|
||||
timeout=timeout,
|
||||
max_retries=max_retries
|
||||
)
|
||||
|
||||
# Save state
|
||||
state = {
|
||||
"name": name,
|
||||
"pid": pid,
|
||||
"task": task,
|
||||
"interval": interval,
|
||||
"timeout": timeout,
|
||||
"max_cost": max_cost,
|
||||
"max_retries": max_retries,
|
||||
"status": "running",
|
||||
"started_at": datetime.now().isoformat(),
|
||||
"executions": 0,
|
||||
"cost": 0.0
|
||||
}
|
||||
state_manager.save_state(name, state)
|
||||
|
||||
print(f"\n✅ Scheduler '{name}' started successfully!")
|
||||
print(f" PID: {pid}")
|
||||
print(f" Logs: ~/.praisonai/logs/{name}.log")
|
||||
print(f"\nManage:")
|
||||
print(f" praisonai schedule list")
|
||||
print(f" praisonai schedule describe {name}")
|
||||
print(f" praisonai schedule logs {name} --follow")
|
||||
print(f" praisonai schedule stop {name}")
|
||||
|
||||
return 0
|
||||
|
||||
@staticmethod
|
||||
def _handle_list(state_manager) -> int:
|
||||
"""Handle 'schedule list' command."""
|
||||
states = state_manager.list_all()
|
||||
|
||||
if not states:
|
||||
print("No schedulers running.")
|
||||
print("\nStart one with:")
|
||||
print(' praisonai schedule start <name> "Your task" --interval hourly')
|
||||
return 0
|
||||
|
||||
# Clean up dead processes
|
||||
state_manager.cleanup_dead_processes()
|
||||
states = state_manager.list_all()
|
||||
|
||||
# Display table
|
||||
print(f"\n{'Name':<20} {'Status':<10} {'PID':<8} {'Interval':<12} {'Task':<40}")
|
||||
print("=" * 90)
|
||||
|
||||
for state in states:
|
||||
name = state.get('name', 'unknown')[:20]
|
||||
pid = state.get('pid', 0)
|
||||
status = "running" if state_manager.is_process_alive(pid) else "stopped"
|
||||
interval = state.get('interval', 'unknown')[:12]
|
||||
task = state.get('task', '')[:40]
|
||||
|
||||
status_icon = "🟢" if status == "running" else "🔴"
|
||||
print(f"{name:<20} {status_icon} {status:<8} {pid:<8} {interval:<12} {task:<40}")
|
||||
|
||||
print(f"\nTotal: {len(states)} scheduler(s)")
|
||||
return 0
|
||||
|
||||
@staticmethod
|
||||
def _handle_stats(state_manager, unknown_args=None, daemon_manager=None) -> int:
|
||||
"""Handle 'schedule stats' command - show aggregate or individual statistics."""
|
||||
# If a name is provided, show individual stats (alias for describe)
|
||||
if unknown_args and len(unknown_args) > 0:
|
||||
name = unknown_args[0]
|
||||
return AgentSchedulerHandler._handle_describe([name], state_manager, None)
|
||||
|
||||
# Otherwise show aggregate stats
|
||||
states = state_manager.list_all()
|
||||
|
||||
if not states:
|
||||
print("No schedulers found.")
|
||||
print("\nStart one with:")
|
||||
print(' praisonai schedule start <name> "Your task" --interval hourly')
|
||||
return 0
|
||||
|
||||
# Calculate aggregate stats
|
||||
total_schedulers = len(states)
|
||||
running = sum(1 for s in states if s.get('status') == 'running')
|
||||
stopped = total_schedulers - running
|
||||
total_executions = sum(s.get('executions', 0) for s in states)
|
||||
total_cost = sum(s.get('cost', 0.0) for s in states)
|
||||
|
||||
print(f"\n{'='*60}")
|
||||
print(f"📊 Aggregate Scheduler Statistics")
|
||||
print(f"{'='*60}")
|
||||
print(f"Total Schedulers: {total_schedulers}")
|
||||
print(f" 🟢 Running: {running}")
|
||||
print(f" 🔴 Stopped: {stopped}")
|
||||
print(f"\nTotal Executions: {total_executions}")
|
||||
print(f"Total Cost: ${total_cost:.4f}")
|
||||
|
||||
if total_executions > 0:
|
||||
avg_cost = total_cost / total_executions
|
||||
print(f"Avg Cost/Execution: ${avg_cost:.4f}")
|
||||
|
||||
print(f"{'='*60}\n")
|
||||
|
||||
# Show per-scheduler breakdown
|
||||
print("Per-Scheduler Breakdown:")
|
||||
print(f"{'Name':<20} {'Status':<10} {'Executions':<12} {'Cost':<12}")
|
||||
print("="*60)
|
||||
|
||||
for state in sorted(states, key=lambda x: x.get('executions', 0), reverse=True):
|
||||
name = state['name'][:20]
|
||||
status = state.get('status', 'unknown')
|
||||
executions = state.get('executions', 0)
|
||||
cost = state.get('cost', 0.0)
|
||||
|
||||
status_icon = "🟢" if status == "running" else "🔴"
|
||||
print(f"{name:<20} {status_icon} {status:<8} {executions:<12} ${cost:<11.4f}")
|
||||
|
||||
return 0
|
||||
|
||||
@staticmethod
|
||||
def _handle_stop_all(state_manager, daemon_manager) -> int:
|
||||
"""Handle 'schedule stop-all' command."""
|
||||
states = state_manager.list_all()
|
||||
|
||||
if not states:
|
||||
print("No schedulers running.")
|
||||
return 0
|
||||
|
||||
print(f"\n🛑 Stopping all schedulers ({len(states)} total)...\n")
|
||||
|
||||
stopped = 0
|
||||
failed = 0
|
||||
|
||||
for state in states:
|
||||
name = state['name']
|
||||
pid = state['pid']
|
||||
|
||||
try:
|
||||
if daemon_manager.stop_daemon(pid):
|
||||
state_manager.delete_state(name)
|
||||
print(f"✅ Stopped '{name}' (PID: {pid})")
|
||||
stopped += 1
|
||||
else:
|
||||
print(f"❌ Failed to stop '{name}' (PID: {pid})")
|
||||
failed += 1
|
||||
except Exception as e:
|
||||
print(f"❌ Error stopping '{name}': {e}")
|
||||
failed += 1
|
||||
|
||||
print(f"\n📊 Summary: {stopped} stopped, {failed} failed")
|
||||
return 0 if failed == 0 else 1
|
||||
|
||||
@staticmethod
|
||||
def _handle_stop(unknown_args, state_manager, daemon_manager) -> int:
|
||||
"""Handle 'schedule stop' command."""
|
||||
if not unknown_args:
|
||||
print("❌ Error: Please provide scheduler name")
|
||||
print("\nUsage: praisonai schedule stop <name>")
|
||||
return 1
|
||||
|
||||
name = unknown_args[0]
|
||||
state = state_manager.load_state(name)
|
||||
|
||||
if not state:
|
||||
print(f"❌ Error: Scheduler '{name}' not found")
|
||||
print("\nList schedulers with: praisonai schedule list")
|
||||
return 1
|
||||
|
||||
pid = state.get('pid')
|
||||
if not pid:
|
||||
print(f"❌ Error: No PID found for scheduler '{name}'")
|
||||
return 1
|
||||
|
||||
print(f"🛑 Stopping scheduler '{name}' (PID: {pid})...")
|
||||
|
||||
success = daemon_manager.stop_daemon(pid)
|
||||
|
||||
if success:
|
||||
state['status'] = 'stopped'
|
||||
state_manager.save_state(name, state)
|
||||
print(f"✅ Scheduler '{name}' stopped successfully")
|
||||
return 0
|
||||
else:
|
||||
print(f"❌ Failed to stop scheduler '{name}'")
|
||||
return 1
|
||||
|
||||
@staticmethod
|
||||
def _handle_logs(unknown_args, daemon_manager) -> int:
|
||||
"""Handle 'schedule logs' command."""
|
||||
if not unknown_args:
|
||||
print("❌ Error: Please provide scheduler name")
|
||||
print("\nUsage: praisonai schedule logs <name> [-f]")
|
||||
return 1
|
||||
|
||||
name = unknown_args[0]
|
||||
follow = '-f' in unknown_args or '--follow' in unknown_args
|
||||
|
||||
if follow:
|
||||
print(f"📋 Following logs for '{name}' (Ctrl+C to stop)...")
|
||||
import subprocess
|
||||
log_file = daemon_manager.log_dir / f"{name}.log"
|
||||
if not log_file.exists():
|
||||
print(f"❌ Error: Log file not found for '{name}'")
|
||||
return 1
|
||||
|
||||
try:
|
||||
subprocess.run(['tail', '-f', str(log_file)])
|
||||
except KeyboardInterrupt:
|
||||
print("\n✅ Stopped following logs")
|
||||
return 0
|
||||
else:
|
||||
logs = daemon_manager.read_logs(name, lines=50)
|
||||
if logs:
|
||||
print(f"📋 Last 50 lines of logs for '{name}':\n")
|
||||
print(logs)
|
||||
return 0
|
||||
else:
|
||||
print(f"❌ Error: No logs found for '{name}'")
|
||||
return 1
|
||||
|
||||
@staticmethod
|
||||
def _handle_restart(unknown_args, state_manager, daemon_manager) -> int:
|
||||
"""Handle 'schedule restart' command."""
|
||||
from datetime import datetime
|
||||
import time
|
||||
|
||||
if not unknown_args:
|
||||
print("❌ Error: Please provide scheduler name")
|
||||
print("\nUsage: praisonai schedule restart <name>")
|
||||
return 1
|
||||
|
||||
name = unknown_args[0]
|
||||
state = state_manager.load_state(name)
|
||||
|
||||
if not state:
|
||||
print(f"❌ Error: Scheduler '{name}' not found")
|
||||
return 1
|
||||
|
||||
print(f"🔄 Restarting scheduler '{name}'...")
|
||||
|
||||
# Stop if running
|
||||
pid = state.get('pid')
|
||||
if pid and state_manager.is_process_alive(pid):
|
||||
daemon_manager.stop_daemon(pid)
|
||||
time.sleep(1)
|
||||
|
||||
# Start again
|
||||
new_pid = daemon_manager.start_scheduler_daemon(
|
||||
name=name,
|
||||
task=state['task'],
|
||||
interval=state['interval'],
|
||||
max_cost=state.get('max_cost'),
|
||||
timeout=state.get('timeout'),
|
||||
max_retries=state.get('max_retries', 3)
|
||||
)
|
||||
|
||||
state['pid'] = new_pid
|
||||
state['status'] = 'running'
|
||||
state['started_at'] = datetime.now().isoformat()
|
||||
state_manager.save_state(name, state)
|
||||
|
||||
print(f"✅ Scheduler '{name}' restarted (PID: {new_pid})")
|
||||
return 0
|
||||
|
||||
@staticmethod
|
||||
def _handle_delete(unknown_args, state_manager) -> int:
|
||||
"""Handle 'schedule delete' command."""
|
||||
if not unknown_args:
|
||||
print("❌ Error: Please provide scheduler name")
|
||||
print("\nUsage: praisonai schedule delete <name>")
|
||||
return 1
|
||||
|
||||
name = unknown_args[0]
|
||||
|
||||
if state_manager.delete_state(name):
|
||||
print(f"✅ Scheduler '{name}' deleted from list")
|
||||
return 0
|
||||
else:
|
||||
print(f"❌ Error: Scheduler '{name}' not found")
|
||||
return 1
|
||||
|
||||
@staticmethod
|
||||
def _handle_describe(unknown_args, state_manager, daemon_manager) -> int:
|
||||
"""Handle 'schedule describe' command."""
|
||||
from datetime import datetime
|
||||
|
||||
if not unknown_args:
|
||||
print("❌ Error: Please provide scheduler name")
|
||||
print("\nUsage: praisonai schedule describe <name>")
|
||||
return 1
|
||||
|
||||
name = unknown_args[0]
|
||||
state = state_manager.load_state(name)
|
||||
|
||||
if not state:
|
||||
print(f"❌ Error: Scheduler '{name}' not found")
|
||||
print("\nList schedulers with: praisonai schedule list")
|
||||
return 1
|
||||
|
||||
# Get process status
|
||||
pid = state.get('pid', 0)
|
||||
is_alive = state_manager.is_process_alive(pid)
|
||||
status = "🟢 running" if is_alive else "🔴 stopped"
|
||||
|
||||
# Calculate uptime
|
||||
started_at = state.get('started_at')
|
||||
uptime = "N/A"
|
||||
if started_at:
|
||||
try:
|
||||
start_time = datetime.fromisoformat(started_at)
|
||||
uptime_delta = datetime.now() - start_time
|
||||
hours = int(uptime_delta.total_seconds() // 3600)
|
||||
minutes = int((uptime_delta.total_seconds() % 3600) // 60)
|
||||
uptime = f"{hours}h {minutes}m"
|
||||
except:
|
||||
pass
|
||||
|
||||
# Display detailed info
|
||||
print(f"\n{'='*60}")
|
||||
print(f"📋 Scheduler Details: {name}")
|
||||
print(f"{'='*60}")
|
||||
print(f"Status: {status}")
|
||||
print(f"PID: {pid}")
|
||||
print(f"Uptime: {uptime}")
|
||||
print(f"Task: {state.get('task', 'N/A')}")
|
||||
print(f"Interval: {state.get('interval', 'N/A')}")
|
||||
print(f"Max Retries: {state.get('max_retries', 'N/A')}")
|
||||
|
||||
if state.get('timeout'):
|
||||
print(f"Timeout: {state['timeout']}s")
|
||||
|
||||
if state.get('max_cost'):
|
||||
print(f"Budget: ${state['max_cost']}")
|
||||
|
||||
print(f"Executions: {state.get('executions', 0)}")
|
||||
print(f"Total Cost: ${state.get('cost', 0.0):.4f}")
|
||||
print(f"Started: {started_at or 'N/A'}")
|
||||
|
||||
# Log file location
|
||||
log_file = daemon_manager.log_dir / f"{name}.log"
|
||||
print(f"Logs: {log_file}")
|
||||
|
||||
print(f"{'='*60}\n")
|
||||
|
||||
return 0
|
||||
|
||||
@staticmethod
|
||||
def _handle_save(unknown_args, state_manager) -> int:
|
||||
"""Handle 'schedule save' command."""
|
||||
import yaml
|
||||
|
||||
if not unknown_args:
|
||||
print("❌ Error: Please provide scheduler name")
|
||||
print("\nUsage: praisonai schedule save <name> [output.yaml]")
|
||||
return 1
|
||||
|
||||
name = unknown_args[0]
|
||||
output_file = unknown_args[1] if len(unknown_args) > 1 else f"{name}.yaml"
|
||||
|
||||
state = state_manager.load_state(name)
|
||||
|
||||
if not state:
|
||||
print(f"❌ Error: Scheduler '{name}' not found")
|
||||
return 1
|
||||
|
||||
# Create YAML config from state
|
||||
yaml_config = {
|
||||
'framework': 'praisonai',
|
||||
'agents': [{
|
||||
'name': name,
|
||||
'role': 'Task Executor',
|
||||
'goal': state.get('task', ''),
|
||||
'instructions': state.get('task', ''),
|
||||
'verbose': True
|
||||
}],
|
||||
'task': state.get('task', ''),
|
||||
'schedule': {
|
||||
'interval': state.get('interval', 'hourly'),
|
||||
'max_retries': state.get('max_retries', 3),
|
||||
'run_immediately': True
|
||||
}
|
||||
}
|
||||
|
||||
# Add optional fields
|
||||
if state.get('timeout'):
|
||||
yaml_config['schedule']['timeout'] = state['timeout']
|
||||
|
||||
if state.get('max_cost'):
|
||||
yaml_config['schedule']['max_cost'] = state['max_cost']
|
||||
|
||||
# Write to file
|
||||
try:
|
||||
with open(output_file, 'w') as f:
|
||||
yaml.dump(yaml_config, f, default_flow_style=False, sort_keys=False)
|
||||
|
||||
print(f"✅ Configuration saved to: {output_file}")
|
||||
print(f"\nRun with:")
|
||||
print(f" praisonai schedule {output_file}")
|
||||
return 0
|
||||
except Exception as e:
|
||||
print(f"❌ Error saving configuration: {e}")
|
||||
return 1
|
||||
|
||||
@staticmethod
|
||||
def handle_schedule_command(args, unknown_args=None) -> int:
|
||||
"""
|
||||
Handle the schedule command for running agents periodically.
|
||||
|
||||
Supports two modes:
|
||||
1. YAML mode: praisonai schedule agents.yaml
|
||||
2. Prompt mode: praisonai schedule "Your task here" --interval hourly
|
||||
|
||||
Args:
|
||||
args: Parsed command-line arguments
|
||||
unknown_args: Additional arguments after the command
|
||||
|
||||
Returns:
|
||||
Exit code (0 for success, 1 for error)
|
||||
"""
|
||||
try:
|
||||
from praisonai.scheduler import AgentScheduler
|
||||
from praisonaiagents import Agent
|
||||
except ImportError:
|
||||
print("Error: praisonai.scheduler or praisonaiagents module not found")
|
||||
print("Please ensure PraisonAI is properly installed")
|
||||
print("pip install praisonai praisonaiagents")
|
||||
return 1
|
||||
|
||||
# Check if first arg is a YAML file or a prompt
|
||||
first_arg = unknown_args[0] if unknown_args else None
|
||||
is_yaml_mode = first_arg and (first_arg.endswith('.yaml') or first_arg.endswith('.yml'))
|
||||
|
||||
# Get overrides from CLI
|
||||
interval_override = getattr(args, 'schedule_interval', None) or 'hourly'
|
||||
max_retries_override = getattr(args, 'schedule_max_retries', None) or 3
|
||||
timeout_override = getattr(args, 'timeout', None)
|
||||
max_cost_override = getattr(args, 'max_cost', None)
|
||||
verbose = getattr(args, 'verbose', False)
|
||||
|
||||
# Set up logging - only show logs if verbose
|
||||
if verbose:
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format='[%(asctime)s] %(levelname)s - %(message)s',
|
||||
datefmt='%Y-%m-%d %H:%M:%S'
|
||||
)
|
||||
else:
|
||||
# Suppress scheduler logs in non-verbose mode
|
||||
logging.getLogger('praisonai.scheduler').setLevel(logging.WARNING)
|
||||
|
||||
try:
|
||||
if is_yaml_mode:
|
||||
# YAML mode: Load from agents.yaml
|
||||
yaml_path = first_arg
|
||||
print(f"🤖 Loading agent configuration from: {yaml_path}")
|
||||
scheduler = AgentScheduler.from_yaml(
|
||||
yaml_path=yaml_path,
|
||||
interval_override=interval_override,
|
||||
max_retries_override=max_retries_override,
|
||||
timeout_override=timeout_override,
|
||||
max_cost_override=max_cost_override
|
||||
)
|
||||
else:
|
||||
# Prompt mode: Create agent from direct prompt
|
||||
if not first_arg:
|
||||
print("❌ Error: Please provide either a YAML file or a task prompt")
|
||||
print("\nExamples:")
|
||||
print(" praisonai schedule agents.yaml")
|
||||
print(' praisonai schedule "Check news every hour" --interval hourly')
|
||||
return 1
|
||||
|
||||
task = first_arg
|
||||
|
||||
# Create agent
|
||||
agent = Agent(
|
||||
name="Scheduled Agent",
|
||||
role="Task Executor",
|
||||
goal=task,
|
||||
instructions=task,
|
||||
verbose=False
|
||||
)
|
||||
|
||||
# Create scheduler
|
||||
scheduler = AgentScheduler(
|
||||
agent=agent,
|
||||
task=task,
|
||||
timeout=timeout_override,
|
||||
max_cost=max_cost_override
|
||||
)
|
||||
|
||||
# Get configuration
|
||||
interval = interval_override
|
||||
max_retries = max_retries_override
|
||||
|
||||
if is_yaml_mode:
|
||||
schedule_config = scheduler._yaml_schedule_config
|
||||
interval = interval_override or schedule_config.get('interval', 'hourly')
|
||||
max_retries = max_retries_override or schedule_config.get('max_retries', 3)
|
||||
|
||||
print(f"Schedule: {interval}")
|
||||
print(f"Max Retries: {max_retries}")
|
||||
if scheduler.timeout:
|
||||
print(f"Timeout: {scheduler.timeout}s")
|
||||
if scheduler.max_cost:
|
||||
print(f"Budget: ${scheduler.max_cost}")
|
||||
print(f"{'='*60}\n")
|
||||
|
||||
# Start scheduler
|
||||
print("⏰ Starting scheduler... (Press Ctrl+C to stop)\n")
|
||||
if is_yaml_mode:
|
||||
scheduler.start_from_yaml_config()
|
||||
else:
|
||||
scheduler.start(
|
||||
schedule_expr=interval,
|
||||
max_retries=max_retries,
|
||||
run_immediately=True
|
||||
)
|
||||
|
||||
# Keep running until interrupted
|
||||
try:
|
||||
while scheduler.is_running:
|
||||
import time
|
||||
time.sleep(1)
|
||||
except KeyboardInterrupt:
|
||||
print("\n\n🛑 Stopping scheduler...")
|
||||
scheduler.stop()
|
||||
|
||||
# Display final stats
|
||||
stats = scheduler.get_stats()
|
||||
print("\n📊 Final Statistics:")
|
||||
print(f" Total Executions: {stats['total_executions']}")
|
||||
print(f" Successful: {stats['successful_executions']}")
|
||||
print(f" Failed: {stats['failed_executions']}")
|
||||
print(f" Success Rate: {stats['success_rate']:.1f}%")
|
||||
print("\n✅ Agent stopped successfully\n")
|
||||
|
||||
return 0
|
||||
|
||||
except FileNotFoundError as e:
|
||||
print(f"❌ Error: {e}")
|
||||
print(f"\nMake sure {yaml_path} exists in the current directory.")
|
||||
print("\nExample agents.yaml structure:")
|
||||
print("""
|
||||
framework: praisonai
|
||||
|
||||
agents:
|
||||
- name: "AI News Monitor"
|
||||
role: "News Analyst"
|
||||
instructions: "Search and summarize AI news"
|
||||
tools:
|
||||
- search_tool
|
||||
|
||||
task: "Search for latest AI news"
|
||||
|
||||
schedule:
|
||||
interval: "hourly"
|
||||
max_retries: 3
|
||||
run_immediately: true
|
||||
""")
|
||||
return 1
|
||||
|
||||
except ValueError as e:
|
||||
print(f"❌ Configuration Error: {e}")
|
||||
return 1
|
||||
|
||||
except Exception as e:
|
||||
print(f"❌ Error: {e}")
|
||||
if verbose:
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
return 1
|
||||
|
||||
@staticmethod
|
||||
def add_schedule_arguments(subparsers):
|
||||
"""
|
||||
Add schedule subcommand and arguments to parser.
|
||||
|
||||
Args:
|
||||
subparsers: Subparsers object from argparse
|
||||
"""
|
||||
schedule_parser = subparsers.add_parser(
|
||||
'schedule',
|
||||
help='Schedule an agent to run continuously at regular intervals'
|
||||
)
|
||||
|
||||
schedule_parser.add_argument(
|
||||
'schedule_yaml',
|
||||
nargs='?',
|
||||
default='agents.yaml',
|
||||
help='Path to agents.yaml file (default: agents.yaml)'
|
||||
)
|
||||
|
||||
schedule_parser.add_argument(
|
||||
'--interval',
|
||||
dest='schedule_interval',
|
||||
type=str,
|
||||
help='Override schedule interval (e.g., "hourly", "*/30m", "daily")'
|
||||
)
|
||||
|
||||
schedule_parser.add_argument(
|
||||
'--max-retries',
|
||||
dest='schedule_max_retries',
|
||||
type=int,
|
||||
help='Override maximum retry attempts (default: from YAML or 3)'
|
||||
)
|
||||
|
||||
schedule_parser.add_argument(
|
||||
'--verbose', '-v',
|
||||
action='store_true',
|
||||
help='Enable verbose logging'
|
||||
)
|
||||
|
||||
schedule_parser.set_defaults(func=AgentSchedulerHandler.handle_schedule_command)
|
||||
|
||||
|
||||
def setup_scheduler_cli(subparsers):
|
||||
"""
|
||||
Setup scheduler CLI commands.
|
||||
|
||||
Args:
|
||||
subparsers: Subparsers object from argparse
|
||||
"""
|
||||
AgentSchedulerHandler.add_schedule_arguments(subparsers)
|
||||
@@ -0,0 +1,471 @@
|
||||
"""
|
||||
@ Mention Autocomplete for PraisonAI CLI.
|
||||
|
||||
Provides file/directory autocomplete when user types @.
|
||||
Uses only stdlib imports for zero performance impact.
|
||||
|
||||
Features:
|
||||
- Detect @ trigger in input
|
||||
- Fuzzy search files/directories
|
||||
- Cache results for performance
|
||||
- Integrate with prompt_toolkit
|
||||
"""
|
||||
|
||||
import os
|
||||
import time
|
||||
import fnmatch
|
||||
from dataclasses import dataclass
|
||||
from typing import List, Optional, Dict, Tuple
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Data Classes
|
||||
# ============================================================================
|
||||
|
||||
@dataclass
|
||||
class AtMentionContext:
|
||||
"""Context for an @ mention in user input."""
|
||||
is_active: bool
|
||||
query: str
|
||||
start_pos: int
|
||||
mention_type: str = "file" # file, directory, web, etc.
|
||||
|
||||
|
||||
@dataclass
|
||||
class FileSuggestion:
|
||||
"""A file or directory suggestion."""
|
||||
path: str
|
||||
file_type: str # "file" or "directory"
|
||||
score: int = 100
|
||||
display: str = ""
|
||||
|
||||
def __post_init__(self):
|
||||
if not self.display:
|
||||
self.display = self.path
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# @ Mention Detection
|
||||
# ============================================================================
|
||||
|
||||
def detect_at_mention(text: str, cursor_pos: int) -> Optional[AtMentionContext]:
|
||||
"""
|
||||
Detect if cursor is within an @ mention context.
|
||||
|
||||
Scans backward from cursor to find @ symbol.
|
||||
Returns None if not in @ context.
|
||||
|
||||
Args:
|
||||
text: Full input text
|
||||
cursor_pos: Current cursor position
|
||||
|
||||
Returns:
|
||||
AtMentionContext if in @ mention, None otherwise
|
||||
"""
|
||||
if cursor_pos <= 0 or cursor_pos > len(text):
|
||||
return None
|
||||
|
||||
# Scan backward from cursor to find @
|
||||
at_pos = -1
|
||||
for i in range(cursor_pos - 1, -1, -1):
|
||||
char = text[i]
|
||||
|
||||
# Found @
|
||||
if char == '@':
|
||||
at_pos = i
|
||||
break
|
||||
|
||||
# Space/newline breaks @ context
|
||||
if char in ' \t\n\r':
|
||||
return None
|
||||
|
||||
if at_pos == -1:
|
||||
return None
|
||||
|
||||
# Extract query between @ and cursor
|
||||
query = text[at_pos + 1:cursor_pos]
|
||||
|
||||
return AtMentionContext(
|
||||
is_active=True,
|
||||
query=query,
|
||||
start_pos=at_pos
|
||||
)
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# File Search Service
|
||||
# ============================================================================
|
||||
|
||||
class FileSearchService:
|
||||
"""
|
||||
Service for searching files in a directory.
|
||||
|
||||
Features:
|
||||
- Walk directory tree
|
||||
- Fuzzy match file paths
|
||||
- Cache results with TTL
|
||||
- Respect .gitignore (basic)
|
||||
- Support absolute paths (@/Users/...) and home paths (@~/...)
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
root_dir: str,
|
||||
cache_ttl: int = 30,
|
||||
max_depth: int = 5,
|
||||
ignore_patterns: Optional[List[str]] = None
|
||||
):
|
||||
self._root = os.path.abspath(root_dir)
|
||||
self._cache_ttl = cache_ttl
|
||||
self._max_depth = max_depth
|
||||
self._ignore_patterns = ignore_patterns or [
|
||||
'.git', '__pycache__', 'node_modules', '.venv', 'venv',
|
||||
'*.pyc', '*.pyo', '.DS_Store', '*.egg-info'
|
||||
]
|
||||
|
||||
# Cache: query -> (results, timestamp)
|
||||
self._cache: Dict[str, Tuple[List[FileSuggestion], float]] = {}
|
||||
|
||||
# File list cache per directory
|
||||
self._file_list: Optional[List[str]] = None
|
||||
self._file_list_time: float = 0
|
||||
self._file_list_dir: Optional[str] = None
|
||||
|
||||
def _should_ignore(self, name: str) -> bool:
|
||||
"""Check if file/dir should be ignored."""
|
||||
for pattern in self._ignore_patterns:
|
||||
if fnmatch.fnmatch(name, pattern):
|
||||
return True
|
||||
return False
|
||||
|
||||
def _get_file_list(self, search_dir: str = None) -> List[str]:
|
||||
"""Get list of all files (cached)."""
|
||||
now = time.time()
|
||||
target_dir = search_dir or self._root
|
||||
|
||||
# Return cached if fresh and same directory
|
||||
if (self._file_list is not None and
|
||||
self._file_list_dir == target_dir and
|
||||
(now - self._file_list_time) < self._cache_ttl):
|
||||
return self._file_list
|
||||
|
||||
files = []
|
||||
|
||||
try:
|
||||
for root, dirs, filenames in os.walk(target_dir):
|
||||
# Calculate depth
|
||||
rel_root = os.path.relpath(root, target_dir)
|
||||
depth = 0 if rel_root == '.' else rel_root.count(os.sep) + 1
|
||||
|
||||
if depth > self._max_depth:
|
||||
dirs.clear() # Don't descend further
|
||||
continue
|
||||
|
||||
# Filter ignored directories
|
||||
dirs[:] = [d for d in dirs if not self._should_ignore(d)]
|
||||
|
||||
# Add directories
|
||||
for d in dirs:
|
||||
rel_path = os.path.relpath(os.path.join(root, d), target_dir)
|
||||
files.append(rel_path + os.sep)
|
||||
|
||||
# Add files
|
||||
for f in filenames:
|
||||
if not self._should_ignore(f):
|
||||
rel_path = os.path.relpath(os.path.join(root, f), target_dir)
|
||||
files.append(rel_path)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
self._file_list = files
|
||||
self._file_list_time = now
|
||||
self._file_list_dir = target_dir
|
||||
return files
|
||||
|
||||
def _fuzzy_match(self, text: str, query: str) -> int:
|
||||
"""
|
||||
Simple fuzzy match scoring.
|
||||
|
||||
Returns score 0-100, higher is better.
|
||||
"""
|
||||
if not query:
|
||||
return 50 # Default score for empty query
|
||||
|
||||
text_lower = text.lower()
|
||||
query_lower = query.lower()
|
||||
|
||||
# Exact match
|
||||
if text_lower == query_lower:
|
||||
return 100
|
||||
|
||||
# Starts with
|
||||
if text_lower.startswith(query_lower):
|
||||
return 90
|
||||
|
||||
# Contains
|
||||
if query_lower in text_lower:
|
||||
return 80
|
||||
|
||||
# Fuzzy: all query chars appear in order
|
||||
text_idx = 0
|
||||
query_idx = 0
|
||||
matches = 0
|
||||
|
||||
while text_idx < len(text_lower) and query_idx < len(query_lower):
|
||||
if text_lower[text_idx] == query_lower[query_idx]:
|
||||
matches += 1
|
||||
query_idx += 1
|
||||
text_idx += 1
|
||||
|
||||
if query_idx == len(query_lower):
|
||||
# All query chars found
|
||||
return 50 + int(30 * matches / len(text))
|
||||
|
||||
return 0
|
||||
|
||||
def search(self, query: str, max_results: int = 20) -> List[FileSuggestion]:
|
||||
"""
|
||||
Search for files matching query.
|
||||
|
||||
Supports:
|
||||
- Relative paths: @file.txt, @src/main.py
|
||||
- Home paths: @~/Documents, @~/.config
|
||||
- Absolute paths: @/Users/name/file.txt, @/etc/hosts
|
||||
|
||||
Args:
|
||||
query: Search query (fuzzy matched)
|
||||
max_results: Maximum results to return
|
||||
|
||||
Returns:
|
||||
List of FileSuggestion sorted by score
|
||||
"""
|
||||
# Determine search directory based on query
|
||||
search_dir = self._root
|
||||
search_query = query
|
||||
prefix = ""
|
||||
|
||||
# Handle home directory paths (@~/)
|
||||
if query.startswith('~'):
|
||||
expanded = os.path.expanduser(query)
|
||||
# Find the directory part and filename part
|
||||
if os.path.isdir(expanded):
|
||||
search_dir = expanded
|
||||
search_query = ""
|
||||
prefix = query.rstrip('/') + '/'
|
||||
else:
|
||||
parent = os.path.dirname(expanded)
|
||||
if os.path.isdir(parent):
|
||||
search_dir = parent
|
||||
search_query = os.path.basename(expanded)
|
||||
prefix = os.path.dirname(query).rstrip('/') + '/'
|
||||
else:
|
||||
# Just expand ~ and search from home
|
||||
search_dir = os.path.expanduser('~')
|
||||
search_query = query[2:] if query.startswith('~/') else query[1:]
|
||||
prefix = "~/"
|
||||
|
||||
# Handle absolute paths (@/Users/..., @/etc/...)
|
||||
elif query.startswith('/'):
|
||||
if os.path.isdir(query):
|
||||
search_dir = query
|
||||
search_query = ""
|
||||
prefix = query.rstrip('/') + '/'
|
||||
else:
|
||||
parent = os.path.dirname(query)
|
||||
if os.path.isdir(parent):
|
||||
search_dir = parent
|
||||
search_query = os.path.basename(query)
|
||||
prefix = parent.rstrip('/') + '/'
|
||||
else:
|
||||
# Search from root with the query
|
||||
search_dir = '/'
|
||||
search_query = query[1:]
|
||||
prefix = "/"
|
||||
|
||||
# Check cache
|
||||
cache_key = f"{search_dir}:{search_query}:{max_results}"
|
||||
now = time.time()
|
||||
|
||||
if cache_key in self._cache:
|
||||
results, cached_time = self._cache[cache_key]
|
||||
if (now - cached_time) < self._cache_ttl:
|
||||
return results
|
||||
|
||||
# Get all files from the appropriate directory
|
||||
all_files = self._get_file_list(search_dir)
|
||||
|
||||
# Score and filter
|
||||
scored: List[Tuple[int, str]] = []
|
||||
for path in all_files:
|
||||
score = self._fuzzy_match(path, search_query)
|
||||
if score > 0:
|
||||
scored.append((score, path))
|
||||
|
||||
# Sort by score descending
|
||||
scored.sort(key=lambda x: (-x[0], x[1]))
|
||||
|
||||
# Build results with proper prefix
|
||||
results = []
|
||||
for score, path in scored[:max_results]:
|
||||
file_type = "directory" if path.endswith(os.sep) else "file"
|
||||
display_path = prefix + path if prefix else path
|
||||
results.append(FileSuggestion(
|
||||
path=display_path,
|
||||
file_type=file_type,
|
||||
score=score
|
||||
))
|
||||
|
||||
# Cache results
|
||||
self._cache[cache_key] = (results, now)
|
||||
|
||||
return results
|
||||
|
||||
def clear_cache(self) -> None:
|
||||
"""Clear all caches."""
|
||||
self._cache.clear()
|
||||
self._file_list = None
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Completers for prompt_toolkit
|
||||
# ============================================================================
|
||||
|
||||
# Lazy import to avoid performance impact
|
||||
_Completer = None
|
||||
_Completion = None
|
||||
|
||||
def _get_prompt_toolkit():
|
||||
"""Lazy import prompt_toolkit."""
|
||||
global _Completer, _Completion
|
||||
if _Completer is None:
|
||||
from prompt_toolkit.completion import Completer, Completion
|
||||
_Completer = Completer
|
||||
_Completion = Completion
|
||||
return _Completer, _Completion
|
||||
|
||||
|
||||
class AtMentionCompleter:
|
||||
"""
|
||||
Completer for @ mentions.
|
||||
|
||||
Shows file suggestions when user types @.
|
||||
Implements prompt_toolkit Completer interface.
|
||||
"""
|
||||
|
||||
def __init__(self, root_dir: str):
|
||||
self._file_service = FileSearchService(root_dir)
|
||||
|
||||
def get_completions(self, document, complete_event):
|
||||
"""Get completions for @ mentions."""
|
||||
_, Completion = _get_prompt_toolkit()
|
||||
|
||||
text = document.text_before_cursor
|
||||
cursor_pos = len(text)
|
||||
|
||||
# Detect @ context
|
||||
context = detect_at_mention(text, cursor_pos)
|
||||
if context is None or not context.is_active:
|
||||
return
|
||||
|
||||
# Search files
|
||||
results = self._file_service.search(context.query, max_results=15)
|
||||
|
||||
# Yield completions
|
||||
for suggestion in results:
|
||||
# Icon based on type
|
||||
icon = "📁 " if suggestion.file_type == "directory" else "📄 "
|
||||
display = f"{icon}{suggestion.path}"
|
||||
|
||||
# Calculate replacement position
|
||||
# Replace from @ to cursor
|
||||
start_position = -(cursor_pos - context.start_pos)
|
||||
|
||||
yield Completion(
|
||||
text=f"@{suggestion.path}",
|
||||
start_position=start_position,
|
||||
display=display
|
||||
)
|
||||
|
||||
|
||||
def create_combined_completer(commands: List[str], root_dir: str):
|
||||
"""
|
||||
Factory function to create a CombinedCompleter.
|
||||
|
||||
Returns a proper Completer subclass that handles both / and @.
|
||||
"""
|
||||
Completer, Completion = _get_prompt_toolkit()
|
||||
file_service = FileSearchService(root_dir)
|
||||
|
||||
class CombinedCompleter(Completer):
|
||||
"""
|
||||
Combined completer for both / commands and @ mentions.
|
||||
|
||||
Properly inherits from prompt_toolkit Completer.
|
||||
"""
|
||||
|
||||
def get_completions(self, document, complete_event):
|
||||
"""Get completions for / or @."""
|
||||
text = document.text_before_cursor.lstrip()
|
||||
|
||||
# Check for slash command
|
||||
if text.startswith('/'):
|
||||
cmd_text = text[1:].lower()
|
||||
for cmd in commands:
|
||||
if cmd.lower().startswith(cmd_text):
|
||||
yield Completion(
|
||||
f'/{cmd}',
|
||||
start_position=-len(text),
|
||||
display=f'/{cmd}'
|
||||
)
|
||||
return
|
||||
|
||||
# Check for @ mention
|
||||
full_text = document.text_before_cursor
|
||||
cursor_pos = len(full_text)
|
||||
|
||||
context = detect_at_mention(full_text, cursor_pos)
|
||||
if context is None or not context.is_active:
|
||||
return
|
||||
|
||||
# Search files
|
||||
results = file_service.search(context.query, max_results=15)
|
||||
|
||||
# Yield completions
|
||||
for suggestion in results:
|
||||
icon = "📁 " if suggestion.file_type == "directory" else "📄 "
|
||||
display = f"{icon}{suggestion.path}"
|
||||
start_position = -(cursor_pos - context.start_pos)
|
||||
|
||||
yield Completion(
|
||||
text=f"@{suggestion.path}",
|
||||
start_position=start_position,
|
||||
display=display
|
||||
)
|
||||
|
||||
return CombinedCompleter()
|
||||
|
||||
|
||||
# Backward compatible alias
|
||||
class CombinedCompleter:
|
||||
"""
|
||||
Wrapper class for backward compatibility.
|
||||
|
||||
Use create_combined_completer() for proper Completer subclass.
|
||||
"""
|
||||
|
||||
def __new__(cls, commands: List[str], root_dir: str):
|
||||
return create_combined_completer(commands, root_dir)
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Exports
|
||||
# ============================================================================
|
||||
|
||||
__all__ = [
|
||||
'AtMentionContext',
|
||||
'FileSuggestion',
|
||||
'detect_at_mention',
|
||||
'FileSearchService',
|
||||
'AtMentionCompleter',
|
||||
'CombinedCompleter',
|
||||
]
|
||||
@@ -45,8 +45,10 @@ class AutoMemoryHandler(FlagHandler):
|
||||
def _get_auto_memory(self, user_id: str = None):
|
||||
"""Get AutoMemory instance lazily."""
|
||||
try:
|
||||
from praisonaiagents.memory import AutoMemory
|
||||
return AutoMemory(user_id=user_id or "default")
|
||||
from praisonaiagents.memory import AutoMemory, FileMemory
|
||||
# AutoMemory requires a FileMemory instance
|
||||
memory = FileMemory(user_id=user_id or "default")
|
||||
return AutoMemory(memory=memory)
|
||||
except ImportError:
|
||||
self.print_status(
|
||||
"AutoMemory requires praisonaiagents. Install with: pip install praisonaiagents",
|
||||
@@ -71,33 +73,36 @@ class AutoMemoryHandler(FlagHandler):
|
||||
config['auto_memory_user_id'] = flag_value.get('user_id', 'default')
|
||||
return config
|
||||
|
||||
def extract_memories(self, text: str, user_id: str = None) -> Dict[str, Any]:
|
||||
def extract_memories(self, text: str, user_id: str = None, user_message: str = None) -> list:
|
||||
"""
|
||||
Extract memories from text.
|
||||
Extract and store memories from text.
|
||||
|
||||
Args:
|
||||
text: Text to extract memories from
|
||||
text: Assistant response text to extract memories from
|
||||
user_id: User ID for memory isolation
|
||||
user_message: Original user message (for context)
|
||||
|
||||
Returns:
|
||||
Dictionary of extracted memories
|
||||
List of extracted memories
|
||||
"""
|
||||
auto_memory = self._get_auto_memory(user_id)
|
||||
if not auto_memory:
|
||||
return {}
|
||||
return []
|
||||
|
||||
try:
|
||||
if hasattr(auto_memory, 'extract'):
|
||||
memories = auto_memory.extract(text)
|
||||
self.print_status(f"🧠 Extracted {len(memories)} memories", "success")
|
||||
return memories
|
||||
elif hasattr(auto_memory, 'process'):
|
||||
result = auto_memory.process(text)
|
||||
return result
|
||||
# Use process_interaction which extracts AND stores memories
|
||||
memories = auto_memory.process_interaction(
|
||||
user_message=user_message or text,
|
||||
assistant_response=text if user_message else None,
|
||||
store=True # Store the memories
|
||||
)
|
||||
if memories:
|
||||
self.print_status(f"🧠 Extracted and stored {len(memories)} memories", "success")
|
||||
return memories
|
||||
except Exception as e:
|
||||
self.log(f"Memory extraction failed: {e}", "error")
|
||||
|
||||
return {}
|
||||
return []
|
||||
|
||||
def store_memory(self, content: str, user_id: str = None, importance: float = 0.5) -> bool:
|
||||
"""
|
||||
@@ -127,13 +132,14 @@ class AutoMemoryHandler(FlagHandler):
|
||||
self.log(f"Memory storage failed: {e}", "error")
|
||||
return False
|
||||
|
||||
def post_process_result(self, result: Any, flag_value: Any) -> Any:
|
||||
def post_process_result(self, result: Any, flag_value: Any, user_message: str = None) -> Any:
|
||||
"""
|
||||
Post-process result to extract and store memories.
|
||||
|
||||
Args:
|
||||
result: Agent output
|
||||
flag_value: Boolean or dict with configuration
|
||||
user_message: Original user message for context
|
||||
|
||||
Returns:
|
||||
Original result (memories are stored)
|
||||
@@ -144,15 +150,18 @@ class AutoMemoryHandler(FlagHandler):
|
||||
user_id = None
|
||||
if isinstance(flag_value, dict):
|
||||
user_id = flag_value.get('user_id')
|
||||
user_message = flag_value.get('user_message', user_message)
|
||||
|
||||
# Extract memories from result
|
||||
# Extract and store memories from result
|
||||
text = str(result)
|
||||
memories = self.extract_memories(text, user_id)
|
||||
memories = self.extract_memories(text, user_id, user_message=user_message)
|
||||
|
||||
if memories:
|
||||
self.print_status("\n🧠 Auto-extracted Memories:", "info")
|
||||
for key, value in memories.items():
|
||||
self.print_status(f" • {key}: {value[:50]}...", "info")
|
||||
for mem in memories:
|
||||
mem_type = mem.get('type', 'unknown')
|
||||
content = mem.get('content', str(mem))[:50]
|
||||
self.print_status(f" • {mem_type}: {content}...", "info")
|
||||
|
||||
return result
|
||||
|
||||
|
||||
@@ -0,0 +1,468 @@
|
||||
"""
|
||||
CLI Compare Feature for PraisonAI.
|
||||
|
||||
Compares different CLI command modes without performance impact.
|
||||
Only loaded when --compare flag is used.
|
||||
"""
|
||||
|
||||
import time
|
||||
import json
|
||||
import logging
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Dict, Any, List, Optional
|
||||
from datetime import datetime
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
COMPARE_MODES = {
|
||||
"basic": {},
|
||||
"tools": {"tools": "internet_search"},
|
||||
"research": {"research": True},
|
||||
"planning": {"planning": True},
|
||||
"memory": {"memory": True},
|
||||
"router": {"router": True},
|
||||
"web_search": {"web_search": True},
|
||||
"web_fetch": {"web_fetch": True},
|
||||
"query_rewrite": {"query_rewrite": True},
|
||||
"expand_prompt": {"expand_prompt": True},
|
||||
}
|
||||
|
||||
|
||||
def get_mode_config(mode_name: str, custom_args: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
|
||||
"""
|
||||
Get configuration for a comparison mode.
|
||||
|
||||
Args:
|
||||
mode_name: Name of the mode
|
||||
custom_args: Optional custom arguments to merge
|
||||
|
||||
Returns:
|
||||
Configuration dictionary for the mode
|
||||
"""
|
||||
config = COMPARE_MODES.get(mode_name, {}).copy()
|
||||
if custom_args:
|
||||
config.update(custom_args)
|
||||
return config
|
||||
|
||||
|
||||
def list_available_modes() -> List[str]:
|
||||
"""
|
||||
List all available comparison modes.
|
||||
|
||||
Returns:
|
||||
List of mode names
|
||||
"""
|
||||
return list(COMPARE_MODES.keys())
|
||||
|
||||
|
||||
def parse_modes(modes_str: str) -> List[str]:
|
||||
"""
|
||||
Parse comma-separated modes string.
|
||||
|
||||
Args:
|
||||
modes_str: Comma-separated mode names
|
||||
|
||||
Returns:
|
||||
List of mode names
|
||||
"""
|
||||
if not modes_str or not modes_str.strip():
|
||||
return ["basic"]
|
||||
|
||||
modes = [m.strip() for m in modes_str.split(",")]
|
||||
return [m for m in modes if m]
|
||||
|
||||
|
||||
@dataclass
|
||||
class ModeResult:
|
||||
"""Result from running a single mode."""
|
||||
mode: str
|
||||
output: str
|
||||
execution_time_ms: float
|
||||
model_used: str
|
||||
tokens: Optional[Dict[str, int]] = None
|
||||
cost: Optional[float] = None
|
||||
tools_used: List[str] = field(default_factory=list)
|
||||
error: Optional[str] = None
|
||||
timestamp: str = field(default_factory=lambda: datetime.now().isoformat())
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
"""Convert to dictionary."""
|
||||
return {
|
||||
"mode": self.mode,
|
||||
"output_preview": self.output[:200] + "..." if len(self.output) > 200 else self.output,
|
||||
"execution_time_ms": self.execution_time_ms,
|
||||
"model_used": self.model_used,
|
||||
"tokens": self.tokens,
|
||||
"cost": self.cost,
|
||||
"tools_used": self.tools_used,
|
||||
"error": self.error,
|
||||
"timestamp": self.timestamp,
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class CompareResult:
|
||||
"""Result from comparing multiple modes."""
|
||||
query: str
|
||||
comparisons: List[ModeResult] = field(default_factory=list)
|
||||
timestamp: str = field(default_factory=lambda: datetime.now().isoformat())
|
||||
|
||||
def get_summary(self) -> Dict[str, Any]:
|
||||
"""Generate summary statistics."""
|
||||
if not self.comparisons:
|
||||
return {}
|
||||
|
||||
times = [(c.mode, c.execution_time_ms) for c in self.comparisons if c.error is None]
|
||||
if not times:
|
||||
return {"error": "All modes failed"}
|
||||
|
||||
sorted_by_time = sorted(times, key=lambda x: x[1])
|
||||
|
||||
summary = {
|
||||
"fastest": sorted_by_time[0][0],
|
||||
"slowest": sorted_by_time[-1][0],
|
||||
"fastest_time_ms": sorted_by_time[0][1],
|
||||
"slowest_time_ms": sorted_by_time[-1][1],
|
||||
}
|
||||
|
||||
total_tokens = 0
|
||||
total_cost = 0.0
|
||||
for c in self.comparisons:
|
||||
if c.tokens:
|
||||
total_tokens += c.tokens.get("input", 0) + c.tokens.get("output", 0)
|
||||
if c.cost:
|
||||
total_cost += c.cost
|
||||
|
||||
if total_tokens > 0:
|
||||
summary["total_tokens"] = total_tokens
|
||||
if total_cost > 0:
|
||||
summary["total_cost"] = total_cost
|
||||
|
||||
return summary
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
"""Convert to dictionary."""
|
||||
return {
|
||||
"query": self.query,
|
||||
"timestamp": self.timestamp,
|
||||
"comparisons": [c.to_dict() for c in self.comparisons],
|
||||
"summary": self.get_summary(),
|
||||
}
|
||||
|
||||
def to_json(self) -> str:
|
||||
"""Convert to JSON string."""
|
||||
return json.dumps(self.to_dict(), indent=2)
|
||||
|
||||
|
||||
def save_compare_result(result: CompareResult, path: str) -> bool:
|
||||
"""
|
||||
Save comparison result to file.
|
||||
|
||||
Args:
|
||||
result: CompareResult to save
|
||||
path: File path
|
||||
|
||||
Returns:
|
||||
True if successful
|
||||
"""
|
||||
try:
|
||||
from pathlib import Path
|
||||
|
||||
file_path = Path(path)
|
||||
file_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
file_path.write_text(result.to_json())
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to save comparison result: {e}")
|
||||
return False
|
||||
|
||||
|
||||
def format_comparison_table(result: CompareResult) -> str:
|
||||
"""
|
||||
Format comparison result as a table string.
|
||||
|
||||
Args:
|
||||
result: CompareResult to format
|
||||
|
||||
Returns:
|
||||
Formatted table string
|
||||
"""
|
||||
try:
|
||||
from rich.console import Console
|
||||
from rich.table import Table
|
||||
from io import StringIO
|
||||
|
||||
console = Console(file=StringIO(), force_terminal=True)
|
||||
|
||||
table = Table(title=f"Comparison: {result.query[:50]}...")
|
||||
table.add_column("Mode", style="cyan")
|
||||
table.add_column("Time (ms)", style="green")
|
||||
table.add_column("Model", style="yellow")
|
||||
table.add_column("Tools", style="magenta")
|
||||
table.add_column("Status", style="bold")
|
||||
|
||||
for c in result.comparisons:
|
||||
status = "✅" if c.error is None else f"❌ {c.error[:20]}"
|
||||
tools = ", ".join(c.tools_used) if c.tools_used else "-"
|
||||
table.add_row(
|
||||
c.mode,
|
||||
f"{c.execution_time_ms:.1f}",
|
||||
c.model_used,
|
||||
tools,
|
||||
status
|
||||
)
|
||||
|
||||
summary = result.get_summary()
|
||||
if summary.get("fastest"):
|
||||
table.add_section()
|
||||
table.add_row(
|
||||
"Summary",
|
||||
f"Fastest: {summary['fastest']}",
|
||||
"",
|
||||
"",
|
||||
f"Δ {summary['slowest_time_ms'] - summary['fastest_time_ms']:.1f}ms"
|
||||
)
|
||||
|
||||
console.print(table)
|
||||
return console.file.getvalue()
|
||||
except ImportError:
|
||||
lines = [f"Comparison: {result.query}"]
|
||||
lines.append("-" * 60)
|
||||
for c in result.comparisons:
|
||||
status = "OK" if c.error is None else f"ERROR: {c.error}"
|
||||
lines.append(f"{c.mode}: {c.execution_time_ms:.1f}ms - {status}")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
class CompareHandler:
|
||||
"""
|
||||
Handler for CLI compare feature.
|
||||
|
||||
Compares different CLI command modes and generates comparison reports.
|
||||
"""
|
||||
|
||||
def __init__(self, verbose: bool = False):
|
||||
"""
|
||||
Initialize the compare handler.
|
||||
|
||||
Args:
|
||||
verbose: Enable verbose output
|
||||
"""
|
||||
self.verbose = verbose
|
||||
self.results: List[ModeResult] = []
|
||||
|
||||
def compare(
|
||||
self,
|
||||
query: str,
|
||||
modes: List[str],
|
||||
model: Optional[str] = None,
|
||||
**kwargs
|
||||
) -> CompareResult:
|
||||
"""
|
||||
Compare query execution across different modes.
|
||||
|
||||
Args:
|
||||
query: The query to execute
|
||||
modes: List of mode names to compare
|
||||
model: Optional model override
|
||||
**kwargs: Additional arguments
|
||||
|
||||
Returns:
|
||||
CompareResult with all comparisons
|
||||
"""
|
||||
result = CompareResult(query=query)
|
||||
|
||||
for mode in modes:
|
||||
if self.verbose:
|
||||
logger.info(f"Running mode: {mode}")
|
||||
|
||||
try:
|
||||
mode_result = self._run_mode(query, mode, model=model, **kwargs)
|
||||
result.comparisons.append(mode_result)
|
||||
except Exception as e:
|
||||
logger.error(f"Error running mode {mode}: {e}")
|
||||
result.comparisons.append(ModeResult(
|
||||
mode=mode,
|
||||
output="",
|
||||
execution_time_ms=0,
|
||||
model_used=model or "unknown",
|
||||
error=str(e)
|
||||
))
|
||||
|
||||
return result
|
||||
|
||||
def _run_mode(
|
||||
self,
|
||||
query: str,
|
||||
mode: str,
|
||||
model: Optional[str] = None,
|
||||
**kwargs
|
||||
) -> ModeResult:
|
||||
"""
|
||||
Run a single mode and capture results.
|
||||
|
||||
Args:
|
||||
query: The query to execute
|
||||
mode: Mode name
|
||||
model: Optional model override
|
||||
**kwargs: Additional arguments
|
||||
|
||||
Returns:
|
||||
ModeResult with execution details
|
||||
"""
|
||||
config = get_mode_config(mode, kwargs)
|
||||
|
||||
if model:
|
||||
config["llm"] = model
|
||||
|
||||
start_time = time.perf_counter()
|
||||
output = ""
|
||||
tokens = None
|
||||
cost = None
|
||||
tools_used = []
|
||||
error = None
|
||||
model_used = model or "gpt-4o-mini"
|
||||
|
||||
try:
|
||||
from praisonaiagents import Agent
|
||||
|
||||
agent_config = {
|
||||
"name": f"CompareAgent_{mode}",
|
||||
"role": "Assistant",
|
||||
"goal": "Complete the given task",
|
||||
"backstory": "You are a helpful AI assistant",
|
||||
"verbose": False,
|
||||
}
|
||||
|
||||
if config.get("llm"):
|
||||
agent_config["llm"] = config["llm"]
|
||||
model_used = config["llm"]
|
||||
|
||||
if config.get("tools"):
|
||||
tools_list = self._load_tools(config["tools"])
|
||||
if tools_list:
|
||||
agent_config["tools"] = tools_list
|
||||
|
||||
if config.get("planning"):
|
||||
agent_config["planning"] = True
|
||||
|
||||
if config.get("memory"):
|
||||
agent_config["memory"] = True
|
||||
|
||||
if config.get("web_search"):
|
||||
agent_config["web_search"] = True
|
||||
|
||||
if config.get("web_fetch"):
|
||||
agent_config["web_fetch"] = True
|
||||
|
||||
agent = Agent(**agent_config)
|
||||
result = agent.start(query)
|
||||
|
||||
if hasattr(result, 'raw'):
|
||||
output = str(result.raw)
|
||||
else:
|
||||
output = str(result)
|
||||
|
||||
if hasattr(agent, 'tools') and agent.tools:
|
||||
for tool in agent.tools:
|
||||
if hasattr(tool, '__name__'):
|
||||
tools_used.append(tool.__name__)
|
||||
elif hasattr(tool, 'name'):
|
||||
tools_used.append(tool.name)
|
||||
|
||||
except Exception as e:
|
||||
error = str(e)
|
||||
logger.error(f"Mode {mode} failed: {e}")
|
||||
|
||||
end_time = time.perf_counter()
|
||||
execution_time_ms = (end_time - start_time) * 1000
|
||||
|
||||
return ModeResult(
|
||||
mode=mode,
|
||||
output=output,
|
||||
execution_time_ms=execution_time_ms,
|
||||
model_used=model_used,
|
||||
tokens=tokens,
|
||||
cost=cost,
|
||||
tools_used=tools_used,
|
||||
error=error
|
||||
)
|
||||
|
||||
def _load_tools(self, tools_str: str) -> List:
|
||||
"""
|
||||
Load tools from string specification.
|
||||
|
||||
Args:
|
||||
tools_str: Comma-separated tool names
|
||||
|
||||
Returns:
|
||||
List of tool functions
|
||||
"""
|
||||
tools = []
|
||||
tool_names = [t.strip() for t in tools_str.split(",")]
|
||||
|
||||
for name in tool_names:
|
||||
try:
|
||||
if name == "internet_search":
|
||||
from praisonaiagents.tools import internet_search
|
||||
tools.append(internet_search)
|
||||
elif name == "calculator":
|
||||
def calculator(expression: str) -> str:
|
||||
"""Evaluate a mathematical expression."""
|
||||
try:
|
||||
return str(eval(expression))
|
||||
except Exception as e:
|
||||
return f"Error: {e}"
|
||||
tools.append(calculator)
|
||||
except ImportError:
|
||||
logger.warning(f"Could not load tool: {name}")
|
||||
|
||||
return tools
|
||||
|
||||
def print_result(self, result: CompareResult) -> None:
|
||||
"""
|
||||
Print comparison result to console.
|
||||
|
||||
Args:
|
||||
result: CompareResult to print
|
||||
"""
|
||||
table_str = format_comparison_table(result)
|
||||
print(table_str)
|
||||
|
||||
def execute(
|
||||
self,
|
||||
query: str,
|
||||
modes_str: str,
|
||||
model: Optional[str] = None,
|
||||
output_path: Optional[str] = None,
|
||||
**kwargs
|
||||
) -> CompareResult:
|
||||
"""
|
||||
Execute comparison from CLI arguments.
|
||||
|
||||
Args:
|
||||
query: The query to execute
|
||||
modes_str: Comma-separated mode names
|
||||
model: Optional model override
|
||||
output_path: Optional path to save results
|
||||
**kwargs: Additional arguments
|
||||
|
||||
Returns:
|
||||
CompareResult with all comparisons
|
||||
"""
|
||||
modes = parse_modes(modes_str)
|
||||
|
||||
if self.verbose:
|
||||
print(f"[bold cyan]Comparing modes: {', '.join(modes)}[/bold cyan]")
|
||||
|
||||
result = self.compare(query, modes, model=model, **kwargs)
|
||||
|
||||
self.print_result(result)
|
||||
|
||||
if output_path:
|
||||
if save_compare_result(result, output_path):
|
||||
print(f"[green]Results saved to: {output_path}[/green]")
|
||||
else:
|
||||
print(f"[red]Failed to save results to: {output_path}[/red]")
|
||||
|
||||
return result
|
||||
@@ -0,0 +1,509 @@
|
||||
"""
|
||||
Evaluation CLI feature for PraisonAI.
|
||||
|
||||
Provides CLI commands for running agent evaluations.
|
||||
"""
|
||||
|
||||
import os
|
||||
import json
|
||||
import logging
|
||||
from typing import Optional, List, Dict, Any
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class EvalHandler:
|
||||
"""Handler for evaluation CLI commands."""
|
||||
|
||||
def __init__(self, verbose: bool = False):
|
||||
"""
|
||||
Initialize the evaluation handler.
|
||||
|
||||
Args:
|
||||
verbose: Enable verbose output
|
||||
"""
|
||||
self.verbose = verbose
|
||||
|
||||
def run_accuracy(
|
||||
self,
|
||||
agent_file: Optional[str] = None,
|
||||
input_text: str = "",
|
||||
expected_output: str = "",
|
||||
iterations: int = 1,
|
||||
model: Optional[str] = None,
|
||||
output_file: Optional[str] = None,
|
||||
prompt: Optional[str] = None,
|
||||
llm: Optional[str] = None
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Run accuracy evaluation on an agent.
|
||||
|
||||
Args:
|
||||
agent_file: Path to agents.yaml file (optional if prompt is provided)
|
||||
input_text: Input to provide to the agent
|
||||
expected_output: Expected output to compare against
|
||||
iterations: Number of evaluation iterations
|
||||
model: LLM model for judging
|
||||
output_file: Path to save results
|
||||
prompt: Direct prompt (alternative to agent_file)
|
||||
llm: LLM model for the agent (when using prompt)
|
||||
|
||||
Returns:
|
||||
Evaluation result dictionary
|
||||
"""
|
||||
try:
|
||||
from praisonaiagents.eval import AccuracyEvaluator
|
||||
from praisonaiagents import Agent
|
||||
except ImportError as e:
|
||||
logger.error(f"Failed to import evaluation modules: {e}")
|
||||
return {"error": str(e)}
|
||||
|
||||
try:
|
||||
# Create agent either from file or from prompt
|
||||
if prompt:
|
||||
# Direct prompt mode - create agent on the fly
|
||||
agent = Agent(
|
||||
name="EvalAgent",
|
||||
role="Assistant",
|
||||
goal="Complete the given task",
|
||||
backstory="You are a helpful assistant.",
|
||||
llm=llm or model or "gpt-4o-mini",
|
||||
verbose=False
|
||||
)
|
||||
# Use prompt as input if input_text not provided
|
||||
if not input_text:
|
||||
input_text = prompt
|
||||
elif agent_file:
|
||||
# Load from agents.yaml
|
||||
try:
|
||||
from praisonai.agents_generator import AgentsGenerator
|
||||
generator = AgentsGenerator(agent_file)
|
||||
agents = generator.generate_agents()
|
||||
|
||||
if not agents:
|
||||
return {"error": "No agents found in configuration"}
|
||||
|
||||
agent = agents[0] if isinstance(agents, list) else agents
|
||||
except Exception as e:
|
||||
return {"error": f"Failed to load agents from {agent_file}: {e}"}
|
||||
else:
|
||||
return {"error": "Either --agent or --prompt must be provided"}
|
||||
|
||||
evaluator = AccuracyEvaluator(
|
||||
agent=agent,
|
||||
input_text=input_text,
|
||||
expected_output=expected_output,
|
||||
num_iterations=iterations,
|
||||
model=model,
|
||||
save_results_path=output_file,
|
||||
verbose=self.verbose
|
||||
)
|
||||
|
||||
result = evaluator.run(print_summary=True)
|
||||
return result.to_dict()
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Accuracy evaluation failed: {e}")
|
||||
return {"error": str(e)}
|
||||
|
||||
def run_performance(
|
||||
self,
|
||||
agent_file: str,
|
||||
input_text: str = "Hello",
|
||||
iterations: int = 10,
|
||||
warmup: int = 2,
|
||||
track_memory: bool = True,
|
||||
output_file: Optional[str] = None
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Run performance evaluation on an agent.
|
||||
|
||||
Args:
|
||||
agent_file: Path to agents.yaml file
|
||||
input_text: Input to provide to the agent
|
||||
iterations: Number of benchmark iterations
|
||||
warmup: Number of warmup runs
|
||||
track_memory: Whether to track memory usage
|
||||
output_file: Path to save results
|
||||
|
||||
Returns:
|
||||
Evaluation result dictionary
|
||||
"""
|
||||
try:
|
||||
from praisonaiagents.eval import PerformanceEvaluator
|
||||
from praisonai.agents_generator import AgentsGenerator
|
||||
except ImportError as e:
|
||||
logger.error(f"Failed to import evaluation modules: {e}")
|
||||
return {"error": str(e)}
|
||||
|
||||
try:
|
||||
generator = AgentsGenerator(agent_file)
|
||||
agents = generator.generate_agents()
|
||||
|
||||
if not agents:
|
||||
return {"error": "No agents found in configuration"}
|
||||
|
||||
agent = agents[0] if isinstance(agents, list) else agents
|
||||
|
||||
evaluator = PerformanceEvaluator(
|
||||
agent=agent,
|
||||
input_text=input_text,
|
||||
num_iterations=iterations,
|
||||
warmup_runs=warmup,
|
||||
track_memory=track_memory,
|
||||
save_results_path=output_file,
|
||||
verbose=self.verbose
|
||||
)
|
||||
|
||||
result = evaluator.run(print_summary=True)
|
||||
return result.to_dict()
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Performance evaluation failed: {e}")
|
||||
return {"error": str(e)}
|
||||
|
||||
def run_reliability(
|
||||
self,
|
||||
agent_file: str,
|
||||
input_text: str,
|
||||
expected_tools: List[str],
|
||||
forbidden_tools: Optional[List[str]] = None,
|
||||
output_file: Optional[str] = None
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Run reliability evaluation on an agent.
|
||||
|
||||
Args:
|
||||
agent_file: Path to agents.yaml file
|
||||
input_text: Input to provide to the agent
|
||||
expected_tools: List of tools that should be called
|
||||
forbidden_tools: List of tools that should NOT be called
|
||||
output_file: Path to save results
|
||||
|
||||
Returns:
|
||||
Evaluation result dictionary
|
||||
"""
|
||||
try:
|
||||
from praisonaiagents.eval import ReliabilityEvaluator
|
||||
from praisonai.agents_generator import AgentsGenerator
|
||||
except ImportError as e:
|
||||
logger.error(f"Failed to import evaluation modules: {e}")
|
||||
return {"error": str(e)}
|
||||
|
||||
try:
|
||||
generator = AgentsGenerator(agent_file)
|
||||
agents = generator.generate_agents()
|
||||
|
||||
if not agents:
|
||||
return {"error": "No agents found in configuration"}
|
||||
|
||||
agent = agents[0] if isinstance(agents, list) else agents
|
||||
|
||||
evaluator = ReliabilityEvaluator(
|
||||
agent=agent,
|
||||
input_text=input_text,
|
||||
expected_tools=expected_tools,
|
||||
forbidden_tools=forbidden_tools,
|
||||
save_results_path=output_file,
|
||||
verbose=self.verbose
|
||||
)
|
||||
|
||||
result = evaluator.run(print_summary=True)
|
||||
return result.to_dict()
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Reliability evaluation failed: {e}")
|
||||
return {"error": str(e)}
|
||||
|
||||
def run_criteria(
|
||||
self,
|
||||
agent_file: str,
|
||||
input_text: str,
|
||||
criteria: str,
|
||||
scoring_type: str = "numeric",
|
||||
threshold: float = 7.0,
|
||||
iterations: int = 1,
|
||||
model: Optional[str] = None,
|
||||
output_file: Optional[str] = None
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Run criteria-based evaluation on an agent.
|
||||
|
||||
Args:
|
||||
agent_file: Path to agents.yaml file
|
||||
input_text: Input to provide to the agent
|
||||
criteria: Criteria to evaluate against
|
||||
scoring_type: "numeric" or "binary"
|
||||
threshold: Score threshold for passing (numeric mode)
|
||||
iterations: Number of evaluation iterations
|
||||
model: LLM model for judging
|
||||
output_file: Path to save results
|
||||
|
||||
Returns:
|
||||
Evaluation result dictionary
|
||||
"""
|
||||
try:
|
||||
from praisonaiagents.eval import CriteriaEvaluator
|
||||
from praisonai.agents_generator import AgentsGenerator
|
||||
except ImportError as e:
|
||||
logger.error(f"Failed to import evaluation modules: {e}")
|
||||
return {"error": str(e)}
|
||||
|
||||
try:
|
||||
generator = AgentsGenerator(agent_file)
|
||||
agents = generator.generate_agents()
|
||||
|
||||
if not agents:
|
||||
return {"error": "No agents found in configuration"}
|
||||
|
||||
agent = agents[0] if isinstance(agents, list) else agents
|
||||
|
||||
evaluator = CriteriaEvaluator(
|
||||
criteria=criteria,
|
||||
agent=agent,
|
||||
input_text=input_text,
|
||||
scoring_type=scoring_type,
|
||||
threshold=threshold,
|
||||
num_iterations=iterations,
|
||||
model=model,
|
||||
save_results_path=output_file,
|
||||
verbose=self.verbose
|
||||
)
|
||||
|
||||
result = evaluator.run(print_summary=True)
|
||||
return result.to_dict()
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Criteria evaluation failed: {e}")
|
||||
return {"error": str(e)}
|
||||
|
||||
def run_batch(
|
||||
self,
|
||||
agent_file: str,
|
||||
test_file: str,
|
||||
eval_type: str = "accuracy",
|
||||
output_file: Optional[str] = None
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Run batch evaluation from a test file.
|
||||
|
||||
Args:
|
||||
agent_file: Path to agents.yaml file
|
||||
test_file: Path to JSON test file with test cases
|
||||
eval_type: Type of evaluation ("accuracy", "criteria")
|
||||
output_file: Path to save results
|
||||
|
||||
Returns:
|
||||
Batch evaluation results
|
||||
"""
|
||||
try:
|
||||
with open(test_file, 'r') as f:
|
||||
test_cases = json.load(f)
|
||||
except Exception as e:
|
||||
return {"error": f"Failed to load test file: {e}"}
|
||||
|
||||
results = []
|
||||
for i, test_case in enumerate(test_cases):
|
||||
if self.verbose:
|
||||
print(f"Running test case {i + 1}/{len(test_cases)}")
|
||||
|
||||
if eval_type == "accuracy":
|
||||
result = self.run_accuracy(
|
||||
agent_file=agent_file,
|
||||
input_text=test_case.get("input", ""),
|
||||
expected_output=test_case.get("expected", ""),
|
||||
iterations=test_case.get("iterations", 1)
|
||||
)
|
||||
elif eval_type == "criteria":
|
||||
result = self.run_criteria(
|
||||
agent_file=agent_file,
|
||||
input_text=test_case.get("input", ""),
|
||||
criteria=test_case.get("criteria", ""),
|
||||
scoring_type=test_case.get("scoring_type", "numeric"),
|
||||
threshold=test_case.get("threshold", 7.0)
|
||||
)
|
||||
else:
|
||||
result = {"error": f"Unknown eval type: {eval_type}"}
|
||||
|
||||
results.append({
|
||||
"test_case": i + 1,
|
||||
"input": test_case.get("input", ""),
|
||||
"result": result
|
||||
})
|
||||
|
||||
batch_result = {
|
||||
"total_tests": len(test_cases),
|
||||
"eval_type": eval_type,
|
||||
"results": results
|
||||
}
|
||||
|
||||
if output_file:
|
||||
try:
|
||||
with open(output_file, 'w') as f:
|
||||
json.dump(batch_result, f, indent=2)
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to save batch results: {e}")
|
||||
|
||||
return batch_result
|
||||
|
||||
|
||||
def handle_eval_command(args) -> None:
|
||||
"""
|
||||
Handle the eval CLI command.
|
||||
|
||||
Args:
|
||||
args: Parsed command line arguments
|
||||
"""
|
||||
handler = EvalHandler(verbose=getattr(args, 'verbose', False))
|
||||
|
||||
eval_type = getattr(args, 'eval_type', 'accuracy')
|
||||
agent_file = getattr(args, 'agent', None)
|
||||
output_file = getattr(args, 'output', None)
|
||||
prompt = getattr(args, 'prompt', None)
|
||||
llm = getattr(args, 'llm', None)
|
||||
|
||||
# If no agent file and no prompt, check if agents.yaml exists
|
||||
if not agent_file and not prompt:
|
||||
import os
|
||||
if os.path.exists('agents.yaml'):
|
||||
agent_file = 'agents.yaml'
|
||||
|
||||
if eval_type == 'accuracy':
|
||||
result = handler.run_accuracy(
|
||||
agent_file=agent_file,
|
||||
input_text=getattr(args, 'input', ''),
|
||||
expected_output=getattr(args, 'expected', ''),
|
||||
iterations=getattr(args, 'iterations', 1),
|
||||
model=getattr(args, 'model', None),
|
||||
output_file=output_file,
|
||||
prompt=prompt,
|
||||
llm=llm
|
||||
)
|
||||
elif eval_type == 'performance':
|
||||
result = handler.run_performance(
|
||||
agent_file=agent_file,
|
||||
input_text=getattr(args, 'input', 'Hello'),
|
||||
iterations=getattr(args, 'iterations', 10),
|
||||
warmup=getattr(args, 'warmup', 2),
|
||||
track_memory=getattr(args, 'memory', True),
|
||||
output_file=output_file
|
||||
)
|
||||
elif eval_type == 'reliability':
|
||||
expected_tools = getattr(args, 'expected_tools', '').split(',')
|
||||
forbidden_tools = getattr(args, 'forbidden_tools', '')
|
||||
forbidden_tools = forbidden_tools.split(',') if forbidden_tools else None
|
||||
|
||||
result = handler.run_reliability(
|
||||
agent_file=agent_file,
|
||||
input_text=getattr(args, 'input', ''),
|
||||
expected_tools=expected_tools,
|
||||
forbidden_tools=forbidden_tools,
|
||||
output_file=output_file
|
||||
)
|
||||
elif eval_type == 'criteria':
|
||||
result = handler.run_criteria(
|
||||
agent_file=agent_file,
|
||||
input_text=getattr(args, 'input', ''),
|
||||
criteria=getattr(args, 'criteria', ''),
|
||||
scoring_type=getattr(args, 'scoring', 'numeric'),
|
||||
threshold=getattr(args, 'threshold', 7.0),
|
||||
iterations=getattr(args, 'iterations', 1),
|
||||
model=getattr(args, 'model', None),
|
||||
output_file=output_file
|
||||
)
|
||||
elif eval_type == 'batch':
|
||||
result = handler.run_batch(
|
||||
agent_file=agent_file,
|
||||
test_file=getattr(args, 'test_file', ''),
|
||||
eval_type=getattr(args, 'batch_type', 'accuracy'),
|
||||
output_file=output_file
|
||||
)
|
||||
else:
|
||||
print(f"Unknown evaluation type: {eval_type}")
|
||||
return
|
||||
|
||||
if 'error' in result:
|
||||
print(f"Error: {result['error']}")
|
||||
elif not getattr(args, 'quiet', False):
|
||||
print(json.dumps(result, indent=2))
|
||||
|
||||
|
||||
def add_eval_parser(subparsers) -> None:
|
||||
"""
|
||||
Add eval subcommand parser.
|
||||
|
||||
Args:
|
||||
subparsers: Argument parser subparsers
|
||||
"""
|
||||
eval_parser = subparsers.add_parser(
|
||||
'eval',
|
||||
help='Run agent evaluations'
|
||||
)
|
||||
|
||||
eval_subparsers = eval_parser.add_subparsers(dest='eval_type')
|
||||
|
||||
accuracy_parser = eval_subparsers.add_parser(
|
||||
'accuracy',
|
||||
help='Run accuracy evaluation'
|
||||
)
|
||||
accuracy_parser.add_argument('--agent', '-a', help='Agent config file (optional if --prompt used)')
|
||||
accuracy_parser.add_argument('--prompt', '-p', help='Direct prompt (alternative to --agent)')
|
||||
accuracy_parser.add_argument('--llm', help='LLM model for agent (when using --prompt)')
|
||||
accuracy_parser.add_argument('--input', '-i', help='Input text (defaults to --prompt if not provided)')
|
||||
accuracy_parser.add_argument('--expected', '-e', required=True, help='Expected output')
|
||||
accuracy_parser.add_argument('--iterations', '-n', type=int, default=1, help='Number of iterations')
|
||||
accuracy_parser.add_argument('--model', '-m', help='Judge model')
|
||||
accuracy_parser.add_argument('--output', '-o', help='Output file')
|
||||
accuracy_parser.add_argument('--verbose', '-v', action='store_true', help='Verbose output')
|
||||
accuracy_parser.add_argument('--quiet', '-q', action='store_true', help='Suppress JSON output')
|
||||
|
||||
perf_parser = eval_subparsers.add_parser(
|
||||
'performance',
|
||||
help='Run performance evaluation'
|
||||
)
|
||||
perf_parser.add_argument('--agent', '-a', default='agents.yaml', help='Agent config file')
|
||||
perf_parser.add_argument('--input', '-i', default='Hello', help='Input text')
|
||||
perf_parser.add_argument('--iterations', '-n', type=int, default=10, help='Number of iterations')
|
||||
perf_parser.add_argument('--warmup', '-w', type=int, default=2, help='Warmup runs')
|
||||
perf_parser.add_argument('--memory', action='store_true', default=True, help='Track memory')
|
||||
perf_parser.add_argument('--output', '-o', help='Output file')
|
||||
perf_parser.add_argument('--verbose', '-v', action='store_true', help='Verbose output')
|
||||
perf_parser.add_argument('--quiet', '-q', action='store_true', help='Suppress JSON output')
|
||||
|
||||
rel_parser = eval_subparsers.add_parser(
|
||||
'reliability',
|
||||
help='Run reliability evaluation'
|
||||
)
|
||||
rel_parser.add_argument('--agent', '-a', default='agents.yaml', help='Agent config file')
|
||||
rel_parser.add_argument('--input', '-i', required=True, help='Input text')
|
||||
rel_parser.add_argument('--expected-tools', '-t', required=True, help='Expected tools (comma-separated)')
|
||||
rel_parser.add_argument('--forbidden-tools', '-f', help='Forbidden tools (comma-separated)')
|
||||
rel_parser.add_argument('--output', '-o', help='Output file')
|
||||
rel_parser.add_argument('--verbose', '-v', action='store_true', help='Verbose output')
|
||||
rel_parser.add_argument('--quiet', '-q', action='store_true', help='Suppress JSON output')
|
||||
|
||||
criteria_parser = eval_subparsers.add_parser(
|
||||
'criteria',
|
||||
help='Run criteria-based evaluation'
|
||||
)
|
||||
criteria_parser.add_argument('--agent', '-a', default='agents.yaml', help='Agent config file')
|
||||
criteria_parser.add_argument('--input', '-i', required=True, help='Input text')
|
||||
criteria_parser.add_argument('--criteria', '-c', required=True, help='Evaluation criteria')
|
||||
criteria_parser.add_argument('--scoring', '-s', choices=['numeric', 'binary'], default='numeric', help='Scoring type')
|
||||
criteria_parser.add_argument('--threshold', type=float, default=7.0, help='Pass threshold')
|
||||
criteria_parser.add_argument('--iterations', '-n', type=int, default=1, help='Number of iterations')
|
||||
criteria_parser.add_argument('--model', '-m', help='Judge model')
|
||||
criteria_parser.add_argument('--output', '-o', help='Output file')
|
||||
criteria_parser.add_argument('--verbose', '-v', action='store_true', help='Verbose output')
|
||||
criteria_parser.add_argument('--quiet', '-q', action='store_true', help='Suppress JSON output')
|
||||
|
||||
batch_parser = eval_subparsers.add_parser(
|
||||
'batch',
|
||||
help='Run batch evaluation from test file'
|
||||
)
|
||||
batch_parser.add_argument('--agent', '-a', default='agents.yaml', help='Agent config file')
|
||||
batch_parser.add_argument('--test-file', '-t', required=True, help='JSON test file')
|
||||
batch_parser.add_argument('--batch-type', '-b', choices=['accuracy', 'criteria'], default='accuracy', help='Evaluation type')
|
||||
batch_parser.add_argument('--output', '-o', help='Output file')
|
||||
batch_parser.add_argument('--verbose', '-v', action='store_true', help='Verbose output')
|
||||
batch_parser.add_argument('--quiet', '-q', action='store_true', help='Suppress JSON output')
|
||||
@@ -69,6 +69,9 @@ class FastContextHandler(FlagHandler):
|
||||
query: Search query
|
||||
path: Path to search
|
||||
**kwargs: Additional search options
|
||||
- use_llm: Use LLM for intelligent search (slower, default: False)
|
||||
- max_turns: Max search turns for LLM mode
|
||||
- model: LLM model for intelligent search
|
||||
|
||||
Returns:
|
||||
List of matching results
|
||||
@@ -86,15 +89,20 @@ class FastContextHandler(FlagHandler):
|
||||
try:
|
||||
from praisonaiagents import FastContext
|
||||
|
||||
# Use fast non-LLM search by default for better performance
|
||||
use_llm = kwargs.get('use_llm', False)
|
||||
|
||||
fc = FastContext(
|
||||
path=path,
|
||||
workspace_path=path,
|
||||
model=kwargs.get('model', 'gpt-4o-mini'),
|
||||
max_turns=kwargs.get('max_turns', 4),
|
||||
parallelism=kwargs.get('parallelism', 8),
|
||||
timeout=kwargs.get('timeout', 30.0)
|
||||
max_turns=kwargs.get('max_turns', 2), # Reduced from 4 for speed
|
||||
max_parallel=kwargs.get('parallelism', 8),
|
||||
timeout=kwargs.get('timeout', 15.0), # Reduced timeout
|
||||
verbose=self.verbose
|
||||
)
|
||||
|
||||
result = fc.search(query)
|
||||
# Use non-LLM search by default (much faster)
|
||||
result = fc.search(query, use_llm=use_llm)
|
||||
|
||||
# Convert to list of dicts
|
||||
matches = []
|
||||
|
||||
@@ -75,7 +75,7 @@ class GuardrailHandler(FlagHandler):
|
||||
|
||||
Args:
|
||||
description: Natural language description of validation criteria
|
||||
llm: Optional LLM instance for validation
|
||||
llm: Optional LLM instance for validation (defaults to gpt-4o-mini)
|
||||
|
||||
Returns:
|
||||
LLMGuardrail instance or None if unavailable
|
||||
@@ -87,6 +87,12 @@ class GuardrailHandler(FlagHandler):
|
||||
)
|
||||
return None
|
||||
|
||||
# Provide default LLM if none specified
|
||||
if llm is None:
|
||||
import os
|
||||
# Try to get from environment or use default
|
||||
llm = os.environ.get('OPENAI_MODEL_NAME', 'gpt-4o-mini')
|
||||
|
||||
return LLMGuardrail(description=description, llm=llm)
|
||||
|
||||
def validate_output(self, output: str, description: str, llm: Any = None) -> Tuple[bool, str]:
|
||||
|
||||
@@ -0,0 +1,350 @@
|
||||
"""
|
||||
PraisonAI CLI Profiling Commands
|
||||
|
||||
Provides CLI commands for profiling agent execution.
|
||||
|
||||
Usage:
|
||||
praisonai profile run "task" --output report.html
|
||||
praisonai profile report --format json
|
||||
praisonai profile benchmark --iterations 10
|
||||
"""
|
||||
|
||||
import time
|
||||
import json
|
||||
from typing import Optional, Dict, Any
|
||||
|
||||
|
||||
class ProfilingHandler:
|
||||
"""
|
||||
Handler for CLI profiling commands.
|
||||
|
||||
Features:
|
||||
- Run tasks with profiling enabled
|
||||
- Generate reports (console, JSON, HTML)
|
||||
- Benchmark agent performance
|
||||
- Export flamegraphs
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
self._profiler = None
|
||||
|
||||
@property
|
||||
def profiler(self):
|
||||
"""Lazy load profiler."""
|
||||
if self._profiler is None:
|
||||
from praisonai.profiler import Profiler
|
||||
self._profiler = Profiler
|
||||
return self._profiler
|
||||
|
||||
def enable(self) -> None:
|
||||
"""Enable profiling."""
|
||||
self.profiler.enable()
|
||||
|
||||
def disable(self) -> None:
|
||||
"""Disable profiling."""
|
||||
self.profiler.disable()
|
||||
|
||||
def clear(self) -> None:
|
||||
"""Clear profiling data."""
|
||||
self.profiler.clear()
|
||||
|
||||
def run_with_profiling(self, task: str, config: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
|
||||
"""
|
||||
Run a task with profiling enabled.
|
||||
|
||||
Args:
|
||||
task: Task description or prompt
|
||||
config: Optional configuration dict
|
||||
|
||||
Returns:
|
||||
Dict with result and profiling data
|
||||
"""
|
||||
self.profiler.enable()
|
||||
self.profiler.clear()
|
||||
|
||||
result = None
|
||||
error = None
|
||||
|
||||
try:
|
||||
# Import and run PraisonAI
|
||||
from praisonai import PraisonAI
|
||||
|
||||
with self.profiler.block("total_execution"):
|
||||
with self.profiler.block("initialization"):
|
||||
praison = PraisonAI(auto=task, **(config or {}))
|
||||
|
||||
with self.profiler.block("agent_run"):
|
||||
result = praison.run()
|
||||
except Exception as e:
|
||||
error = str(e)
|
||||
finally:
|
||||
self.profiler.disable()
|
||||
|
||||
return {
|
||||
'result': result,
|
||||
'error': error,
|
||||
'summary': self.profiler.get_summary(),
|
||||
'statistics': self.profiler.get_statistics()
|
||||
}
|
||||
|
||||
def report(self, format: str = "console", output: Optional[str] = None) -> str:
|
||||
"""
|
||||
Generate profiling report.
|
||||
|
||||
Args:
|
||||
format: Output format (console, json, html)
|
||||
output: Optional file path to save report
|
||||
|
||||
Returns:
|
||||
Report content as string
|
||||
"""
|
||||
if format == "console":
|
||||
content = self.profiler.report(output="console")
|
||||
elif format == "json":
|
||||
content = self.profiler.export_json()
|
||||
elif format == "html":
|
||||
content = self.profiler.export_html()
|
||||
else:
|
||||
raise ValueError(f"Unknown format: {format}")
|
||||
|
||||
if output:
|
||||
with open(output, 'w') as f:
|
||||
f.write(content)
|
||||
print(f"Report saved to: {output}")
|
||||
|
||||
return content
|
||||
|
||||
def benchmark(self, task: str, iterations: int = 5,
|
||||
warmup: int = 1, config: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
|
||||
"""
|
||||
Benchmark agent performance.
|
||||
|
||||
Args:
|
||||
task: Task to benchmark
|
||||
iterations: Number of iterations
|
||||
warmup: Number of warmup runs
|
||||
config: Optional configuration
|
||||
|
||||
Returns:
|
||||
Benchmark results with statistics
|
||||
"""
|
||||
from praisonai.profiler import Profiler
|
||||
|
||||
times = []
|
||||
errors = []
|
||||
|
||||
# Warmup runs
|
||||
for i in range(warmup):
|
||||
print(f"Warmup {i + 1}/{warmup}...")
|
||||
try:
|
||||
self.run_with_profiling(task, config)
|
||||
except Exception as e:
|
||||
print(f"Warmup error: {e}")
|
||||
|
||||
# Benchmark runs
|
||||
for i in range(iterations):
|
||||
print(f"Iteration {i + 1}/{iterations}...")
|
||||
Profiler.clear()
|
||||
|
||||
start = time.perf_counter()
|
||||
try:
|
||||
self.run_with_profiling(task, config)
|
||||
duration_ms = (time.perf_counter() - start) * 1000
|
||||
times.append(duration_ms)
|
||||
except Exception as e:
|
||||
errors.append(str(e))
|
||||
duration_ms = (time.perf_counter() - start) * 1000
|
||||
times.append(duration_ms)
|
||||
|
||||
# Calculate statistics
|
||||
if times:
|
||||
sorted_times = sorted(times)
|
||||
n = len(sorted_times)
|
||||
|
||||
results = {
|
||||
'iterations': iterations,
|
||||
'successful': iterations - len(errors),
|
||||
'failed': len(errors),
|
||||
'times_ms': times,
|
||||
'mean_ms': sum(times) / n,
|
||||
'min_ms': min(times),
|
||||
'max_ms': max(times),
|
||||
'p50_ms': sorted_times[n // 2],
|
||||
'p95_ms': sorted_times[int(n * 0.95)] if n > 1 else sorted_times[-1],
|
||||
'p99_ms': sorted_times[int(n * 0.99)] if n > 1 else sorted_times[-1],
|
||||
'errors': errors
|
||||
}
|
||||
else:
|
||||
results = {
|
||||
'iterations': iterations,
|
||||
'successful': 0,
|
||||
'failed': len(errors),
|
||||
'errors': errors
|
||||
}
|
||||
|
||||
return results
|
||||
|
||||
def export_flamegraph(self, filepath: str) -> None:
|
||||
"""Export flamegraph to SVG file."""
|
||||
self.profiler.export_flamegraph(filepath)
|
||||
print(f"Flamegraph saved to: {filepath}")
|
||||
|
||||
def get_summary(self) -> Dict[str, Any]:
|
||||
"""Get profiling summary."""
|
||||
return self.profiler.get_summary()
|
||||
|
||||
def get_statistics(self, category: Optional[str] = None) -> Dict[str, float]:
|
||||
"""Get profiling statistics."""
|
||||
return self.profiler.get_statistics(category)
|
||||
|
||||
def print_summary(self) -> None:
|
||||
"""Print profiling summary to console."""
|
||||
summary = self.get_summary()
|
||||
stats = self.get_statistics()
|
||||
|
||||
print("\n" + "=" * 60)
|
||||
print("PROFILING SUMMARY")
|
||||
print("=" * 60)
|
||||
|
||||
print(f"\nTotal Time: {summary['total_time_ms']:.2f}ms")
|
||||
print(f"Operations: {summary['timing_count']}")
|
||||
print(f"Imports: {summary['import_count']}")
|
||||
print(f"Flow Steps: {summary['flow_steps']}")
|
||||
|
||||
print("\nStatistics:")
|
||||
print(f" P50 (Median): {stats['p50']:.2f}ms")
|
||||
print(f" P95: {stats['p95']:.2f}ms")
|
||||
print(f" P99: {stats['p99']:.2f}ms")
|
||||
print(f" Mean: {stats['mean']:.2f}ms")
|
||||
print(f" Std Dev: {stats['std_dev']:.2f}ms")
|
||||
|
||||
print("\nSlowest Operations:")
|
||||
for name, duration in summary['slowest_operations'][:5]:
|
||||
print(f" {name}: {duration:.2f}ms")
|
||||
|
||||
print("=" * 60)
|
||||
|
||||
|
||||
def handle_profile_command(args) -> int:
|
||||
"""
|
||||
Handle profile CLI command.
|
||||
|
||||
Args:
|
||||
args: Parsed command line arguments
|
||||
|
||||
Returns:
|
||||
Exit code (0 for success)
|
||||
"""
|
||||
handler = ProfilingHandler()
|
||||
|
||||
subcommand = getattr(args, 'profile_command', None)
|
||||
|
||||
if subcommand == 'run':
|
||||
task = getattr(args, 'task', '')
|
||||
output = getattr(args, 'output', None)
|
||||
format_type = getattr(args, 'format', 'console')
|
||||
|
||||
print(f"Running with profiling: {task[:50]}...")
|
||||
result = handler.run_with_profiling(task)
|
||||
|
||||
if result['error']:
|
||||
print(f"Error: {result['error']}")
|
||||
|
||||
handler.report(format=format_type, output=output)
|
||||
return 0 if not result['error'] else 1
|
||||
|
||||
elif subcommand == 'report':
|
||||
output = getattr(args, 'output', None)
|
||||
format_type = getattr(args, 'format', 'console')
|
||||
|
||||
handler.report(format=format_type, output=output)
|
||||
return 0
|
||||
|
||||
elif subcommand == 'benchmark':
|
||||
task = getattr(args, 'task', '')
|
||||
iterations = getattr(args, 'iterations', 5)
|
||||
warmup = getattr(args, 'warmup', 1)
|
||||
output = getattr(args, 'output', None)
|
||||
|
||||
print(f"Benchmarking: {task[:50]}...")
|
||||
results = handler.benchmark(task, iterations=iterations, warmup=warmup)
|
||||
|
||||
print("\n" + "=" * 60)
|
||||
print("BENCHMARK RESULTS")
|
||||
print("=" * 60)
|
||||
print(f"Iterations: {results['iterations']}")
|
||||
print(f"Successful: {results['successful']}")
|
||||
print(f"Failed: {results['failed']}")
|
||||
|
||||
if 'mean_ms' in results:
|
||||
print("\nTiming:")
|
||||
print(f" Mean: {results['mean_ms']:.2f}ms")
|
||||
print(f" Min: {results['min_ms']:.2f}ms")
|
||||
print(f" Max: {results['max_ms']:.2f}ms")
|
||||
print(f" P50: {results['p50_ms']:.2f}ms")
|
||||
print(f" P95: {results['p95_ms']:.2f}ms")
|
||||
|
||||
if output:
|
||||
with open(output, 'w') as f:
|
||||
json.dump(results, f, indent=2)
|
||||
print(f"\nResults saved to: {output}")
|
||||
|
||||
return 0
|
||||
|
||||
elif subcommand == 'flamegraph':
|
||||
output = getattr(args, 'output', 'profile.svg')
|
||||
handler.export_flamegraph(output)
|
||||
return 0
|
||||
|
||||
elif subcommand == 'summary':
|
||||
handler.print_summary()
|
||||
return 0
|
||||
|
||||
else:
|
||||
print("Usage: praisonai profile <run|report|benchmark|flamegraph|summary>")
|
||||
return 1
|
||||
|
||||
|
||||
def add_profile_parser(subparsers) -> None:
|
||||
"""Add profile subcommand to argument parser."""
|
||||
profile_parser = subparsers.add_parser(
|
||||
'profile',
|
||||
help='Profile agent execution'
|
||||
)
|
||||
|
||||
profile_subparsers = profile_parser.add_subparsers(dest='profile_command')
|
||||
|
||||
# Run with profiling
|
||||
run_parser = profile_subparsers.add_parser('run', help='Run task with profiling')
|
||||
run_parser.add_argument('task', help='Task to run')
|
||||
run_parser.add_argument('--output', '-o', help='Output file path')
|
||||
run_parser.add_argument('--format', '-f', choices=['console', 'json', 'html'],
|
||||
default='console', help='Output format')
|
||||
|
||||
# Generate report
|
||||
report_parser = profile_subparsers.add_parser('report', help='Generate profiling report')
|
||||
report_parser.add_argument('--output', '-o', help='Output file path')
|
||||
report_parser.add_argument('--format', '-f', choices=['console', 'json', 'html'],
|
||||
default='console', help='Output format')
|
||||
|
||||
# Benchmark
|
||||
bench_parser = profile_subparsers.add_parser('benchmark', help='Benchmark agent performance')
|
||||
bench_parser.add_argument('task', help='Task to benchmark')
|
||||
bench_parser.add_argument('--iterations', '-n', type=int, default=5, help='Number of iterations')
|
||||
bench_parser.add_argument('--warmup', '-w', type=int, default=1, help='Warmup runs')
|
||||
bench_parser.add_argument('--output', '-o', help='Output file for results')
|
||||
|
||||
# Flamegraph
|
||||
flame_parser = profile_subparsers.add_parser('flamegraph', help='Export flamegraph')
|
||||
flame_parser.add_argument('--output', '-o', default='profile.svg', help='Output SVG file')
|
||||
|
||||
# Summary
|
||||
profile_subparsers.add_parser('summary', help='Print profiling summary') # noqa: F841
|
||||
|
||||
|
||||
# Exports
|
||||
__all__ = [
|
||||
'ProfilingHandler',
|
||||
'handle_profile_command',
|
||||
'add_profile_parser',
|
||||
]
|
||||
@@ -0,0 +1,310 @@
|
||||
"""
|
||||
Skills CLI Feature Handler
|
||||
|
||||
Provides CLI commands for managing Agent Skills:
|
||||
- list: List available skills
|
||||
- validate: Validate a skill directory
|
||||
- create: Create a new skill from template
|
||||
- prompt: Generate prompt XML for skills
|
||||
"""
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import Optional, List
|
||||
|
||||
|
||||
class SkillsHandler:
|
||||
"""Handler for skills CLI commands."""
|
||||
|
||||
def __init__(self, verbose: bool = True):
|
||||
"""Initialize the skills handler.
|
||||
|
||||
Args:
|
||||
verbose: Whether to print verbose output
|
||||
"""
|
||||
self.verbose = verbose
|
||||
|
||||
def list_skills(
|
||||
self,
|
||||
skill_dirs: Optional[List[str]] = None,
|
||||
include_defaults: bool = True
|
||||
) -> List[dict]:
|
||||
"""List all available skills.
|
||||
|
||||
Args:
|
||||
skill_dirs: Optional list of directories to scan
|
||||
include_defaults: Whether to include default skill directories
|
||||
|
||||
Returns:
|
||||
List of skill info dictionaries
|
||||
"""
|
||||
from praisonaiagents.skills import discover_skills
|
||||
|
||||
skills = discover_skills(skill_dirs, include_defaults)
|
||||
|
||||
result = []
|
||||
for skill in skills:
|
||||
info = {
|
||||
"name": skill.name,
|
||||
"description": skill.description,
|
||||
"path": str(skill.path) if skill.path else None,
|
||||
"license": skill.license,
|
||||
}
|
||||
result.append(info)
|
||||
|
||||
if self.verbose:
|
||||
print(f" {skill.name}: {skill.description[:60]}...")
|
||||
|
||||
if self.verbose and not result:
|
||||
print("No skills found.")
|
||||
|
||||
return result
|
||||
|
||||
def validate_skill(self, skill_path: str) -> dict:
|
||||
"""Validate a skill directory.
|
||||
|
||||
Args:
|
||||
skill_path: Path to the skill directory
|
||||
|
||||
Returns:
|
||||
Validation result dictionary with 'valid' and 'errors' keys
|
||||
"""
|
||||
from praisonaiagents.skills import validate
|
||||
|
||||
path = Path(skill_path).expanduser().resolve()
|
||||
errors = validate(path)
|
||||
|
||||
result = {
|
||||
"valid": len(errors) == 0,
|
||||
"path": str(path),
|
||||
"errors": errors
|
||||
}
|
||||
|
||||
if self.verbose:
|
||||
if result["valid"]:
|
||||
print(f"✓ Skill at {path} is valid")
|
||||
else:
|
||||
print(f"✗ Skill at {path} has errors:")
|
||||
for error in errors:
|
||||
print(f" - {error}")
|
||||
|
||||
return result
|
||||
|
||||
def create_skill(
|
||||
self,
|
||||
name: str,
|
||||
description: str = "A custom skill",
|
||||
output_dir: Optional[str] = None
|
||||
) -> str:
|
||||
"""Create a new skill from template.
|
||||
|
||||
Args:
|
||||
name: Skill name (kebab-case)
|
||||
description: Skill description
|
||||
output_dir: Directory to create skill in (default: current dir)
|
||||
|
||||
Returns:
|
||||
Path to created skill directory
|
||||
"""
|
||||
# Validate name format
|
||||
import re
|
||||
if not re.match(r'^[a-z][a-z0-9-]*[a-z0-9]$', name) and len(name) > 1:
|
||||
if not re.match(r'^[a-z]$', name):
|
||||
raise ValueError(
|
||||
f"Invalid skill name '{name}'. "
|
||||
"Must be lowercase, use hyphens, and not start/end with hyphen."
|
||||
)
|
||||
|
||||
base_dir = Path(output_dir or os.getcwd())
|
||||
skill_dir = base_dir / name
|
||||
|
||||
if skill_dir.exists():
|
||||
raise ValueError(f"Directory already exists: {skill_dir}")
|
||||
|
||||
# Create directory structure
|
||||
skill_dir.mkdir(parents=True)
|
||||
(skill_dir / "scripts").mkdir()
|
||||
(skill_dir / "references").mkdir()
|
||||
(skill_dir / "assets").mkdir()
|
||||
|
||||
# Create SKILL.md
|
||||
skill_md_content = f"""---
|
||||
name: {name}
|
||||
description: {description}
|
||||
license: Apache-2.0
|
||||
compatibility: Works with PraisonAI Agents
|
||||
metadata:
|
||||
author: user
|
||||
version: "1.0"
|
||||
---
|
||||
|
||||
# {name.replace('-', ' ').title()}
|
||||
|
||||
## Overview
|
||||
|
||||
{description}
|
||||
|
||||
## Usage
|
||||
|
||||
Describe how to use this skill.
|
||||
|
||||
## Instructions
|
||||
|
||||
1. Step one
|
||||
2. Step two
|
||||
3. Step three
|
||||
"""
|
||||
(skill_dir / "SKILL.md").write_text(skill_md_content)
|
||||
|
||||
# Create placeholder files
|
||||
(skill_dir / "scripts" / ".gitkeep").write_text("")
|
||||
(skill_dir / "references" / ".gitkeep").write_text("")
|
||||
(skill_dir / "assets" / ".gitkeep").write_text("")
|
||||
|
||||
if self.verbose:
|
||||
print(f"✓ Created skill at {skill_dir}")
|
||||
print(" - SKILL.md")
|
||||
print(" - scripts/")
|
||||
print(" - references/")
|
||||
print(" - assets/")
|
||||
|
||||
return str(skill_dir)
|
||||
|
||||
def generate_prompt(
|
||||
self,
|
||||
skill_dirs: Optional[List[str]] = None,
|
||||
include_defaults: bool = True
|
||||
) -> str:
|
||||
"""Generate prompt XML for available skills.
|
||||
|
||||
Args:
|
||||
skill_dirs: Optional list of directories to scan
|
||||
include_defaults: Whether to include default skill directories
|
||||
|
||||
Returns:
|
||||
XML string with <available_skills> block
|
||||
"""
|
||||
from praisonaiagents.skills import SkillManager
|
||||
|
||||
manager = SkillManager()
|
||||
|
||||
if skill_dirs:
|
||||
manager.discover(skill_dirs, include_defaults=include_defaults)
|
||||
elif include_defaults:
|
||||
manager.discover(include_defaults=True)
|
||||
|
||||
prompt = manager.to_prompt()
|
||||
|
||||
if self.verbose:
|
||||
print(prompt)
|
||||
|
||||
return prompt
|
||||
|
||||
|
||||
def handle_skills_command(args) -> int:
|
||||
"""Handle skills subcommand from CLI.
|
||||
|
||||
Args:
|
||||
args: Parsed command line arguments
|
||||
|
||||
Returns:
|
||||
Exit code (0 for success, non-zero for error)
|
||||
"""
|
||||
handler = SkillsHandler(verbose=True)
|
||||
|
||||
try:
|
||||
if args.skills_command == "list":
|
||||
dirs = args.dirs if hasattr(args, 'dirs') and args.dirs else None
|
||||
handler.list_skills(dirs, include_defaults=True)
|
||||
|
||||
elif args.skills_command == "validate":
|
||||
if not hasattr(args, 'path') or not args.path:
|
||||
print("Error: --path is required for validate command")
|
||||
return 1
|
||||
result = handler.validate_skill(args.path)
|
||||
return 0 if result["valid"] else 1
|
||||
|
||||
elif args.skills_command == "create":
|
||||
if not hasattr(args, 'name') or not args.name:
|
||||
print("Error: --name is required for create command")
|
||||
return 1
|
||||
description = args.description if hasattr(args, 'description') else "A custom skill"
|
||||
output_dir = args.output if hasattr(args, 'output') else None
|
||||
handler.create_skill(args.name, description, output_dir)
|
||||
|
||||
elif args.skills_command == "prompt":
|
||||
dirs = args.dirs if hasattr(args, 'dirs') and args.dirs else None
|
||||
handler.generate_prompt(dirs, include_defaults=True)
|
||||
|
||||
else:
|
||||
print(f"Unknown skills command: {args.skills_command}")
|
||||
return 1
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error: {e}")
|
||||
return 1
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
def add_skills_parser(subparsers) -> None:
|
||||
"""Add skills subcommand to argument parser.
|
||||
|
||||
Args:
|
||||
subparsers: Subparsers object from argparse
|
||||
"""
|
||||
# subparsers is already the skills subparsers object
|
||||
# We just need to add the subcommands to it
|
||||
|
||||
# list command
|
||||
list_parser = subparsers.add_parser(
|
||||
'list',
|
||||
help='List available skills'
|
||||
)
|
||||
list_parser.add_argument(
|
||||
'--dirs',
|
||||
nargs='+',
|
||||
help='Directories to scan for skills'
|
||||
)
|
||||
|
||||
# validate command
|
||||
validate_parser = subparsers.add_parser(
|
||||
'validate',
|
||||
help='Validate a skill directory'
|
||||
)
|
||||
validate_parser.add_argument(
|
||||
'--path',
|
||||
required=True,
|
||||
help='Path to skill directory'
|
||||
)
|
||||
|
||||
# create command
|
||||
create_parser = subparsers.add_parser(
|
||||
'create',
|
||||
help='Create a new skill from template'
|
||||
)
|
||||
create_parser.add_argument(
|
||||
'--name',
|
||||
required=True,
|
||||
help='Skill name (kebab-case)'
|
||||
)
|
||||
create_parser.add_argument(
|
||||
'--description',
|
||||
default='A custom skill',
|
||||
help='Skill description'
|
||||
)
|
||||
create_parser.add_argument(
|
||||
'--output',
|
||||
help='Output directory (default: current directory)'
|
||||
)
|
||||
|
||||
# prompt command
|
||||
prompt_parser = subparsers.add_parser(
|
||||
'prompt',
|
||||
help='Generate prompt XML for skills'
|
||||
)
|
||||
prompt_parser.add_argument(
|
||||
'--dirs',
|
||||
nargs='+',
|
||||
help='Directories to scan for skills'
|
||||
)
|
||||
@@ -5,6 +5,42 @@ import argparse
|
||||
import warnings
|
||||
import os
|
||||
|
||||
# Suppress Pydantic serialization warnings from LiteLLM BEFORE any imports
|
||||
# These warnings occur when LiteLLM's response objects have field mismatches
|
||||
# Using both filterwarnings AND patching warnings.warn for complete suppression
|
||||
|
||||
warnings.filterwarnings("ignore", message=".*Pydantic serializer warnings.*")
|
||||
warnings.filterwarnings("ignore", message=".*PydanticSerializationUnexpectedValue.*")
|
||||
warnings.filterwarnings("ignore", message=".*Expected \\d+ fields but got.*")
|
||||
warnings.filterwarnings("ignore", message=".*Expected `StreamingChoices`.*")
|
||||
warnings.filterwarnings("ignore", message=".*Expected `Message`.*")
|
||||
warnings.filterwarnings("ignore", message=".*serialized value may not be as expected.*")
|
||||
warnings.filterwarnings("ignore", category=UserWarning, module="pydantic.*")
|
||||
|
||||
# Patch warnings.showwarning to intercept ALL warnings including those from crewai's patched warn
|
||||
# This is the final output function that actually displays warnings
|
||||
_SUPPRESSED_PATTERNS = [
|
||||
"Pydantic serializer warnings",
|
||||
"PydanticSerializationUnexpectedValue",
|
||||
"Expected", # Catches "Expected N fields but got M"
|
||||
"StreamingChoices",
|
||||
"serialized value may not be as expected",
|
||||
"duckduckgo_search", # Suppress duckduckgo rename warning
|
||||
]
|
||||
|
||||
_original_showwarning = warnings.showwarning
|
||||
|
||||
def _patched_showwarning(message, category, filename, lineno, file=None, line=None):
|
||||
msg_str = str(message)
|
||||
for pattern in _SUPPRESSED_PATTERNS:
|
||||
if pattern in msg_str:
|
||||
return
|
||||
if category is UserWarning and "pydantic" in filename.lower():
|
||||
return
|
||||
_original_showwarning(message, category, filename, lineno, file, line)
|
||||
|
||||
warnings.showwarning = _patched_showwarning
|
||||
|
||||
# Suppress crewai RuntimeWarning about module loading order (only in non-debug mode)
|
||||
# This warning is harmless and occurs when running as `python -m praisonai.cli.main`
|
||||
if os.environ.get('LOGLEVEL', 'INFO').upper() != 'DEBUG':
|
||||
@@ -59,49 +95,27 @@ def _get_chainlit_run():
|
||||
from chainlit.cli import chainlit_run
|
||||
return chainlit_run
|
||||
|
||||
try:
|
||||
import gradio as gr
|
||||
GRADIO_AVAILABLE = True
|
||||
except ImportError:
|
||||
pass
|
||||
# Use find_spec for fast availability checks (no actual import)
|
||||
import importlib.util
|
||||
GRADIO_AVAILABLE = importlib.util.find_spec("gradio") is not None
|
||||
CALL_MODULE_AVAILABLE = importlib.util.find_spec("praisonai.api.call") is not None
|
||||
CREWAI_AVAILABLE = importlib.util.find_spec("crewai") is not None
|
||||
AUTOGEN_AVAILABLE = importlib.util.find_spec("autogen") is not None
|
||||
PRAISONAI_AVAILABLE = importlib.util.find_spec("praisonaiagents") is not None
|
||||
TRAIN_AVAILABLE = importlib.util.find_spec("unsloth") is not None
|
||||
|
||||
try:
|
||||
import praisonai.api.call as call_module
|
||||
CALL_MODULE_AVAILABLE = True
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
try:
|
||||
import crewai
|
||||
CREWAI_AVAILABLE = True
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
try:
|
||||
import autogen
|
||||
AUTOGEN_AVAILABLE = True
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
try:
|
||||
from praisonaiagents import Agent as PraisonAgent, Task as PraisonTask, PraisonAIAgents
|
||||
PRAISONAI_AVAILABLE = True
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
try:
|
||||
from unsloth import FastLanguageModel
|
||||
TRAIN_AVAILABLE = True
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
logging.basicConfig(level=os.environ.get('LOGLEVEL', 'INFO'), format='%(asctime)s - %(levelname)s - %(message)s')
|
||||
logging.basicConfig(level=os.environ.get('LOGLEVEL', 'WARNING'), format='%(asctime)s - %(levelname)s - %(message)s')
|
||||
logging.getLogger('alembic').setLevel(logging.ERROR)
|
||||
logging.getLogger('gradio').setLevel(logging.ERROR)
|
||||
logging.getLogger('gradio').setLevel(os.environ.get('GRADIO_LOGLEVEL', 'WARNING'))
|
||||
logging.getLogger('rust_logger').setLevel(logging.WARNING)
|
||||
logging.getLogger('duckduckgo').setLevel(logging.ERROR)
|
||||
logging.getLogger('_client').setLevel(logging.ERROR)
|
||||
# Suppress praisonaiagents INFO logs unless LOGLEVEL is explicitly set to debug/info
|
||||
if os.environ.get('LOGLEVEL', '').upper() not in ('DEBUG', 'INFO'):
|
||||
logging.getLogger('praisonaiagents').setLevel(logging.WARNING)
|
||||
logging.getLogger('praisonaiagents.llm').setLevel(logging.WARNING)
|
||||
logging.getLogger('praisonaiagents.llm.llm').setLevel(logging.WARNING)
|
||||
|
||||
def stream_subprocess(command, env=None):
|
||||
"""
|
||||
@@ -215,7 +229,14 @@ class PraisonAI:
|
||||
# Store the original agent_file from constructor
|
||||
original_agent_file = self.agent_file
|
||||
|
||||
args = self.parse_args()
|
||||
# Parse args - this returns both args and unknown_args
|
||||
parse_result = self.parse_args()
|
||||
if isinstance(parse_result, tuple):
|
||||
args, unknown_args = parse_result
|
||||
else:
|
||||
args = parse_result
|
||||
unknown_args = []
|
||||
|
||||
# Store args for use in handle_direct_prompt
|
||||
self.args = args
|
||||
invocation_cmd = "praisonai"
|
||||
@@ -245,10 +266,36 @@ class PraisonAI:
|
||||
file_input = self.read_file_if_provided(getattr(args, 'file', None))
|
||||
|
||||
if args.command:
|
||||
if args.command.startswith("tests.test") or args.command.startswith("tests/test"): # Argument used for testing purposes
|
||||
# Handle schedule command
|
||||
if args.command == "schedule":
|
||||
from praisonai.cli.features.agent_scheduler import AgentSchedulerHandler
|
||||
# Check for subcommands (start, list, stop, logs, restart)
|
||||
subcommand = unknown_args[0] if unknown_args and not unknown_args[0].startswith('-') else None
|
||||
|
||||
if subcommand in ['start', 'list', 'stop', 'logs', 'restart', 'delete', 'describe', 'save', 'stop-all', 'stats']:
|
||||
exit_code = AgentSchedulerHandler.handle_daemon_command(subcommand, args, unknown_args[1:] if len(unknown_args) > 1 else [])
|
||||
else:
|
||||
# Legacy mode: direct scheduling (foreground) or daemon mode
|
||||
daemon_mode = getattr(args, 'daemon', False)
|
||||
exit_code = AgentSchedulerHandler.handle_schedule_command(args, unknown_args, daemon_mode=daemon_mode)
|
||||
|
||||
sys.exit(exit_code)
|
||||
elif args.command.startswith("tests.test") or args.command.startswith("tests/test"): # Argument used for testing purposes
|
||||
print("test")
|
||||
return "test"
|
||||
else:
|
||||
# Handle --compare flag for CLI mode comparison
|
||||
if hasattr(args, 'compare') and args.compare:
|
||||
from .features.compare import CompareHandler
|
||||
handler = CompareHandler(verbose=getattr(args, 'verbose', False))
|
||||
result = handler.execute(
|
||||
args.command,
|
||||
args.compare,
|
||||
model=getattr(args, 'llm', None),
|
||||
output_path=getattr(args, 'compare_output', None)
|
||||
)
|
||||
return result
|
||||
|
||||
# Combine command with any available inputs (stdin and/or file)
|
||||
combined_inputs = []
|
||||
if stdin_input:
|
||||
@@ -602,7 +649,7 @@ class PraisonAI:
|
||||
return default_args
|
||||
|
||||
# Define special commands
|
||||
special_commands = ['chat', 'code', 'call', 'realtime', 'train', 'ui', 'context', 'research', 'memory', 'rules', 'workflow', 'hooks', 'knowledge', 'session', 'tools', 'todo', 'docs', 'mcp', 'commit', 'serve']
|
||||
special_commands = ['chat', 'code', 'call', 'realtime', 'train', 'ui', 'context', 'research', 'memory', 'rules', 'workflow', 'hooks', 'knowledge', 'session', 'tools', 'todo', 'docs', 'mcp', 'commit', 'serve', 'schedule', 'skills']
|
||||
|
||||
parser = argparse.ArgumentParser(prog="praisonai", description="praisonAI command-line interface")
|
||||
parser.add_argument("--framework", choices=["crewai", "autogen", "praisonai"], help="Specify the framework")
|
||||
@@ -720,9 +767,29 @@ class PraisonAI:
|
||||
# Autonomy Mode - control AI action approval
|
||||
parser.add_argument("--autonomy", type=str, choices=["suggest", "auto_edit", "full_auto"], help="Set autonomy mode for AI actions")
|
||||
|
||||
# Tool Approval - control tool execution approval
|
||||
parser.add_argument("--trust", action="store_true", help="Auto-approve all tool executions (skip approval prompts)")
|
||||
parser.add_argument("--approve-level", type=str, choices=["low", "medium", "high", "critical"],
|
||||
help="Auto-approve tools up to this risk level (e.g., --approve-level high approves low/medium/high but prompts for critical)")
|
||||
|
||||
# Sandbox Execution - secure command execution
|
||||
parser.add_argument("--sandbox", type=str, choices=["off", "basic", "strict"], help="Enable sandboxed command execution")
|
||||
|
||||
# External Agent - use external AI CLI tools
|
||||
parser.add_argument("--external-agent", type=str, choices=["claude", "gemini", "codex", "cursor"],
|
||||
help="Use external AI CLI tool (claude, gemini, codex, cursor)")
|
||||
|
||||
# Compare - compare different CLI modes
|
||||
parser.add_argument("--compare", type=str, help="Compare CLI modes (comma-separated: basic,tools,research,planning)")
|
||||
parser.add_argument("--compare-output", type=str, help="Save comparison results to file")
|
||||
|
||||
# Agent Scheduler - for schedule command
|
||||
parser.add_argument("--interval", dest="schedule_interval", type=str, help="Schedule interval (e.g., 'hourly', '*/30m', 'daily')")
|
||||
parser.add_argument("--schedule-max-retries", dest="schedule_max_retries", type=int, help="Maximum retry attempts for scheduled execution")
|
||||
parser.add_argument("--timeout", type=int, help="Maximum execution time per run in seconds")
|
||||
parser.add_argument('--max-cost', type=float, help='Maximum total cost budget in USD')
|
||||
parser.add_argument('--daemon', action='store_true', help=argparse.SUPPRESS) # Hidden flag for daemon mode
|
||||
|
||||
# If we're in a test environment, parse with empty args to avoid pytest interference
|
||||
if in_test_env:
|
||||
args, unknown_args = parser.parse_known_args([])
|
||||
@@ -783,7 +850,7 @@ class PraisonAI:
|
||||
except ModuleNotFoundError as e:
|
||||
missing_module = str(e).split("'")[1]
|
||||
print(f"[red]ERROR: Missing dependency {missing_module}. Install with:[/red]")
|
||||
print(f"\npip install \"praisonai[chat]\"\n")
|
||||
print("\npip install \"praisonai[chat]\"\n")
|
||||
sys.exit(1)
|
||||
sys.exit(0)
|
||||
|
||||
@@ -1011,6 +1078,23 @@ class PraisonAI:
|
||||
self.handle_commit_command(unknown_args)
|
||||
sys.exit(0)
|
||||
|
||||
elif args.command == 'skills':
|
||||
if not PRAISONAI_AVAILABLE:
|
||||
print("[red]ERROR: PraisonAI Agents is not installed. Install with:[/red]")
|
||||
print("\npip install praisonaiagents\n")
|
||||
sys.exit(1)
|
||||
|
||||
from .features.skills import handle_skills_command, add_skills_parser
|
||||
|
||||
# Create a parser for skills command
|
||||
skills_parser = argparse.ArgumentParser(prog="praisonai skills")
|
||||
skills_subparsers = skills_parser.add_subparsers(dest='skills_command', help='Skills commands')
|
||||
add_skills_parser(skills_subparsers)
|
||||
skills_args = skills_parser.parse_args(unknown_args)
|
||||
|
||||
exit_code = handle_skills_command(skills_args)
|
||||
sys.exit(exit_code)
|
||||
|
||||
# Only check framework availability for agent-related operations
|
||||
if not args.command and (args.init or args.auto or args.framework):
|
||||
if not CREWAI_AVAILABLE and not AUTOGEN_AVAILABLE and not PRAISONAI_AVAILABLE:
|
||||
@@ -1027,7 +1111,7 @@ class PraisonAI:
|
||||
args.direct_prompt = args.command
|
||||
args.command = None
|
||||
|
||||
return args
|
||||
return args, unknown_args
|
||||
|
||||
def _rewrite_query(self, query: str, rewrite_tools: str = None, verbose: bool = False) -> str:
|
||||
"""
|
||||
@@ -1294,12 +1378,47 @@ class PraisonAI:
|
||||
|
||||
console.print(table)
|
||||
|
||||
# Show recent memories
|
||||
# Show recent short-term memories
|
||||
print("\n[bold]Recent Short-term Memories:[/bold]")
|
||||
short_term = memory.get_short_term(limit=5)
|
||||
for i, item in enumerate(short_term, 1):
|
||||
content = item.get('content', str(item))[:100]
|
||||
print(f" {i}. {content}")
|
||||
if short_term:
|
||||
for i, item in enumerate(short_term, 1):
|
||||
content = item.get('content', str(item))[:100]
|
||||
print(f" {i}. {content}")
|
||||
else:
|
||||
print(" [dim]No short-term memories[/dim]")
|
||||
|
||||
# Show long-term memories
|
||||
print("\n[bold]Long-term Memories:[/bold]")
|
||||
long_term = memory.get_long_term(limit=10)
|
||||
if long_term:
|
||||
for i, item in enumerate(long_term, 1):
|
||||
# Handle both dict and MemoryItem objects
|
||||
if hasattr(item, 'content'):
|
||||
content = str(item.content)[:100]
|
||||
importance = getattr(item, 'importance', 0)
|
||||
else:
|
||||
content = item.get('content', str(item))[:100]
|
||||
importance = item.get('importance', 0)
|
||||
print(f" {i}. [{importance:.1f}] {content}")
|
||||
else:
|
||||
print(" [dim]No long-term memories[/dim]")
|
||||
|
||||
# Show entities
|
||||
print("\n[bold]Entities:[/bold]")
|
||||
entities = memory.get_all_entities()
|
||||
if entities:
|
||||
for entity in entities[:10]:
|
||||
# Handle both dict and Entity objects
|
||||
if hasattr(entity, 'name'):
|
||||
name = entity.name
|
||||
entity_type = getattr(entity, 'entity_type', 'unknown')
|
||||
else:
|
||||
name = entity.get('name', 'Unknown')
|
||||
entity_type = entity.get('entity_type', 'unknown')
|
||||
print(f" • {name} ({entity_type})")
|
||||
else:
|
||||
print(" [dim]No entities[/dim]")
|
||||
|
||||
elif action == 'add':
|
||||
if not action_args:
|
||||
@@ -1532,6 +1651,7 @@ class PraisonAI:
|
||||
args: Parsed command line arguments
|
||||
"""
|
||||
try:
|
||||
from praisonaiagents import Agent as PraisonAgent
|
||||
from praisonaiagents.memory import WorkflowManager
|
||||
from rich import print
|
||||
from rich.table import Table
|
||||
@@ -2773,6 +2893,7 @@ Provide ONLY the commit message, no explanations."""
|
||||
Returns:
|
||||
Final output from the workflow
|
||||
"""
|
||||
from praisonaiagents import Agent as PraisonAgent
|
||||
from rich import print
|
||||
|
||||
workflow_str = self.args.workflow
|
||||
@@ -2919,6 +3040,8 @@ Provide ONLY the commit message, no explanations."""
|
||||
prompt = f"{mention_context}# Task:\n{prompt}"
|
||||
|
||||
if PRAISONAI_AVAILABLE:
|
||||
from praisonaiagents import Agent as PraisonAgent
|
||||
|
||||
agent_config = {
|
||||
"name": "DirectAgent",
|
||||
"role": "Assistant",
|
||||
@@ -2926,10 +3049,18 @@ Provide ONLY the commit message, no explanations."""
|
||||
"backstory": "You are a helpful AI assistant"
|
||||
}
|
||||
|
||||
# Suppress verbose output in interactive mode (no Agent Info, Task, Response panels)
|
||||
if getattr(self, '_interactive_mode', False):
|
||||
# Set verbose=False by default, only enable with --verbose flag
|
||||
# This shows minimal "Generating..." status instead of full Agent Info panels
|
||||
if hasattr(self, 'args') and getattr(self.args, 'verbose', False):
|
||||
agent_config["verbose"] = True
|
||||
else:
|
||||
agent_config["verbose"] = False
|
||||
|
||||
# Load default tools (same as interactive mode)
|
||||
default_tools = self._load_interactive_tools()
|
||||
if default_tools:
|
||||
agent_config["tools"] = default_tools
|
||||
|
||||
# Add llm if specified
|
||||
if hasattr(self, 'args') and self.args.llm:
|
||||
# Check if max_tokens is specified - pass as dict config
|
||||
@@ -2937,7 +3068,7 @@ Provide ONLY the commit message, no explanations."""
|
||||
if max_tokens:
|
||||
# Pass llm as dict with model and max_tokens
|
||||
agent_config["llm"] = {"model": self.args.llm, "max_tokens": max_tokens}
|
||||
print(f"[bold cyan]Max tokens set to: {max_tokens}[/bold cyan]")
|
||||
logging.debug(f"Max tokens set to: {max_tokens}")
|
||||
else:
|
||||
agent_config["llm"] = self.args.llm
|
||||
|
||||
@@ -3010,6 +3141,35 @@ Provide ONLY the commit message, no explanations."""
|
||||
|
||||
# ===== NEW CLI FEATURES INTEGRATION =====
|
||||
|
||||
# Tool Approval - Auto-approve tools based on --trust or --approve-level
|
||||
if getattr(self.args, 'trust', False) or getattr(self.args, 'approve_level', None):
|
||||
from praisonaiagents.approval import set_approval_callback, ApprovalDecision
|
||||
|
||||
if getattr(self.args, 'trust', False):
|
||||
# Auto-approve all tools
|
||||
def auto_approve_all(function_name, arguments, risk_level):
|
||||
return ApprovalDecision(approved=True, reason="Auto-approved via --trust flag")
|
||||
set_approval_callback(auto_approve_all)
|
||||
print("[bold yellow]⚠️ Trust mode enabled - all tool executions will be auto-approved[/bold yellow]")
|
||||
elif getattr(self.args, 'approve_level', None):
|
||||
# Auto-approve up to specified risk level
|
||||
max_level = self.args.approve_level
|
||||
risk_order = {"low": 1, "medium": 2, "high": 3, "critical": 4}
|
||||
max_level_value = risk_order.get(max_level, 3)
|
||||
|
||||
def level_based_approve(function_name, arguments, risk_level):
|
||||
tool_level_value = risk_order.get(risk_level, 4)
|
||||
if tool_level_value <= max_level_value:
|
||||
return ApprovalDecision(approved=True, reason=f"Auto-approved (level {risk_level} <= {max_level})")
|
||||
else:
|
||||
# Signal to pause Live display before showing approval prompt
|
||||
# This is handled by the approval_pending flag in status_info
|
||||
from praisonaiagents.approval import console_approval_callback
|
||||
return console_approval_callback(function_name, arguments, risk_level)
|
||||
|
||||
set_approval_callback(level_based_approve)
|
||||
print(f"[bold cyan]Auto-approve enabled for tools up to '{max_level}' risk level[/bold cyan]")
|
||||
|
||||
# Router - Smart model selection (must be before agent creation)
|
||||
if getattr(self.args, 'router', False):
|
||||
from .features.router import RouterHandler
|
||||
@@ -3029,9 +3189,8 @@ Provide ONLY the commit message, no explanations."""
|
||||
telemetry = TelemetryHandler(verbose=getattr(self.args, 'verbose', False))
|
||||
telemetry.enable()
|
||||
|
||||
# Auto Memory - Automatic memory extraction
|
||||
# Auto Memory - Automatic memory extraction (handled post-processing, not as Agent param)
|
||||
if getattr(self.args, 'auto_memory', False):
|
||||
agent_config["auto_memory"] = True
|
||||
print("[bold cyan]Auto Memory enabled - will extract and store memories[/bold cyan]")
|
||||
|
||||
# MCP - Model Context Protocol tools
|
||||
@@ -3050,6 +3209,43 @@ Provide ONLY the commit message, no explanations."""
|
||||
existing_tools = list(mcp_tools)
|
||||
agent_config['tools'] = existing_tools
|
||||
|
||||
# External Agent - Use external AI CLI tools directly
|
||||
if getattr(self.args, 'external_agent', None):
|
||||
from rich.console import Console
|
||||
ext_console = Console()
|
||||
external_agent_name = self.args.external_agent
|
||||
try:
|
||||
from .features.external_agents import ExternalAgentsHandler
|
||||
handler = ExternalAgentsHandler(verbose=getattr(self.args, 'verbose', False))
|
||||
|
||||
# Get workspace from current directory
|
||||
import os
|
||||
workspace = os.getcwd()
|
||||
|
||||
integration = handler.get_integration(external_agent_name, workspace=workspace)
|
||||
|
||||
if integration.is_available:
|
||||
ext_console.print(f"[bold cyan]🔌 Using external agent: {external_agent_name}[/bold cyan]")
|
||||
|
||||
# Run the external agent directly instead of PraisonAI agent
|
||||
import asyncio
|
||||
try:
|
||||
result = asyncio.run(integration.execute(prompt))
|
||||
ext_console.print(f"\n[bold green]Result from {external_agent_name}:[/bold green]")
|
||||
ext_console.print(result)
|
||||
# Return empty string to avoid duplicate printing by caller
|
||||
return ""
|
||||
except Exception as e:
|
||||
ext_console.print(f"[red]Error executing {external_agent_name}: {e}[/red]")
|
||||
return None
|
||||
else:
|
||||
ext_console.print(f"[yellow]⚠️ External agent '{external_agent_name}' is not installed[/yellow]")
|
||||
ext_console.print(f"[dim]Install with: {handler._get_install_instructions(external_agent_name)}[/dim]")
|
||||
return None
|
||||
except Exception as e:
|
||||
ext_console.print(f"[red]Error setting up external agent: {e}[/red]")
|
||||
return None
|
||||
|
||||
# Fast Context - Codebase search
|
||||
if getattr(self.args, 'fast_context', None):
|
||||
from .features.fast_context import FastContextHandler
|
||||
@@ -3112,7 +3308,13 @@ Provide ONLY the commit message, no explanations."""
|
||||
flow.display_workflow_start("Direct Prompt", ["DirectAgent"])
|
||||
|
||||
agent = PraisonAgent(**agent_config)
|
||||
result = agent.start(prompt)
|
||||
|
||||
# Run with minimal status display when verbose=False
|
||||
is_verbose = agent_config.get("verbose", False)
|
||||
if not is_verbose:
|
||||
result = self._run_with_status_display(agent, prompt)
|
||||
else:
|
||||
result = agent.start(prompt)
|
||||
|
||||
# ===== POST-PROCESSING WITH NEW FEATURES =====
|
||||
|
||||
@@ -3134,7 +3336,10 @@ Provide ONLY the commit message, no explanations."""
|
||||
if hasattr(self, 'args') and getattr(self.args, 'auto_memory', False):
|
||||
from .features.auto_memory import AutoMemoryHandler
|
||||
auto_mem = AutoMemoryHandler(verbose=getattr(self.args, 'verbose', False))
|
||||
auto_mem.post_process_result(result, {'user_id': getattr(self.args, 'user_id', None)})
|
||||
auto_mem.post_process_result(
|
||||
result,
|
||||
{'user_id': getattr(self.args, 'user_id', None), 'user_message': prompt}
|
||||
)
|
||||
|
||||
# Todo - Generate todo list from response
|
||||
if hasattr(self, 'args') and getattr(self.args, 'todo', False):
|
||||
@@ -3236,6 +3441,207 @@ Now, {final_instruction.lower()}:"""
|
||||
print("pip install praisonaiagents # For PraisonAIAgents\n")
|
||||
sys.exit(1)
|
||||
|
||||
def _run_with_status_display(self, agent, prompt):
|
||||
"""
|
||||
Run agent with minimal status display (spinner + tool/handoff updates).
|
||||
|
||||
Shows:
|
||||
- "Generating..." with spinner while processing
|
||||
- Real-time tool call notifications via registered callback
|
||||
- Agent handoff notifications
|
||||
"""
|
||||
import threading
|
||||
import time
|
||||
from rich.console import Console
|
||||
from rich.live import Live
|
||||
from rich.text import Text
|
||||
|
||||
# Import callback registration from praisonaiagents
|
||||
from praisonaiagents import register_display_callback, sync_display_callbacks
|
||||
|
||||
console = Console()
|
||||
|
||||
# Get tool names for display
|
||||
tool_names = []
|
||||
if hasattr(agent, 'tools') and agent.tools:
|
||||
for tool in agent.tools:
|
||||
if hasattr(tool, '__name__'):
|
||||
tool_names.append(tool.__name__)
|
||||
elif hasattr(tool, 'name'):
|
||||
tool_names.append(tool.name)
|
||||
|
||||
status_info = {
|
||||
'status': 'Generating...',
|
||||
'tool_calls': [],
|
||||
'handoffs': [],
|
||||
'done': False,
|
||||
'result': None,
|
||||
'error': None,
|
||||
'start_time': time.time(),
|
||||
'available_tools': tool_names,
|
||||
'approval_pending': False, # Flag to pause Live display during approval
|
||||
'live_instance': None # Reference to Live instance for stopping
|
||||
}
|
||||
|
||||
def tool_call_callback(message):
|
||||
"""Callback triggered when a tool is called."""
|
||||
# Extract tool name from message
|
||||
if "Calling function:" in message:
|
||||
# Format: "Calling function: function_name"
|
||||
parts = message.split("Calling function:")
|
||||
if len(parts) > 1:
|
||||
tool_name = parts[1].strip()
|
||||
if tool_name and tool_name not in status_info['tool_calls']:
|
||||
status_info['tool_calls'].append(tool_name)
|
||||
status_info['status'] = f"Using {tool_name}..."
|
||||
elif "Function " in message and " returned:" in message:
|
||||
# Format: "Function function_name returned: ..."
|
||||
# Tool execution completed, update status
|
||||
status_info['status'] = "Processing result..."
|
||||
|
||||
# Register callback for tool calls
|
||||
register_display_callback('tool_call', tool_call_callback)
|
||||
|
||||
def build_status_display():
|
||||
"""Build the status display text."""
|
||||
elapsed = time.time() - status_info['start_time']
|
||||
|
||||
# Main status with spinner
|
||||
text = Text()
|
||||
text.append("⏳ ", style="cyan")
|
||||
text.append(f"{status_info['status']} ", style="bold")
|
||||
text.append(f"({elapsed:.1f}s)", style="dim")
|
||||
|
||||
# Show available tools on first line
|
||||
if status_info['available_tools'] and not status_info['tool_calls']:
|
||||
tools_str = ', '.join(status_info['available_tools'][:3])
|
||||
if len(status_info['available_tools']) > 3:
|
||||
tools_str += f" +{len(status_info['available_tools']) - 3} more"
|
||||
text.append(f"\n 🔧 Tools: {tools_str}", style="dim")
|
||||
|
||||
# Show recent tool calls
|
||||
if status_info['tool_calls']:
|
||||
text.append("\n")
|
||||
for tool in status_info['tool_calls'][-3:]: # Show last 3
|
||||
text.append(f" ⚙ {tool}", style="dim yellow")
|
||||
text.append("\n")
|
||||
|
||||
# Show handoffs
|
||||
if status_info['handoffs']:
|
||||
for handoff in status_info['handoffs'][-2:]: # Show last 2
|
||||
text.append(f" → {handoff}", style="dim cyan")
|
||||
text.append("\n")
|
||||
|
||||
return text
|
||||
|
||||
def run_agent():
|
||||
"""Run the agent in background thread."""
|
||||
try:
|
||||
status_info['result'] = agent.start(prompt)
|
||||
except Exception as e:
|
||||
status_info['error'] = e
|
||||
finally:
|
||||
status_info['done'] = True
|
||||
|
||||
# Set up approval callback that stops Live display before prompting
|
||||
# Only if not using --trust (which auto-approves everything)
|
||||
if not getattr(self.args, 'trust', False):
|
||||
from praisonaiagents.approval import set_approval_callback, ApprovalDecision
|
||||
from rich.prompt import Confirm
|
||||
from rich.panel import Panel
|
||||
|
||||
def cli_approval_with_live_pause(function_name, arguments, risk_level):
|
||||
"""Approval callback that stops Live display before prompting."""
|
||||
# Signal to stop Live display
|
||||
status_info['approval_pending'] = True
|
||||
|
||||
# Wait a moment for Live to stop
|
||||
time.sleep(0.2)
|
||||
|
||||
# Now show the approval prompt
|
||||
risk_colors = {"critical": "bold red", "high": "red", "medium": "yellow", "low": "blue"}
|
||||
risk_color = risk_colors.get(risk_level, "white")
|
||||
|
||||
tool_info = f"[bold]Function:[/] {function_name}\n"
|
||||
tool_info += f"[bold]Risk Level:[/] [{risk_color}]{risk_level.upper()}[/{risk_color}]\n"
|
||||
tool_info += "[bold]Arguments:[/]\n"
|
||||
for key, value in arguments.items():
|
||||
str_value = str(value)[:100] + "..." if len(str(value)) > 100 else str(value)
|
||||
tool_info += f" {key}: {str_value}\n"
|
||||
|
||||
console.print(Panel(tool_info.strip(), title="🔒 Tool Approval Required", border_style=risk_color))
|
||||
|
||||
try:
|
||||
approved = Confirm.ask(f"[{risk_color}]Execute this {risk_level} risk tool?[/{risk_color}]", default=False)
|
||||
status_info['approval_pending'] = False
|
||||
|
||||
if approved:
|
||||
console.print("[green]✅ Approved[/green]")
|
||||
return ApprovalDecision(approved=True, reason="User approved")
|
||||
else:
|
||||
console.print("[red]❌ Denied[/red]")
|
||||
return ApprovalDecision(approved=False, reason="User denied")
|
||||
except (KeyboardInterrupt, EOFError):
|
||||
status_info['approval_pending'] = False
|
||||
console.print("\n[red]❌ Cancelled[/red]")
|
||||
return ApprovalDecision(approved=False, reason="User cancelled")
|
||||
|
||||
# Only set if not already set by --approve-level
|
||||
if not getattr(self.args, 'approve_level', None):
|
||||
set_approval_callback(cli_approval_with_live_pause)
|
||||
|
||||
# Start agent in background thread
|
||||
thread = threading.Thread(target=run_agent, daemon=True)
|
||||
thread.start()
|
||||
|
||||
# Show live status while processing
|
||||
try:
|
||||
with Live(build_status_display(), console=console, refresh_per_second=4, transient=True) as live:
|
||||
status_info['live_instance'] = live
|
||||
while not status_info['done']:
|
||||
# Check if approval is pending - stop Live to show prompt
|
||||
if status_info['approval_pending']:
|
||||
break
|
||||
live.update(build_status_display())
|
||||
time.sleep(0.1)
|
||||
|
||||
# If approval was pending, wait for it to complete then restart Live
|
||||
while status_info['approval_pending']:
|
||||
time.sleep(0.1)
|
||||
|
||||
# Continue with Live display if not done
|
||||
if not status_info['done']:
|
||||
with Live(build_status_display(), console=console, refresh_per_second=4, transient=True) as live:
|
||||
while not status_info['done']:
|
||||
if status_info['approval_pending']:
|
||||
break
|
||||
live.update(build_status_display())
|
||||
time.sleep(0.1)
|
||||
except KeyboardInterrupt:
|
||||
console.print("\n[dim]Interrupted[/dim]")
|
||||
# Unregister callback
|
||||
if 'tool_call' in sync_display_callbacks:
|
||||
del sync_display_callbacks['tool_call']
|
||||
return None
|
||||
|
||||
# Wait for thread to complete
|
||||
thread.join(timeout=1.0)
|
||||
|
||||
# Unregister callback to avoid memory leaks
|
||||
if 'tool_call' in sync_display_callbacks:
|
||||
del sync_display_callbacks['tool_call']
|
||||
|
||||
# Handle result
|
||||
if status_info['error']:
|
||||
console.print(f"[red]Error: {status_info['error']}[/red]")
|
||||
return None
|
||||
|
||||
# Show tool calls that were made (if any)
|
||||
if status_info['tool_calls']:
|
||||
console.print(f"[dim]Tools used: {', '.join(status_info['tool_calls'])}[/dim]")
|
||||
|
||||
return status_info['result']
|
||||
|
||||
def _handle_serve_command(self, args, unknown_args):
|
||||
"""
|
||||
Handle the serve command - start API server for agents.
|
||||
@@ -3845,22 +4251,23 @@ Now, {final_instruction.lower()}:"""
|
||||
prompt_session = None
|
||||
try:
|
||||
from prompt_toolkit import PromptSession
|
||||
from prompt_toolkit.completion import WordCompleter
|
||||
from prompt_toolkit.auto_suggest import AutoSuggestFromHistory
|
||||
from prompt_toolkit.history import InMemoryHistory
|
||||
from praisonai.cli.features.at_mentions import CombinedCompleter
|
||||
|
||||
# Create completer for slash commands
|
||||
# Create combined completer for / commands and @ mentions
|
||||
commands = ['help', 'exit', 'quit', 'clear', 'tools', 'profile', 'model', 'stats', 'compact', 'undo', 'queue', 'q']
|
||||
slash_completer = WordCompleter(
|
||||
[f'/{cmd}' for cmd in commands],
|
||||
ignore_case=True
|
||||
combined_completer = CombinedCompleter(
|
||||
commands=commands,
|
||||
root_dir=os.getcwd()
|
||||
)
|
||||
|
||||
prompt_session = PromptSession(
|
||||
message="❯ ",
|
||||
completer=slash_completer,
|
||||
completer=combined_completer,
|
||||
auto_suggest=AutoSuggestFromHistory(),
|
||||
history=InMemoryHistory()
|
||||
history=InMemoryHistory(),
|
||||
complete_while_typing=True
|
||||
)
|
||||
except ImportError:
|
||||
pass # Fall back to simple input
|
||||
@@ -4334,16 +4741,14 @@ Provide a concise summary (max 200 words):"""
|
||||
|
||||
def _process_interactive_prompt_async(self, prompt, tools_list, console, session_state=None):
|
||||
"""
|
||||
Process a prompt asynchronously with live status display.
|
||||
Process a prompt asynchronously with non-blocking status display.
|
||||
|
||||
Shows real-time status (tools, commands) while processing in background.
|
||||
Shows status updates without blocking user input.
|
||||
Allows user to queue more messages while processing.
|
||||
"""
|
||||
import threading
|
||||
import time
|
||||
from rich.live import Live
|
||||
from rich.panel import Panel
|
||||
from rich.text import Text
|
||||
import sys
|
||||
from praisonai.cli.features.message_queue import ProcessingState
|
||||
|
||||
state_manager = session_state['state_manager']
|
||||
@@ -4355,8 +4760,11 @@ Provide a concise summary (max 200 words):"""
|
||||
live_status.clear()
|
||||
live_status.update_status("Thinking...")
|
||||
|
||||
# Print initial status (non-blocking - user can still type)
|
||||
console.print(f"[dim cyan]⏳ {live_status.current_status}[/dim cyan]")
|
||||
|
||||
# Result container for thread communication
|
||||
result_container = {'response': None, 'error': None, 'done': False}
|
||||
result_container = {'response': None, 'error': None, 'done': False, 'last_status': ''}
|
||||
|
||||
def process_in_background():
|
||||
"""Run agent processing in background thread."""
|
||||
@@ -4402,37 +4810,36 @@ Provide a concise summary (max 200 words):"""
|
||||
bg_thread = threading.Thread(target=process_in_background, daemon=True)
|
||||
bg_thread.start()
|
||||
|
||||
# Show live status while processing
|
||||
try:
|
||||
with Live(console=console, refresh_per_second=4, transient=True) as live:
|
||||
while not result_container['done']:
|
||||
# Build status display
|
||||
status_text = Text()
|
||||
status_text.append("⏳ ", style="cyan")
|
||||
status_text.append(live_status.current_status or "Processing...", style="dim")
|
||||
|
||||
# Show queued messages if any
|
||||
queue_count = message_queue.count
|
||||
if queue_count > 0:
|
||||
status_text.append(f"\n📋 Queued: {queue_count}", style="dim yellow")
|
||||
|
||||
live.update(Panel(status_text, border_style="dim"))
|
||||
time.sleep(0.1)
|
||||
except KeyboardInterrupt:
|
||||
console.print("\n[dim]Processing interrupted[/dim]")
|
||||
# Wait for completion with periodic status updates (non-blocking style)
|
||||
start_time = time.time()
|
||||
last_status_update = ""
|
||||
while not result_container['done']:
|
||||
time.sleep(0.2)
|
||||
current_status = live_status.current_status
|
||||
# Only print status if it changed
|
||||
if current_status != last_status_update:
|
||||
# Use carriage return to update in place
|
||||
elapsed = time.time() - start_time
|
||||
queue_count = message_queue.count
|
||||
queue_info = f" | 📋 Queued: {queue_count}" if queue_count > 0 else ""
|
||||
sys.stdout.write(f"\r[dim cyan]⏳ {current_status} ({elapsed:.1f}s){queue_info}[/dim cyan]")
|
||||
sys.stdout.flush()
|
||||
last_status_update = current_status
|
||||
|
||||
# Clear the status line
|
||||
sys.stdout.write("\r" + " " * 80 + "\r")
|
||||
sys.stdout.flush()
|
||||
|
||||
# Wait for thread to finish
|
||||
bg_thread.join(timeout=1.0)
|
||||
|
||||
# Handle result
|
||||
if result_container['error']:
|
||||
console.print(f"\n[red]Error: {result_container['error']}[/red]")
|
||||
console.print(f"[red]Error: {result_container['error']}[/red]")
|
||||
elif result_container['response']:
|
||||
# Print response with streaming effect
|
||||
console.print()
|
||||
response_str = result_container['response']
|
||||
words = response_str.split()
|
||||
import sys
|
||||
for i, word in enumerate(words):
|
||||
console.print(word + " ", end="")
|
||||
sys.stdout.flush()
|
||||
|
||||
@@ -0,0 +1,54 @@
|
||||
"""
|
||||
Simple CLI entry point for praisonai schedule command.
|
||||
|
||||
This provides a standalone entry point that can be called from main.py
|
||||
"""
|
||||
|
||||
import sys
|
||||
from praisonai.cli.features.agent_scheduler import AgentSchedulerHandler
|
||||
|
||||
|
||||
def main():
|
||||
"""Main entry point for schedule CLI command."""
|
||||
import argparse
|
||||
|
||||
parser = argparse.ArgumentParser(
|
||||
description='Schedule an agent to run continuously at regular intervals'
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'schedule_yaml',
|
||||
nargs='?',
|
||||
default='agents.yaml',
|
||||
help='Path to agents.yaml file (default: agents.yaml)'
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--interval',
|
||||
dest='schedule_interval',
|
||||
type=str,
|
||||
help='Override schedule interval (e.g., "hourly", "*/30m", "daily")'
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--max-retries',
|
||||
dest='schedule_max_retries',
|
||||
type=int,
|
||||
help='Override maximum retry attempts (default: from YAML or 3)'
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--verbose', '-v',
|
||||
action='store_true',
|
||||
help='Enable verbose logging'
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
# Call the handler
|
||||
exit_code = AgentSchedulerHandler.handle_schedule_command(args)
|
||||
sys.exit(exit_code)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -57,7 +57,7 @@ class CloudDeployer:
|
||||
file.write("FROM python:3.11-slim\n")
|
||||
file.write("WORKDIR /app\n")
|
||||
file.write("COPY . .\n")
|
||||
file.write("RUN pip install flask praisonai==2.3.52 gunicorn markdown\n")
|
||||
file.write("RUN pip install flask praisonai==2.3.77 gunicorn markdown\n")
|
||||
file.write("EXPOSE 8080\n")
|
||||
file.write('CMD ["gunicorn", "-b", "0.0.0.0:8080", "api:app"]\n')
|
||||
|
||||
|
||||
@@ -135,7 +135,8 @@ class BaseCLIIntegration(ABC):
|
||||
*cmd,
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.PIPE,
|
||||
cwd=self.workspace
|
||||
cwd=self.workspace,
|
||||
env=self.get_env()
|
||||
)
|
||||
|
||||
try:
|
||||
@@ -173,7 +174,8 @@ class BaseCLIIntegration(ABC):
|
||||
*cmd,
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.PIPE,
|
||||
cwd=self.workspace
|
||||
cwd=self.workspace,
|
||||
env=self.get_env()
|
||||
)
|
||||
|
||||
try:
|
||||
@@ -208,7 +210,8 @@ class BaseCLIIntegration(ABC):
|
||||
*cmd,
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.PIPE,
|
||||
cwd=self.workspace
|
||||
cwd=self.workspace,
|
||||
env=self.get_env()
|
||||
)
|
||||
|
||||
try:
|
||||
|
||||
@@ -127,23 +127,16 @@ class ClaudeCodeIntegration(BaseCLIIntegration):
|
||||
"""
|
||||
cmd = ["claude"]
|
||||
|
||||
# Add continue flag if needed
|
||||
if continue_session or self._session_active:
|
||||
cmd.append("--continue")
|
||||
|
||||
# Add skip permissions flag
|
||||
if self.skip_permissions:
|
||||
cmd.append("--dangerously-skip-permissions")
|
||||
|
||||
# Add print mode flag
|
||||
# Add print mode flag for non-interactive output
|
||||
cmd.append("-p")
|
||||
|
||||
# Add prompt
|
||||
cmd.append(prompt)
|
||||
|
||||
# Add output format
|
||||
cmd.extend(["--output-format", self.output_format])
|
||||
|
||||
# Add continue flag if needed
|
||||
if continue_session or self._session_active:
|
||||
cmd.append("--continue")
|
||||
|
||||
# Add model if specified
|
||||
if self.model:
|
||||
cmd.extend(["--model", self.model])
|
||||
@@ -160,6 +153,13 @@ class ClaudeCodeIntegration(BaseCLIIntegration):
|
||||
if self.disallowed_tools:
|
||||
cmd.extend(["--disallowedTools", ",".join(self.disallowed_tools)])
|
||||
|
||||
# Add verbose if needed
|
||||
if options.get('verbose'):
|
||||
cmd.append("--verbose")
|
||||
|
||||
# Add prompt last
|
||||
cmd.append(prompt)
|
||||
|
||||
return cmd
|
||||
|
||||
async def execute(self, prompt: str, **options) -> str:
|
||||
|
||||
@@ -93,6 +93,9 @@ class CodexCLIIntegration(BaseCLIIntegration):
|
||||
"""
|
||||
cmd = ["codex", "exec"]
|
||||
|
||||
# Add working directory
|
||||
cmd.extend(["-C", self.workspace])
|
||||
|
||||
# Add task
|
||||
cmd.append(task)
|
||||
|
||||
|
||||
@@ -97,6 +97,9 @@ class CursorCLIIntegration(BaseCLIIntegration):
|
||||
# Add print mode flag
|
||||
cmd.append("-p")
|
||||
|
||||
# Add workspace
|
||||
cmd.extend(["--workspace", self.workspace])
|
||||
|
||||
# Add force flag if enabled
|
||||
if self.force:
|
||||
cmd.append("--force")
|
||||
|
||||
@@ -92,12 +92,6 @@ class GeminiCLIIntegration(BaseCLIIntegration):
|
||||
"""
|
||||
cmd = ["gemini"]
|
||||
|
||||
# Add print mode flag
|
||||
cmd.append("-p")
|
||||
|
||||
# Add prompt
|
||||
cmd.append(prompt)
|
||||
|
||||
# Add model
|
||||
cmd.extend(["-m", self.model])
|
||||
|
||||
@@ -112,6 +106,12 @@ class GeminiCLIIntegration(BaseCLIIntegration):
|
||||
if self.sandbox:
|
||||
cmd.append("--sandbox")
|
||||
|
||||
# Add YOLO mode for non-interactive execution
|
||||
cmd.append("--yolo")
|
||||
|
||||
# Add prompt as positional argument (must be last)
|
||||
cmd.append(prompt)
|
||||
|
||||
return cmd
|
||||
|
||||
async def execute(self, prompt: str, **options) -> str:
|
||||
|
||||
File diff suppressed because it is too large.
Load diff
@@ -1,13 +1,31 @@
|
||||
import re
|
||||
"""
|
||||
DEPRECATED: This module is deprecated. Use praisonai.scheduler instead.
|
||||
|
||||
This file is kept for backward compatibility only.
|
||||
All new code should import from praisonai.scheduler.
|
||||
"""
|
||||
|
||||
import warnings
|
||||
import logging
|
||||
import threading
|
||||
import time
|
||||
import logging
|
||||
from datetime import datetime, timedelta
|
||||
from typing import Union, Optional, Callable, Dict, Any
|
||||
from typing import Optional, Dict, Any
|
||||
from abc import ABC, abstractmethod
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Issue deprecation warning
|
||||
warnings.warn(
|
||||
"praisonai.scheduler module (root level) is deprecated. "
|
||||
"Use 'from praisonai.scheduler import ...' instead.",
|
||||
DeprecationWarning,
|
||||
stacklevel=2
|
||||
)
|
||||
|
||||
# Import from new scheduler module for backward compatibility
|
||||
from .scheduler.base import ScheduleParser
|
||||
|
||||
# Keep old classes for backward compatibility
|
||||
class DeployerInterface(ABC):
|
||||
"""Abstract interface for deployers to ensure provider compatibility."""
|
||||
|
||||
@@ -32,45 +50,6 @@ class CloudDeployerAdapter(DeployerInterface):
|
||||
logger.error(f"Deployment failed: {e}")
|
||||
return False
|
||||
|
||||
class ScheduleParser:
|
||||
"""Parse schedule expressions into intervals."""
|
||||
|
||||
@staticmethod
|
||||
def parse(schedule_expr: str) -> int:
|
||||
"""
|
||||
Parse schedule expression and return interval in seconds.
|
||||
|
||||
Supported formats:
|
||||
- "daily" -> 86400 seconds
|
||||
- "hourly" -> 3600 seconds
|
||||
- "*/30m" -> 1800 seconds (every 30 minutes)
|
||||
- "*/6h" -> 21600 seconds (every 6 hours)
|
||||
- "60" -> 60 seconds (plain number)
|
||||
"""
|
||||
schedule_expr = schedule_expr.strip().lower()
|
||||
|
||||
if schedule_expr == "daily":
|
||||
return 86400
|
||||
elif schedule_expr == "hourly":
|
||||
return 3600
|
||||
elif schedule_expr.isdigit():
|
||||
return int(schedule_expr)
|
||||
elif schedule_expr.startswith("*/"):
|
||||
# Handle */30m, */6h patterns
|
||||
interval_part = schedule_expr[2:]
|
||||
if interval_part.endswith("m"):
|
||||
minutes = int(interval_part[:-1])
|
||||
return minutes * 60
|
||||
elif interval_part.endswith("h"):
|
||||
hours = int(interval_part[:-1])
|
||||
return hours * 3600
|
||||
elif interval_part.endswith("s"):
|
||||
return int(interval_part[:-1])
|
||||
else:
|
||||
return int(interval_part)
|
||||
else:
|
||||
raise ValueError(f"Unsupported schedule format: {schedule_expr}")
|
||||
|
||||
class DeploymentScheduler:
|
||||
"""
|
||||
Minimal deployment scheduler with provider-agnostic design.
|
||||
|
||||
@@ -0,0 +1,274 @@
|
||||
# PraisonAI Scheduler - 24/7 Agent Scheduling
|
||||
|
||||
Complete implementation of 24/7 agent scheduling for PraisonAI, enabling autonomous agent operations at regular intervals.
|
||||
|
||||
## ✅ Implementation Status
|
||||
|
||||
### **Completed Features**
|
||||
|
||||
#### 1. **Core Scheduler Module** (`praisonai/scheduler/`)
|
||||
- ✅ `base.py` - ScheduleParser, ExecutorInterface, PraisonAgentExecutor
|
||||
- ✅ `agent_scheduler.py` - AgentScheduler with full 24/7 capabilities
|
||||
- ✅ `__init__.py` - Clean module exports with lazy loading
|
||||
|
||||
#### 2. **Schedule Parser**
|
||||
Supports multiple schedule formats:
|
||||
- `"hourly"` → Every hour (3600s)
|
||||
- `"daily"` → Every day (86400s)
|
||||
- `"*/30m"` → Every 30 minutes
|
||||
- `"*/6h"` → Every 6 hours
|
||||
- `"*/5s"` → Every 5 seconds
|
||||
- `"3600"` → Custom seconds (3600s)
|
||||
|
||||
#### 3. **AgentScheduler Features**
|
||||
- ✅ Thread-based execution (daemon threads)
|
||||
- ✅ Configurable intervals
|
||||
- ✅ Retry logic with exponential backoff (30s, 60s, 90s...)
|
||||
- ✅ Success/failure callbacks
|
||||
- ✅ Execution statistics tracking
|
||||
- ✅ Graceful shutdown with timeout
|
||||
- ✅ `run_immediately` option
|
||||
- ✅ Thread-safe operation
|
||||
|
||||
#### 4. **Testing**
|
||||
- ✅ 53 unit tests (100% passing)
|
||||
- ✅ 18 tests for base components
|
||||
- ✅ 35 tests for AgentScheduler
|
||||
- ✅ Real-world testing with live API calls
|
||||
- ✅ Verified with OpenAI API and search tools
|
||||
|
||||
#### 5. **Backward Compatibility**
|
||||
- ✅ Old `scheduler.py` imports from new module
|
||||
- ✅ Deprecation warnings added
|
||||
- ✅ Existing code continues to work
|
||||
|
||||
## 🚀 Quick Start
|
||||
|
||||
### Basic Usage
|
||||
|
||||
```python
|
||||
from praisonaiagents import Agent
|
||||
from praisonai.scheduler import AgentScheduler
|
||||
|
||||
# Create your agent
|
||||
agent = Agent(
|
||||
name="NewsChecker",
|
||||
instructions="Check latest AI news and summarize",
|
||||
tools=[search_tool]
|
||||
)
|
||||
|
||||
# Create scheduler
|
||||
scheduler = AgentScheduler(
|
||||
agent=agent,
|
||||
task="Search for latest AI news and provide top 3 stories"
|
||||
)
|
||||
|
||||
# Start scheduling (runs every hour)
|
||||
scheduler.start(
|
||||
schedule_expr="hourly",
|
||||
max_retries=3,
|
||||
run_immediately=True
|
||||
)
|
||||
|
||||
# Keep running until stopped
|
||||
try:
|
||||
while scheduler.is_running:
|
||||
import time
|
||||
time.sleep(1)
|
||||
except KeyboardInterrupt:
|
||||
scheduler.stop()
|
||||
print(scheduler.get_stats())
|
||||
```
|
||||
|
||||
### With Callbacks
|
||||
|
||||
```python
|
||||
def on_success(result):
|
||||
print(f"✅ Success: {result}")
|
||||
|
||||
def on_failure(error):
|
||||
print(f"❌ Failed: {error}")
|
||||
|
||||
scheduler = AgentScheduler(
|
||||
agent=agent,
|
||||
task="Your task",
|
||||
on_success=on_success,
|
||||
on_failure=on_failure
|
||||
)
|
||||
|
||||
scheduler.start("*/30m") # Every 30 minutes
|
||||
```
|
||||
|
||||
### Schedule Formats
|
||||
|
||||
```python
|
||||
# Predefined intervals
|
||||
scheduler.start("hourly") # Every hour
|
||||
scheduler.start("daily") # Every 24 hours
|
||||
|
||||
# Custom intervals
|
||||
scheduler.start("*/30m") # Every 30 minutes
|
||||
scheduler.start("*/6h") # Every 6 hours
|
||||
scheduler.start("*/5s") # Every 5 seconds
|
||||
|
||||
# Seconds
|
||||
scheduler.start("3600") # Every 3600 seconds (1 hour)
|
||||
```
|
||||
|
||||
## 📊 Statistics Tracking
|
||||
|
||||
```python
|
||||
stats = scheduler.get_stats()
|
||||
# Returns:
|
||||
# {
|
||||
# "is_running": True/False,
|
||||
# "total_executions": 10,
|
||||
# "successful_executions": 9,
|
||||
# "failed_executions": 1,
|
||||
# "success_rate": 90.0
|
||||
# }
|
||||
```
|
||||
|
||||
## 🔧 Advanced Features
|
||||
|
||||
### Retry Logic
|
||||
- Automatic retry on failure
|
||||
- Exponential backoff (30s, 60s, 90s...)
|
||||
- Configurable max retries
|
||||
|
||||
```python
|
||||
scheduler.start(
|
||||
schedule_expr="hourly",
|
||||
max_retries=5 # Retry up to 5 times
|
||||
)
|
||||
```
|
||||
|
||||
### Immediate Execution
|
||||
```python
|
||||
scheduler.start(
|
||||
schedule_expr="hourly",
|
||||
run_immediately=True # Run once before starting schedule
|
||||
)
|
||||
```
|
||||
|
||||
### One-Time Execution
|
||||
```python
|
||||
result = scheduler.execute_once() # Execute immediately, no scheduling
|
||||
```
|
||||
|
||||
### Graceful Shutdown
|
||||
```python
|
||||
scheduler.stop() # Waits up to 10 seconds for current execution to finish
|
||||
```
|
||||
|
||||
## 📁 File Structure
|
||||
|
||||
```
|
||||
praisonai/scheduler/
|
||||
├── __init__.py # Module exports
|
||||
├── base.py # ScheduleParser, ExecutorInterface
|
||||
├── agent_scheduler.py # AgentScheduler implementation
|
||||
└── README.md # This file
|
||||
|
||||
tests/unit/scheduler/
|
||||
├── __init__.py
|
||||
├── test_base.py # 18 tests for base components
|
||||
└── test_agent_scheduler.py # 35 tests for AgentScheduler
|
||||
```
|
||||
|
||||
## ✅ Test Results
|
||||
|
||||
```bash
|
||||
$ pytest tests/unit/scheduler/ -v
|
||||
============================= test session starts ==============================
|
||||
collected 53 items
|
||||
|
||||
tests/unit/scheduler/test_agent_scheduler.py::... PASSED [100%]
|
||||
tests/unit/scheduler/test_base.py::... PASSED [100%]
|
||||
|
||||
============================== 53 passed in 0.54s ===============================
|
||||
```
|
||||
|
||||
## 🎯 Real-World Example
|
||||
|
||||
See `examples/python/scheduled_agents/news_checker_live.py` for a complete working example that:
|
||||
- Checks AI news every 2 minutes (configurable to hourly)
|
||||
- Uses real OpenAI API
|
||||
- Uses DuckDuckGo search
|
||||
- Displays results with callbacks
|
||||
- Tracks statistics
|
||||
- Handles Ctrl+C gracefully
|
||||
|
||||
## 🔄 Migration from Old Scheduler
|
||||
|
||||
Old code continues to work with deprecation warning:
|
||||
|
||||
```python
|
||||
# Old (still works)
|
||||
from praisonai.scheduler import ScheduleParser
|
||||
|
||||
# New (recommended)
|
||||
from praisonai.scheduler import ScheduleParser
|
||||
```
|
||||
|
||||
## 📝 API Reference
|
||||
|
||||
### AgentScheduler
|
||||
|
||||
**Constructor:**
|
||||
```python
|
||||
AgentScheduler(
|
||||
agent, # PraisonAI Agent instance
|
||||
task: str, # Task description
|
||||
config: Optional[Dict] = None,
|
||||
on_success: Optional[Callable] = None,
|
||||
on_failure: Optional[Callable] = None
|
||||
)
|
||||
```
|
||||
|
||||
**Methods:**
|
||||
- `start(schedule_expr, max_retries=3, run_immediately=False) -> bool`
|
||||
- `stop() -> bool`
|
||||
- `execute_once() -> Any`
|
||||
- `get_stats() -> Dict`
|
||||
|
||||
### ScheduleParser
|
||||
|
||||
**Methods:**
|
||||
- `parse(schedule_expr: str) -> int` - Returns interval in seconds
|
||||
|
||||
### ExecutorInterface
|
||||
|
||||
**Abstract Methods:**
|
||||
- `execute(task: str) -> Any` - Must be implemented by subclasses
|
||||
|
||||
## 🎉 Success Metrics
|
||||
|
||||
- ✅ 53/53 tests passing (100%)
|
||||
- ✅ Real-world testing successful
|
||||
- ✅ Verified with live API calls
|
||||
- ✅ Thread-safe operation confirmed
|
||||
- ✅ Memory stable over multiple executions
|
||||
- ✅ Graceful shutdown working
|
||||
- ✅ Backward compatibility maintained
|
||||
|
||||
## 🚀 Production Ready
|
||||
|
||||
The scheduler is **production-ready** and can be used for:
|
||||
- 24/7 news monitoring
|
||||
- Periodic data collection
|
||||
- Scheduled report generation
|
||||
- Continuous monitoring tasks
|
||||
- Automated agent workflows
|
||||
|
||||
## 📚 Examples
|
||||
|
||||
See `examples/python/scheduled_agents/` for:
|
||||
- `simple_test.py` - Basic functionality test
|
||||
- `news_checker_live.py` - Real-world news checking agent
|
||||
- `news_checker_agent.py` - Full-featured example with documentation
|
||||
|
||||
## 🔗 Related
|
||||
|
||||
- Main PraisonAI docs: [docs.praison.ai](https://docs.praison.ai)
|
||||
- PraisonAI Agents: [github.com/MervinPraison/PraisonAI](https://github.com/MervinPraison/PraisonAI)
|
||||
@@ -0,0 +1,36 @@
|
||||
"""
|
||||
PraisonAI Scheduler Module
|
||||
|
||||
This module provides scheduling capabilities for running agents and deployments
|
||||
at regular intervals, enabling 24/7 autonomous operations.
|
||||
|
||||
Components:
|
||||
- ScheduleParser: Parse schedule expressions (hourly, daily, */30m, etc.)
|
||||
- ExecutorInterface: Abstract interface for executors
|
||||
- AgentScheduler: Schedule agent execution at regular intervals
|
||||
- DeploymentScheduler: Schedule deployment operations
|
||||
"""
|
||||
|
||||
from .base import ScheduleParser, ExecutorInterface, PraisonAgentExecutor
|
||||
|
||||
__all__ = [
|
||||
'ScheduleParser',
|
||||
'ExecutorInterface',
|
||||
'PraisonAgentExecutor',
|
||||
]
|
||||
|
||||
# Lazy imports for modules that will be created later
|
||||
def __getattr__(name):
|
||||
if name == 'AgentScheduler':
|
||||
from .agent_scheduler import AgentScheduler
|
||||
return AgentScheduler
|
||||
elif name == 'create_agent_scheduler':
|
||||
from .agent_scheduler import create_agent_scheduler
|
||||
return create_agent_scheduler
|
||||
elif name == 'DeploymentScheduler':
|
||||
from .deployment_scheduler import DeploymentScheduler
|
||||
return DeploymentScheduler
|
||||
elif name == 'create_deployment_scheduler':
|
||||
from .deployment_scheduler import create_deployment_scheduler
|
||||
return create_deployment_scheduler
|
||||
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
||||
@@ -0,0 +1,455 @@
|
||||
"""
|
||||
Agent Scheduler for PraisonAI - Run agents periodically 24/7.
|
||||
|
||||
This module provides scheduling capabilities for running PraisonAI agents
|
||||
at regular intervals, enabling 24/7 autonomous agent operations.
|
||||
"""
|
||||
|
||||
import threading
|
||||
import time
|
||||
import logging
|
||||
from datetime import datetime
|
||||
from typing import Optional, Dict, Any, Callable
|
||||
|
||||
from .base import ScheduleParser, PraisonAgentExecutor
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class AgentScheduler:
|
||||
"""
|
||||
Scheduler for running PraisonAI agents periodically.
|
||||
|
||||
Features:
|
||||
- Interval-based scheduling (hourly, daily, custom)
|
||||
- Thread-safe operation
|
||||
- Automatic retry on failure
|
||||
- Execution logging and monitoring
|
||||
- Graceful shutdown
|
||||
|
||||
Example:
|
||||
scheduler = AgentScheduler(agent, task="Check news")
|
||||
scheduler.start(schedule_expr="hourly", max_retries=3)
|
||||
# Agent runs every hour automatically
|
||||
scheduler.stop() # Stop when needed
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
agent,
|
||||
task: str,
|
||||
config: Optional[Dict[str, Any]] = None,
|
||||
on_success: Optional[Callable] = None,
|
||||
on_failure: Optional[Callable] = None,
|
||||
timeout: Optional[int] = None,
|
||||
max_cost: Optional[float] = 1.00
|
||||
):
|
||||
"""
|
||||
Initialize agent scheduler.
|
||||
|
||||
Args:
|
||||
agent: PraisonAI Agent instance
|
||||
task: Task description to execute
|
||||
config: Optional configuration dict
|
||||
on_success: Callback function on successful execution
|
||||
on_failure: Callback function on failed execution
|
||||
timeout: Maximum execution time per run in seconds (None = no limit)
|
||||
max_cost: Maximum total cost in USD (default: $1.00 for safety)
|
||||
"""
|
||||
self.agent = agent
|
||||
self.task = task
|
||||
self.config = config or {}
|
||||
self.on_success = on_success
|
||||
self.on_failure = on_failure
|
||||
self.timeout = timeout
|
||||
self.max_cost = max_cost
|
||||
|
||||
self.is_running = False
|
||||
self._stop_event = threading.Event()
|
||||
self._thread = None
|
||||
self._executor = PraisonAgentExecutor(agent)
|
||||
self._execution_count = 0
|
||||
self._success_count = 0
|
||||
self._failure_count = 0
|
||||
self._total_cost = 0.0
|
||||
self._start_time = None
|
||||
|
||||
def start(
|
||||
self,
|
||||
schedule_expr: str,
|
||||
max_retries: int = 3,
|
||||
run_immediately: bool = False
|
||||
) -> bool:
|
||||
"""
|
||||
Start scheduled agent execution.
|
||||
|
||||
Args:
|
||||
schedule_expr: Schedule expression (e.g., "hourly", "*/6h", "3600")
|
||||
max_retries: Maximum retry attempts on failure
|
||||
run_immediately: If True, run agent immediately before starting schedule
|
||||
|
||||
Returns:
|
||||
True if scheduler started successfully
|
||||
"""
|
||||
if self.is_running:
|
||||
logger.warning("Scheduler is already running")
|
||||
return False
|
||||
|
||||
try:
|
||||
interval = ScheduleParser.parse(schedule_expr)
|
||||
self.is_running = True
|
||||
self._stop_event.clear()
|
||||
|
||||
logger.debug(f"Starting agent scheduler: {getattr(self.agent, 'name', 'Agent')}")
|
||||
logger.debug(f"Task: {self.task}")
|
||||
logger.debug(f"Schedule: {schedule_expr} ({interval}s interval)")
|
||||
self.is_running = True
|
||||
self._stop_event.clear()
|
||||
self._start_time = datetime.now()
|
||||
|
||||
# Run immediately if requested
|
||||
if run_immediately:
|
||||
logger.debug("Running agent immediately before starting schedule...")
|
||||
self._execute_with_retry(max_retries)
|
||||
|
||||
self._thread = threading.Thread(
|
||||
target=self._run_schedule,
|
||||
args=(interval, max_retries),
|
||||
daemon=True
|
||||
)
|
||||
self._thread.start()
|
||||
|
||||
logger.debug("Agent scheduler started successfully")
|
||||
if self.timeout:
|
||||
logger.info(f"Timeout per execution: {self.timeout}s")
|
||||
if self.max_cost:
|
||||
logger.info(f"Budget limit: ${self.max_cost}")
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to start scheduler: {e}")
|
||||
self.is_running = False
|
||||
return False
|
||||
|
||||
def stop(self) -> bool:
|
||||
"""
|
||||
Stop the scheduler gracefully.
|
||||
|
||||
Returns:
|
||||
True if stopped successfully
|
||||
"""
|
||||
if not self.is_running:
|
||||
logger.debug("Scheduler is not running")
|
||||
return True
|
||||
|
||||
logger.debug("Stopping agent scheduler...")
|
||||
self._stop_event.set()
|
||||
|
||||
if self._thread and self._thread.is_alive():
|
||||
self._thread.join(timeout=10)
|
||||
|
||||
self.is_running = False
|
||||
logger.debug("Agent scheduler stopped")
|
||||
logger.debug(f"Execution stats - Total: {self._execution_count}, Success: {self._success_count}, Failed: {self._failure_count}")
|
||||
return True
|
||||
|
||||
def get_stats(self) -> Dict[str, Any]:
|
||||
"""
|
||||
Get execution statistics.
|
||||
|
||||
Returns:
|
||||
Dictionary with execution stats including cost
|
||||
"""
|
||||
runtime = (datetime.now() - self._start_time).total_seconds() if self._start_time else 0
|
||||
return {
|
||||
"is_running": self.is_running,
|
||||
"total_executions": self._execution_count,
|
||||
"successful_executions": self._success_count,
|
||||
"failed_executions": self._failure_count,
|
||||
"success_rate": (self._success_count / self._execution_count * 100) if self._execution_count > 0 else 0,
|
||||
"total_cost_usd": round(self._total_cost, 4),
|
||||
"runtime_seconds": round(runtime, 1),
|
||||
"cost_per_execution": round(self._total_cost / self._execution_count, 4) if self._execution_count > 0 else 0
|
||||
}
|
||||
|
||||
def _update_state_if_daemon(self):
|
||||
"""Update state file with execution stats if running as daemon."""
|
||||
try:
|
||||
import os
|
||||
# Check if we're running as a daemon by looking for state file
|
||||
state_dir = os.path.expanduser("~/.praisonai/schedulers")
|
||||
if not os.path.exists(state_dir):
|
||||
return
|
||||
|
||||
# Try to find our state file by checking all state files for matching PID
|
||||
current_pid = os.getpid()
|
||||
for state_file in os.listdir(state_dir):
|
||||
if not state_file.endswith('.json'):
|
||||
continue
|
||||
|
||||
state_path = os.path.join(state_dir, state_file)
|
||||
try:
|
||||
import json
|
||||
with open(state_path, 'r') as f:
|
||||
state = json.load(f)
|
||||
|
||||
# Check if this is our state file
|
||||
if state.get('pid') == current_pid:
|
||||
# Update execution stats
|
||||
state['executions'] = self._execution_count
|
||||
state['cost'] = round(self._total_cost, 4)
|
||||
|
||||
# Write back
|
||||
with open(state_path, 'w') as f:
|
||||
json.dump(state, f, indent=2)
|
||||
break
|
||||
except Exception:
|
||||
continue
|
||||
except Exception as e:
|
||||
# Silently fail - don't break scheduler if state update fails
|
||||
logger.debug(f"Failed to update state: {e}")
|
||||
|
||||
def _run_schedule(self, interval: int, max_retries: int):
|
||||
"""Internal method to run scheduled agent executions."""
|
||||
while not self._stop_event.is_set():
|
||||
# Check budget limit
|
||||
if self.max_cost and self._total_cost >= self.max_cost:
|
||||
logger.warning(f"Budget limit reached: ${self._total_cost:.4f} >= ${self.max_cost}")
|
||||
logger.warning("Stopping scheduler to prevent additional costs")
|
||||
self.stop()
|
||||
break
|
||||
|
||||
logger.debug(f"[{datetime.now().strftime('%Y-%m-%d %H:%M:%S')}] Starting scheduled agent execution")
|
||||
|
||||
self._execute_with_retry(max_retries)
|
||||
|
||||
# Wait for next scheduled time
|
||||
logger.debug(f"Next execution in {interval} seconds ({interval/3600:.1f} hours)")
|
||||
if self.max_cost:
|
||||
remaining = self.max_cost - self._total_cost
|
||||
logger.debug(f"Budget remaining: ${remaining:.4f}")
|
||||
self._stop_event.wait(interval)
|
||||
|
||||
def _execute_with_retry(self, max_retries: int):
|
||||
"""Execute agent with retry logic and timeout."""
|
||||
self._execution_count += 1
|
||||
success = False
|
||||
result = None
|
||||
|
||||
for attempt in range(max_retries):
|
||||
try:
|
||||
logger.debug(f"Attempt {attempt + 1}/{max_retries}")
|
||||
|
||||
# Execute with timeout if specified
|
||||
if self.timeout:
|
||||
import signal
|
||||
|
||||
def timeout_handler(signum, frame):
|
||||
raise TimeoutError(f"Execution exceeded {self.timeout}s timeout")
|
||||
|
||||
# Set timeout alarm (Unix only)
|
||||
try:
|
||||
signal.signal(signal.SIGALRM, timeout_handler)
|
||||
signal.alarm(self.timeout)
|
||||
result = self._executor.execute(self.task)
|
||||
signal.alarm(0) # Cancel alarm
|
||||
except AttributeError:
|
||||
# Windows doesn't support SIGALRM, use threading.Timer fallback
|
||||
logger.warning("Timeout not supported on this platform, executing without timeout")
|
||||
result = self._executor.execute(self.task)
|
||||
else:
|
||||
result = self._executor.execute(self.task)
|
||||
|
||||
logger.debug(f"Agent execution successful on attempt {attempt + 1}")
|
||||
logger.debug(f"Result: {result}")
|
||||
|
||||
# Always print result to stdout (even in non-verbose mode)
|
||||
print(f"\n✅ Agent Response:\n{result}\n")
|
||||
|
||||
# Estimate cost (rough: ~$0.0001 per execution for gpt-4o-mini)
|
||||
estimated_cost = 0.0001 # Base cost estimate
|
||||
self._total_cost += estimated_cost
|
||||
logger.debug(f"Estimated cost this run: ${estimated_cost:.4f}, Total: ${self._total_cost:.4f}")
|
||||
|
||||
self._success_count += 1
|
||||
success = True
|
||||
|
||||
if self.on_success:
|
||||
try:
|
||||
self.on_success(result)
|
||||
except Exception as e:
|
||||
logger.error(f"Callback error in on_success: {e}")
|
||||
|
||||
# Update state file after successful execution
|
||||
self._update_state_if_daemon()
|
||||
|
||||
break
|
||||
|
||||
except TimeoutError as e:
|
||||
logger.error(f"Execution timeout on attempt {attempt + 1}: {e}")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Agent execution failed on attempt {attempt + 1}: {e}")
|
||||
|
||||
if attempt < max_retries - 1:
|
||||
wait_time = 30 * (attempt + 1) # Exponential backoff
|
||||
logger.debug(f"Waiting {wait_time}s before retry...")
|
||||
time.sleep(wait_time)
|
||||
|
||||
if not success:
|
||||
self._failure_count += 1
|
||||
logger.error(f"Agent execution failed after {max_retries} attempts")
|
||||
|
||||
if self.on_failure:
|
||||
try:
|
||||
self.on_failure(f"Failed after {max_retries} attempts")
|
||||
except Exception as e:
|
||||
logger.error(f"Callback error in on_failure: {e}")
|
||||
|
||||
# Update state file even after failure
|
||||
self._update_state_if_daemon()
|
||||
|
||||
def execute_once(self) -> Any:
|
||||
"""
|
||||
Execute agent immediately (one-time execution).
|
||||
|
||||
Returns:
|
||||
Agent execution result
|
||||
"""
|
||||
logger.debug("Executing agent once")
|
||||
try:
|
||||
result = self._executor.execute(self.task)
|
||||
logger.debug(f"One-time execution successful: {result}")
|
||||
|
||||
if self.on_success:
|
||||
try:
|
||||
self.on_success(result)
|
||||
except Exception as e:
|
||||
logger.error(f"Callback error in on_success: {e}")
|
||||
|
||||
return result
|
||||
except Exception as e:
|
||||
logger.error(f"One-time execution failed: {e}")
|
||||
raise
|
||||
|
||||
@classmethod
|
||||
def from_yaml(
|
||||
cls,
|
||||
yaml_path: str = "agents.yaml",
|
||||
interval_override: Optional[str] = None,
|
||||
max_retries_override: Optional[int] = None,
|
||||
timeout_override: Optional[int] = None,
|
||||
max_cost_override: Optional[float] = None,
|
||||
on_success: Optional[Callable] = None,
|
||||
on_failure: Optional[Callable] = None
|
||||
) -> 'AgentScheduler':
|
||||
"""
|
||||
Create AgentScheduler from agents.yaml file.
|
||||
|
||||
Args:
|
||||
yaml_path: Path to agents.yaml file
|
||||
interval_override: Override schedule interval from YAML
|
||||
max_retries_override: Override max_retries from YAML
|
||||
on_success: Callback function on successful execution
|
||||
on_failure: Callback function on failed execution
|
||||
|
||||
Returns:
|
||||
Configured AgentScheduler instance
|
||||
|
||||
Example:
|
||||
scheduler = AgentScheduler.from_yaml("agents.yaml")
|
||||
scheduler.start()
|
||||
|
||||
Example agents.yaml:
|
||||
framework: praisonai
|
||||
|
||||
agents:
|
||||
- name: "AI News Monitor"
|
||||
role: "News Analyst"
|
||||
instructions: "Search and summarize AI news"
|
||||
tools:
|
||||
- search_tool
|
||||
|
||||
task: "Search for latest AI news"
|
||||
|
||||
schedule:
|
||||
interval: "hourly"
|
||||
max_retries: 3
|
||||
run_immediately: true
|
||||
"""
|
||||
from .yaml_loader import load_agent_yaml_with_schedule, create_agent_from_config
|
||||
|
||||
# Load configuration from YAML
|
||||
agent_config, schedule_config = load_agent_yaml_with_schedule(yaml_path)
|
||||
|
||||
# Create agent from config
|
||||
agent = create_agent_from_config(agent_config)
|
||||
|
||||
# Get task
|
||||
task = agent_config.get('task', '')
|
||||
if not task:
|
||||
raise ValueError("No task specified in YAML file")
|
||||
|
||||
# Apply overrides to schedule config
|
||||
if interval_override:
|
||||
schedule_config['interval'] = interval_override
|
||||
if max_retries_override is not None:
|
||||
schedule_config['max_retries'] = max_retries_override
|
||||
if timeout_override is not None:
|
||||
schedule_config['timeout'] = timeout_override
|
||||
if max_cost_override is not None:
|
||||
schedule_config['max_cost'] = max_cost_override
|
||||
|
||||
# Create scheduler instance with timeout and cost limits
|
||||
scheduler = cls(
|
||||
agent=agent,
|
||||
task=task,
|
||||
config=agent_config,
|
||||
timeout=schedule_config.get('timeout'),
|
||||
max_cost=schedule_config.get('max_cost'),
|
||||
on_success=on_success,
|
||||
on_failure=on_failure
|
||||
)
|
||||
|
||||
# Store schedule config for later use
|
||||
scheduler._yaml_schedule_config = schedule_config
|
||||
|
||||
return scheduler
|
||||
|
||||
def start_from_yaml_config(self) -> bool:
|
||||
"""
|
||||
Start scheduler using configuration from YAML file.
|
||||
|
||||
Must be called after from_yaml() class method.
|
||||
|
||||
Returns:
|
||||
True if started successfully
|
||||
"""
|
||||
if not hasattr(self, '_yaml_schedule_config'):
|
||||
raise ValueError("No YAML configuration found. Use from_yaml() first.")
|
||||
|
||||
schedule_config = self._yaml_schedule_config
|
||||
interval = schedule_config.get('interval', 'hourly')
|
||||
max_retries = schedule_config.get('max_retries', 3)
|
||||
run_immediately = schedule_config.get('run_immediately', False)
|
||||
|
||||
return self.start(interval, max_retries, run_immediately)
|
||||
|
||||
|
||||
def create_agent_scheduler(
|
||||
agent,
|
||||
task: str,
|
||||
config: Optional[Dict[str, Any]] = None
|
||||
) -> AgentScheduler:
|
||||
"""
|
||||
Factory function to create agent scheduler.
|
||||
|
||||
Args:
|
||||
agent: PraisonAI Agent instance
|
||||
task: Task description
|
||||
config: Optional configuration
|
||||
|
||||
Returns:
|
||||
Configured AgentScheduler instance
|
||||
"""
|
||||
return AgentScheduler(agent, task, config)
|
||||
@@ -0,0 +1,124 @@
|
||||
"""
|
||||
Base components for PraisonAI Scheduler.
|
||||
|
||||
This module provides shared functionality for all schedulers:
|
||||
- ScheduleParser: Parse schedule expressions into intervals
|
||||
- ExecutorInterface: Abstract interface for executors
|
||||
- PraisonAgentExecutor: Executor for PraisonAI agents
|
||||
"""
|
||||
|
||||
import logging
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import Any
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ScheduleParser:
|
||||
"""Parse schedule expressions into intervals in seconds."""
|
||||
|
||||
@staticmethod
|
||||
def parse(schedule_expr: str) -> int:
|
||||
"""
|
||||
Parse schedule expression and return interval in seconds.
|
||||
|
||||
Supported formats:
|
||||
- "daily" -> 86400 seconds
|
||||
- "hourly" -> 3600 seconds
|
||||
- "*/30m" -> 1800 seconds (every 30 minutes)
|
||||
- "*/6h" -> 21600 seconds (every 6 hours)
|
||||
- "*/30s" -> 30 seconds (every 30 seconds)
|
||||
- "3600" -> 3600 seconds (plain number)
|
||||
|
||||
Args:
|
||||
schedule_expr: Schedule expression string
|
||||
|
||||
Returns:
|
||||
Interval in seconds
|
||||
|
||||
Raises:
|
||||
ValueError: If schedule format is not supported
|
||||
|
||||
Examples:
|
||||
>>> ScheduleParser.parse("hourly")
|
||||
3600
|
||||
>>> ScheduleParser.parse("*/30m")
|
||||
1800
|
||||
>>> ScheduleParser.parse("daily")
|
||||
86400
|
||||
"""
|
||||
schedule_expr = schedule_expr.strip().lower()
|
||||
|
||||
if schedule_expr == "daily":
|
||||
return 86400
|
||||
elif schedule_expr == "hourly":
|
||||
return 3600
|
||||
elif schedule_expr.isdigit():
|
||||
return int(schedule_expr)
|
||||
elif schedule_expr.startswith("*/"):
|
||||
interval_part = schedule_expr[2:]
|
||||
if interval_part.endswith("m"):
|
||||
minutes = int(interval_part[:-1])
|
||||
return minutes * 60
|
||||
elif interval_part.endswith("h"):
|
||||
hours = int(interval_part[:-1])
|
||||
return hours * 3600
|
||||
elif interval_part.endswith("s"):
|
||||
return int(interval_part[:-1])
|
||||
else:
|
||||
return int(interval_part)
|
||||
else:
|
||||
raise ValueError(f"Unsupported schedule format: {schedule_expr}")
|
||||
|
||||
|
||||
class ExecutorInterface(ABC):
|
||||
"""Abstract interface for executors."""
|
||||
|
||||
@abstractmethod
|
||||
def execute(self, task: str) -> Any:
|
||||
"""
|
||||
Execute a task.
|
||||
|
||||
Args:
|
||||
task: Task description or instruction
|
||||
|
||||
Returns:
|
||||
Execution result
|
||||
|
||||
Raises:
|
||||
Exception: If execution fails
|
||||
"""
|
||||
pass
|
||||
|
||||
|
||||
class PraisonAgentExecutor(ExecutorInterface):
|
||||
"""Executor for PraisonAI agents."""
|
||||
|
||||
def __init__(self, agent):
|
||||
"""
|
||||
Initialize executor with a PraisonAI agent.
|
||||
|
||||
Args:
|
||||
agent: PraisonAI Agent instance (must have start() method)
|
||||
"""
|
||||
self.agent = agent
|
||||
|
||||
def execute(self, task: str) -> Any:
|
||||
"""
|
||||
Execute the agent with given task.
|
||||
|
||||
Args:
|
||||
task: Task description for the agent
|
||||
|
||||
Returns:
|
||||
Agent execution result
|
||||
|
||||
Raises:
|
||||
Exception: If agent execution fails
|
||||
"""
|
||||
try:
|
||||
result = self.agent.start(task)
|
||||
return result
|
||||
except Exception as e:
|
||||
logger.error(f"Agent execution failed: {e}")
|
||||
raise
|
||||
@@ -0,0 +1,215 @@
|
||||
"""
|
||||
Daemon process manager for running schedulers in background.
|
||||
"""
|
||||
import os
|
||||
import sys
|
||||
import signal
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
from typing import Optional, Dict, List
|
||||
from datetime import datetime
|
||||
|
||||
|
||||
class DaemonManager:
|
||||
"""Manages daemon processes for schedulers."""
|
||||
|
||||
def __init__(self, log_dir: Optional[Path] = None, max_log_size_mb: float = 10.0):
|
||||
"""
|
||||
Initialize daemon manager.
|
||||
|
||||
Args:
|
||||
log_dir: Directory for log files. Defaults to ~/.praisonai/logs
|
||||
max_log_size_mb: Maximum log file size in MB before rotation
|
||||
"""
|
||||
if log_dir is None:
|
||||
home = Path.home()
|
||||
log_dir = home / ".praisonai" / "logs"
|
||||
|
||||
self.log_dir = Path(log_dir)
|
||||
self.log_dir.mkdir(parents=True, exist_ok=True)
|
||||
self.max_log_size_bytes = int(max_log_size_mb * 1024 * 1024)
|
||||
|
||||
def start_daemon(
|
||||
self,
|
||||
name: str,
|
||||
task: str,
|
||||
interval: str,
|
||||
command: List[str]
|
||||
) -> int:
|
||||
"""
|
||||
Start a daemon process.
|
||||
|
||||
Args:
|
||||
name: Daemon name
|
||||
task: Task description
|
||||
interval: Schedule interval
|
||||
command: Command to run as list
|
||||
|
||||
Returns:
|
||||
Process ID
|
||||
"""
|
||||
log_file = self.log_dir / f"{name}.log"
|
||||
|
||||
# Open log file - use original working approach
|
||||
with open(log_file, 'a') as log:
|
||||
log.flush()
|
||||
|
||||
# Start process as daemon
|
||||
proc = subprocess.Popen(
|
||||
command,
|
||||
stdout=log,
|
||||
stderr=subprocess.STDOUT,
|
||||
stdin=subprocess.DEVNULL,
|
||||
start_new_session=True, # Detach from terminal
|
||||
cwd=os.getcwd()
|
||||
)
|
||||
|
||||
return proc.pid
|
||||
|
||||
def start_scheduler_daemon(
|
||||
self,
|
||||
name: str,
|
||||
task: str,
|
||||
interval: str,
|
||||
max_cost: Optional[float] = None,
|
||||
timeout: Optional[int] = None,
|
||||
max_retries: int = 3
|
||||
) -> int:
|
||||
"""
|
||||
Start a PraisonAI scheduler as daemon.
|
||||
|
||||
Args:
|
||||
name: Scheduler name
|
||||
task: Task to schedule
|
||||
interval: Schedule interval
|
||||
max_cost: Maximum cost budget
|
||||
timeout: Timeout per execution
|
||||
max_retries: Maximum retry attempts
|
||||
|
||||
Returns:
|
||||
Process ID
|
||||
"""
|
||||
# Build command - use CLI without --daemon flag (original working approach)
|
||||
command = [
|
||||
sys.executable,
|
||||
"-m",
|
||||
"praisonai.cli.main",
|
||||
"schedule",
|
||||
task,
|
||||
"--interval", interval,
|
||||
"--max-retries", str(max_retries)
|
||||
]
|
||||
|
||||
if timeout:
|
||||
command.extend(["--timeout", str(timeout)])
|
||||
|
||||
if max_cost:
|
||||
command.extend(["--max-cost", str(max_cost)])
|
||||
|
||||
return self.start_daemon(name, task, interval, command)
|
||||
|
||||
def stop_daemon(self, pid: int, timeout: int = 10) -> bool:
|
||||
"""
|
||||
Stop a daemon process gracefully.
|
||||
|
||||
Args:
|
||||
pid: Process ID
|
||||
timeout: Timeout in seconds
|
||||
|
||||
Returns:
|
||||
True if stopped successfully
|
||||
"""
|
||||
try:
|
||||
# Try graceful shutdown first (SIGTERM)
|
||||
os.kill(pid, signal.SIGTERM)
|
||||
|
||||
# Wait for process to terminate
|
||||
import time
|
||||
for _ in range(timeout * 10):
|
||||
try:
|
||||
os.kill(pid, 0) # Check if still alive
|
||||
time.sleep(0.1)
|
||||
except (OSError, ProcessLookupError):
|
||||
return True # Process terminated
|
||||
|
||||
# Force kill if still alive
|
||||
try:
|
||||
os.kill(pid, signal.SIGKILL)
|
||||
time.sleep(0.2) # Give it time to die
|
||||
except (OSError, ProcessLookupError):
|
||||
pass
|
||||
|
||||
return True
|
||||
|
||||
except (OSError, ProcessLookupError):
|
||||
return False
|
||||
|
||||
def get_status(self, pid: int) -> Optional[Dict]:
|
||||
"""
|
||||
Get daemon process status.
|
||||
|
||||
Args:
|
||||
pid: Process ID
|
||||
|
||||
Returns:
|
||||
Status dictionary or None if not found
|
||||
"""
|
||||
try:
|
||||
os.kill(pid, 0) # Check if process exists
|
||||
|
||||
return {
|
||||
"pid": pid,
|
||||
"is_alive": True
|
||||
}
|
||||
except (OSError, ProcessLookupError):
|
||||
return {
|
||||
"pid": pid,
|
||||
"is_alive": False
|
||||
}
|
||||
|
||||
def read_logs(self, name: str, lines: int = 50) -> Optional[str]:
|
||||
"""
|
||||
Read daemon logs.
|
||||
|
||||
Args:
|
||||
name: Daemon name
|
||||
lines: Number of lines to read from end
|
||||
|
||||
Returns:
|
||||
Log content or None if not found
|
||||
"""
|
||||
log_file = self.log_dir / f"{name}.log"
|
||||
|
||||
if not log_file.exists():
|
||||
return None
|
||||
|
||||
try:
|
||||
with open(log_file) as f:
|
||||
all_lines = f.readlines()
|
||||
return ''.join(all_lines[-lines:])
|
||||
except IOError:
|
||||
return None
|
||||
|
||||
def rotate_log(self, name: str) -> bool:
|
||||
"""
|
||||
Rotate log file if it exceeds max size.
|
||||
|
||||
Args:
|
||||
name: Daemon name
|
||||
|
||||
Returns:
|
||||
True if rotated
|
||||
"""
|
||||
log_file = self.log_dir / f"{name}.log"
|
||||
|
||||
if not log_file.exists():
|
||||
return False
|
||||
|
||||
if log_file.stat().st_size > self.max_log_size_bytes:
|
||||
# Rotate: rename to .log.1, .log.2, etc.
|
||||
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
|
||||
rotated_file = self.log_dir / f"{name}.log.{timestamp}"
|
||||
log_file.rename(rotated_file)
|
||||
return True
|
||||
|
||||
return False
|
||||
@@ -0,0 +1,155 @@
|
||||
"""
|
||||
Scheduler state manager for persistent storage of scheduler processes.
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import signal
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Optional
|
||||
from datetime import datetime
|
||||
|
||||
|
||||
class SchedulerStateManager:
|
||||
"""Manages persistent state for scheduler processes."""
|
||||
|
||||
def __init__(self, state_dir: Optional[Path] = None):
|
||||
"""
|
||||
Initialize state manager.
|
||||
|
||||
Args:
|
||||
state_dir: Directory to store state files. Defaults to ~/.praisonai/schedulers
|
||||
"""
|
||||
if state_dir is None:
|
||||
home = Path.home()
|
||||
state_dir = home / ".praisonai" / "schedulers"
|
||||
|
||||
self.state_dir = Path(state_dir)
|
||||
self.state_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
def save_state(self, name: str, state: Dict) -> None:
|
||||
"""
|
||||
Save scheduler state to JSON file.
|
||||
|
||||
Args:
|
||||
name: Scheduler name
|
||||
state: State dictionary to save
|
||||
"""
|
||||
state_file = self.state_dir / f"{name}.json"
|
||||
|
||||
with open(state_file, 'w') as f:
|
||||
json.dump(state, f, indent=2)
|
||||
|
||||
def load_state(self, name: str) -> Optional[Dict]:
|
||||
"""
|
||||
Load scheduler state from JSON file.
|
||||
|
||||
Args:
|
||||
name: Scheduler name
|
||||
|
||||
Returns:
|
||||
State dictionary or None if not found
|
||||
"""
|
||||
state_file = self.state_dir / f"{name}.json"
|
||||
|
||||
if not state_file.exists():
|
||||
return None
|
||||
|
||||
try:
|
||||
with open(state_file) as f:
|
||||
return json.load(f)
|
||||
except (json.JSONDecodeError, IOError):
|
||||
return None
|
||||
|
||||
def delete_state(self, name: str) -> bool:
|
||||
"""
|
||||
Delete scheduler state file.
|
||||
|
||||
Args:
|
||||
name: Scheduler name
|
||||
|
||||
Returns:
|
||||
True if deleted, False if not found
|
||||
"""
|
||||
state_file = self.state_dir / f"{name}.json"
|
||||
|
||||
if state_file.exists():
|
||||
state_file.unlink()
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
def list_all(self) -> List[Dict]:
|
||||
"""
|
||||
List all scheduler states.
|
||||
|
||||
Returns:
|
||||
List of state dictionaries
|
||||
"""
|
||||
states = []
|
||||
|
||||
for state_file in self.state_dir.glob("*.json"):
|
||||
try:
|
||||
with open(state_file) as f:
|
||||
state = json.load(f)
|
||||
states.append(state)
|
||||
except (json.JSONDecodeError, IOError):
|
||||
continue
|
||||
|
||||
return states
|
||||
|
||||
def generate_unique_name(self, base_name: str = "scheduler") -> str:
|
||||
"""
|
||||
Generate a unique scheduler name.
|
||||
|
||||
Args:
|
||||
base_name: Base name for the scheduler
|
||||
|
||||
Returns:
|
||||
Unique name like "scheduler-0", "scheduler-1", etc.
|
||||
"""
|
||||
existing_states = self.list_all()
|
||||
existing_names = {s.get("name", "") for s in existing_states}
|
||||
|
||||
counter = 0
|
||||
while True:
|
||||
name = f"{base_name}-{counter}"
|
||||
if name not in existing_names:
|
||||
return name
|
||||
counter += 1
|
||||
|
||||
def is_process_alive(self, pid: int) -> bool:
|
||||
"""
|
||||
Check if a process is still alive.
|
||||
|
||||
Args:
|
||||
pid: Process ID
|
||||
|
||||
Returns:
|
||||
True if process is alive, False otherwise
|
||||
"""
|
||||
try:
|
||||
# Send signal 0 to check if process exists
|
||||
os.kill(pid, 0)
|
||||
return True
|
||||
except (OSError, ProcessLookupError):
|
||||
return False
|
||||
|
||||
def cleanup_dead_processes(self) -> int:
|
||||
"""
|
||||
Clean up state files for dead processes.
|
||||
|
||||
Returns:
|
||||
Number of dead processes cleaned up
|
||||
"""
|
||||
cleaned = 0
|
||||
states = self.list_all()
|
||||
|
||||
for state in states:
|
||||
pid = state.get("pid")
|
||||
name = state.get("name")
|
||||
|
||||
if pid and name and not self.is_process_alive(pid):
|
||||
self.delete_state(name)
|
||||
cleaned += 1
|
||||
|
||||
return cleaned
|
||||
@@ -0,0 +1,193 @@
|
||||
"""
|
||||
YAML loader for agent scheduler configuration.
|
||||
|
||||
Loads agent and schedule configuration from agents.yaml files.
|
||||
"""
|
||||
|
||||
import yaml
|
||||
import logging
|
||||
from typing import Dict, Any, Tuple
|
||||
from pathlib import Path
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def load_agent_yaml_with_schedule(yaml_path: str) -> Tuple[Dict[str, Any], Dict[str, Any]]:
|
||||
"""
|
||||
Load agents.yaml file and extract agent and schedule configuration.
|
||||
|
||||
Args:
|
||||
yaml_path: Path to agents.yaml file
|
||||
|
||||
Returns:
|
||||
Tuple of (agent_config, schedule_config)
|
||||
|
||||
Raises:
|
||||
FileNotFoundError: If YAML file doesn't exist
|
||||
ValueError: If YAML is invalid or missing required fields
|
||||
|
||||
Example YAML structure:
|
||||
framework: praisonai
|
||||
|
||||
agents:
|
||||
- name: "AI News Monitor"
|
||||
role: "News Analyst"
|
||||
goal: "Monitor AI news"
|
||||
instructions: "Search and summarize"
|
||||
tools:
|
||||
- search_tool
|
||||
verbose: true
|
||||
|
||||
task: "Search for latest AI news"
|
||||
|
||||
schedule:
|
||||
interval: "hourly"
|
||||
max_retries: 3
|
||||
run_immediately: true
|
||||
"""
|
||||
yaml_path = Path(yaml_path)
|
||||
|
||||
if not yaml_path.exists():
|
||||
raise FileNotFoundError(f"YAML file not found: {yaml_path}")
|
||||
|
||||
try:
|
||||
with open(yaml_path, 'r') as f:
|
||||
config = yaml.safe_load(f)
|
||||
except yaml.YAMLError as e:
|
||||
raise ValueError(f"Invalid YAML file: {e}")
|
||||
|
||||
if not config:
|
||||
raise ValueError("Empty YAML file")
|
||||
|
||||
# Extract agent configuration
|
||||
agent_config = {}
|
||||
|
||||
# Get agents list (can be single agent or multiple)
|
||||
agents = config.get('agents', [])
|
||||
if not agents:
|
||||
raise ValueError("No agents defined in YAML file")
|
||||
|
||||
# Use first agent for scheduling (can be extended for multi-agent later)
|
||||
if isinstance(agents, list):
|
||||
agent_config = agents[0]
|
||||
else:
|
||||
agent_config = agents
|
||||
|
||||
# Get task
|
||||
task = config.get('task', '')
|
||||
if not task:
|
||||
# Try to get from agent's tasks
|
||||
if 'tasks' in agent_config and agent_config['tasks']:
|
||||
first_task = list(agent_config['tasks'].values())[0]
|
||||
task = first_task.get('description', '')
|
||||
|
||||
agent_config['task'] = task
|
||||
|
||||
# Get framework (default to praisonai)
|
||||
agent_config['framework'] = config.get('framework', 'praisonai')
|
||||
|
||||
# Extract schedule configuration (optional)
|
||||
schedule_section = config.get('schedule', {})
|
||||
|
||||
# Set defaults for schedule if not provided
|
||||
schedule_config = {
|
||||
'interval': schedule_section.get('interval', 'hourly'),
|
||||
'max_retries': schedule_section.get('max_retries', 3),
|
||||
'run_immediately': schedule_section.get('run_immediately', False),
|
||||
'timeout': schedule_section.get('timeout'), # Optional timeout in seconds
|
||||
'max_cost': schedule_section.get('max_cost', 1.00) # Default $1.00 budget limit for safety
|
||||
}
|
||||
|
||||
logger.info(f"Loaded agent '{agent_config.get('name', 'Unknown')}' with schedule interval '{schedule_config['interval']}'")
|
||||
|
||||
return agent_config, schedule_config
|
||||
|
||||
|
||||
def create_agent_from_config(agent_config: Dict[str, Any]) -> Any:
|
||||
"""
|
||||
Create a PraisonAI Agent instance from configuration.
|
||||
|
||||
Args:
|
||||
agent_config: Agent configuration dictionary
|
||||
|
||||
Returns:
|
||||
Agent instance
|
||||
|
||||
Raises:
|
||||
ImportError: If praisonaiagents is not installed
|
||||
"""
|
||||
try:
|
||||
from praisonaiagents import Agent
|
||||
except ImportError:
|
||||
raise ImportError("praisonaiagents package is required. Install with: pip install praisonaiagents")
|
||||
|
||||
# Extract agent parameters
|
||||
name = agent_config.get('name', 'Scheduled Agent')
|
||||
role = agent_config.get('role', '')
|
||||
goal = agent_config.get('goal', '')
|
||||
instructions = agent_config.get('instructions', agent_config.get('backstory', ''))
|
||||
verbose = agent_config.get('verbose', False)
|
||||
|
||||
# Handle tools
|
||||
tools = []
|
||||
tool_names = agent_config.get('tools', [])
|
||||
|
||||
if tool_names:
|
||||
# Try to import tools
|
||||
for tool_name in tool_names:
|
||||
try:
|
||||
# Try common tool imports
|
||||
if tool_name == 'search_tool' or tool_name == 'InternetSearchTool':
|
||||
try:
|
||||
from tools import search_tool
|
||||
tools.append(search_tool)
|
||||
except ImportError:
|
||||
logger.warning(f"Could not import {tool_name}, skipping")
|
||||
# Add more tool imports as needed
|
||||
except Exception as e:
|
||||
logger.warning(f"Error loading tool {tool_name}: {e}")
|
||||
|
||||
# Create agent
|
||||
agent = Agent(
|
||||
name=name,
|
||||
role=role,
|
||||
goal=goal,
|
||||
instructions=instructions,
|
||||
tools=tools,
|
||||
verbose=verbose
|
||||
)
|
||||
|
||||
return agent
|
||||
|
||||
|
||||
def validate_schedule_config(schedule_config: Dict[str, Any]) -> None:
|
||||
"""
|
||||
Validate schedule configuration.
|
||||
|
||||
Args:
|
||||
schedule_config: Schedule configuration dictionary
|
||||
|
||||
Raises:
|
||||
ValueError: If configuration is invalid
|
||||
"""
|
||||
required_fields = ['interval']
|
||||
for field in required_fields:
|
||||
if field not in schedule_config:
|
||||
raise ValueError(f"Missing required schedule field: {field}")
|
||||
|
||||
# Validate interval format
|
||||
interval = schedule_config['interval']
|
||||
valid_intervals = ['hourly', 'daily']
|
||||
|
||||
if interval not in valid_intervals:
|
||||
# Check if it's a custom format (*/Nm, */Nh, */Ns, or plain number)
|
||||
if not (interval.startswith('*/') or interval.isdigit()):
|
||||
raise ValueError(
|
||||
f"Invalid interval format: {interval}. "
|
||||
f"Use 'hourly', 'daily', '*/30m', '*/6h', or seconds as number"
|
||||
)
|
||||
|
||||
# Validate max_retries
|
||||
max_retries = schedule_config.get('max_retries', 3)
|
||||
if not isinstance(max_retries, int) or max_retries < 0:
|
||||
raise ValueError(f"max_retries must be a non-negative integer, got: {max_retries}")
|
||||
@@ -1 +1 @@
|
||||
__version__ = "2.3.52"
|
||||
__version__ = "2.3.77"
|
||||
@@ -12,7 +12,7 @@ dependencies = [
|
||||
"rich>=13.7",
|
||||
"markdown>=3.5",
|
||||
"pyparsing>=3.0.0",
|
||||
"praisonaiagents>=0.1.17",
|
||||
"praisonaiagents>=0.1.24",
|
||||
"python-dotenv>=0.19.0",
|
||||
"instructor>=1.3.3",
|
||||
"PyYAML>=6.0",
|
||||
|
||||
@@ -83,13 +83,19 @@ def setup_test_environment(request):
|
||||
# Real tests (marked with @pytest.mark.real) should use actual environment variables
|
||||
is_real_test = False
|
||||
|
||||
# Check if this test is marked as a real test
|
||||
# Check if this test is marked as a real test or integration test
|
||||
if hasattr(request, 'node') and hasattr(request.node, 'iter_markers'):
|
||||
for marker in request.node.iter_markers():
|
||||
if marker.name == 'real':
|
||||
if marker.name in ('real', 'integration'):
|
||||
is_real_test = True
|
||||
break
|
||||
|
||||
# Also check if the test file is in the integration directory
|
||||
if hasattr(request, 'fspath') and request.fspath:
|
||||
test_path = str(request.fspath)
|
||||
if '/integration/' in test_path or '\\integration\\' in test_path:
|
||||
is_real_test = True
|
||||
|
||||
# Store original values to restore later
|
||||
original_values = {}
|
||||
|
||||
|
||||
Loaded 100 of 119 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user